diff --git a/venv/lib/python3.10/site-packages/accelerate-1.11.0.dist-info/INSTALLER b/venv/lib/python3.10/site-packages/accelerate-1.11.0.dist-info/INSTALLER
new file mode 100644
index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/accelerate-1.11.0.dist-info/INSTALLER
@@ -0,0 +1 @@
+pip
diff --git a/venv/lib/python3.10/site-packages/accelerate-1.11.0.dist-info/METADATA b/venv/lib/python3.10/site-packages/accelerate-1.11.0.dist-info/METADATA
new file mode 100644
index 0000000000000000000000000000000000000000..5f62c40714289dd03d3ead4c74e99b807baf35b2
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/accelerate-1.11.0.dist-info/METADATA
@@ -0,0 +1,392 @@
+Metadata-Version: 2.4
+Name: accelerate
+Version: 1.11.0
+Summary: Accelerate
+Home-page: https://github.com/huggingface/accelerate
+Author: The HuggingFace team
+Author-email: zach.mueller@huggingface.co
+License: Apache
+Keywords: deep learning
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Education
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.10.0
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy>=1.17
+Requires-Dist: packaging>=20.0
+Requires-Dist: psutil
+Requires-Dist: pyyaml
+Requires-Dist: torch>=2.0.0
+Requires-Dist: huggingface_hub>=0.21.0
+Requires-Dist: safetensors>=0.4.3
+Provides-Extra: quality
+Requires-Dist: ruff==0.13.1; extra == "quality"
+Provides-Extra: docs
+Provides-Extra: test-prod
+Requires-Dist: pytest>=7.2.0; extra == "test-prod"
+Requires-Dist: pytest-xdist; extra == "test-prod"
+Requires-Dist: pytest-subtests; extra == "test-prod"
+Requires-Dist: parameterized; extra == "test-prod"
+Requires-Dist: pytest-order; extra == "test-prod"
+Provides-Extra: test-dev
+Requires-Dist: datasets; extra == "test-dev"
+Requires-Dist: diffusers; extra == "test-dev"
+Requires-Dist: evaluate; extra == "test-dev"
+Requires-Dist: torchdata>=0.8.0; extra == "test-dev"
+Requires-Dist: torchpippy>=0.2.0; extra == "test-dev"
+Requires-Dist: transformers; extra == "test-dev"
+Requires-Dist: scipy; extra == "test-dev"
+Requires-Dist: scikit-learn; extra == "test-dev"
+Requires-Dist: tqdm; extra == "test-dev"
+Requires-Dist: bitsandbytes; extra == "test-dev"
+Requires-Dist: timm; extra == "test-dev"
+Provides-Extra: testing
+Requires-Dist: pytest>=7.2.0; extra == "testing"
+Requires-Dist: pytest-xdist; extra == "testing"
+Requires-Dist: pytest-subtests; extra == "testing"
+Requires-Dist: parameterized; extra == "testing"
+Requires-Dist: pytest-order; extra == "testing"
+Requires-Dist: datasets; extra == "testing"
+Requires-Dist: diffusers; extra == "testing"
+Requires-Dist: evaluate; extra == "testing"
+Requires-Dist: torchdata>=0.8.0; extra == "testing"
+Requires-Dist: torchpippy>=0.2.0; extra == "testing"
+Requires-Dist: transformers; extra == "testing"
+Requires-Dist: scipy; extra == "testing"
+Requires-Dist: scikit-learn; extra == "testing"
+Requires-Dist: tqdm; extra == "testing"
+Requires-Dist: bitsandbytes; extra == "testing"
+Requires-Dist: timm; extra == "testing"
+Provides-Extra: deepspeed
+Requires-Dist: deepspeed; extra == "deepspeed"
+Provides-Extra: rich
+Requires-Dist: rich; extra == "rich"
+Provides-Extra: test-fp8
+Requires-Dist: torchao; extra == "test-fp8"
+Provides-Extra: test-trackers
+Requires-Dist: wandb; extra == "test-trackers"
+Requires-Dist: comet-ml; extra == "test-trackers"
+Requires-Dist: tensorboard; extra == "test-trackers"
+Requires-Dist: dvclive; extra == "test-trackers"
+Requires-Dist: matplotlib; extra == "test-trackers"
+Requires-Dist: swanlab[dashboard]; extra == "test-trackers"
+Requires-Dist: trackio; extra == "test-trackers"
+Provides-Extra: dev
+Requires-Dist: ruff==0.13.1; extra == "dev"
+Requires-Dist: pytest>=7.2.0; extra == "dev"
+Requires-Dist: pytest-xdist; extra == "dev"
+Requires-Dist: pytest-subtests; extra == "dev"
+Requires-Dist: parameterized; extra == "dev"
+Requires-Dist: pytest-order; extra == "dev"
+Requires-Dist: datasets; extra == "dev"
+Requires-Dist: diffusers; extra == "dev"
+Requires-Dist: evaluate; extra == "dev"
+Requires-Dist: torchdata>=0.8.0; extra == "dev"
+Requires-Dist: torchpippy>=0.2.0; extra == "dev"
+Requires-Dist: transformers; extra == "dev"
+Requires-Dist: scipy; extra == "dev"
+Requires-Dist: scikit-learn; extra == "dev"
+Requires-Dist: tqdm; extra == "dev"
+Requires-Dist: bitsandbytes; extra == "dev"
+Requires-Dist: timm; extra == "dev"
+Requires-Dist: rich; extra == "dev"
+Provides-Extra: sagemaker
+Requires-Dist: sagemaker; extra == "sagemaker"
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license
+Dynamic: license-file
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+
+
+
+
+ 
+
+
+
+
+
+ 
+ 
+ 
+ 
+
+
+
+
Run your *raw* PyTorch training script on any kind of device
+
+
+
+ 
+
+
+## Easy to integrate
+
+🤗 Accelerate was created for PyTorch users who like to write the training loop of PyTorch models but are reluctant to write and maintain the boilerplate code needed to use multi-GPUs/TPU/fp16.
+
+🤗 Accelerate abstracts exactly and only the boilerplate code related to multi-GPUs/TPU/fp16 and leaves the rest of your code unchanged.
+
+Here is an example:
+
+```diff
+ import torch
+ import torch.nn.functional as F
+ from datasets import load_dataset
++ from accelerate import Accelerator
+
++ accelerator = Accelerator()
+- device = 'cpu'
++ device = accelerator.device
+
+ model = torch.nn.Transformer().to(device)
+ optimizer = torch.optim.Adam(model.parameters())
+
+ dataset = load_dataset('my_dataset')
+ data = torch.utils.data.DataLoader(dataset, shuffle=True)
+
++ model, optimizer, data = accelerator.prepare(model, optimizer, data)
+
+ model.train()
+ for epoch in range(10):
+ for source, targets in data:
+ source = source.to(device)
+ targets = targets.to(device)
+
+ optimizer.zero_grad()
+
+ output = model(source)
+ loss = F.cross_entropy(output, targets)
+
+- loss.backward()
++ accelerator.backward(loss)
+
+ optimizer.step()
+```
+
+As you can see in this example, by adding 5-lines to any standard PyTorch training script you can now run on any kind of single or distributed node setting (single CPU, single GPU, multi-GPUs and TPUs) as well as with or without mixed precision (fp8, fp16, bf16).
+
+In particular, the same code can then be run without modification on your local machine for debugging or your training environment.
+
+🤗 Accelerate even handles the device placement for you (which requires a few more changes to your code, but is safer in general), so you can even simplify your training loop further:
+
+```diff
+ import torch
+ import torch.nn.functional as F
+ from datasets import load_dataset
++ from accelerate import Accelerator
+
+- device = 'cpu'
++ accelerator = Accelerator()
+
+- model = torch.nn.Transformer().to(device)
++ model = torch.nn.Transformer()
+ optimizer = torch.optim.Adam(model.parameters())
+
+ dataset = load_dataset('my_dataset')
+ data = torch.utils.data.DataLoader(dataset, shuffle=True)
+
++ model, optimizer, data = accelerator.prepare(model, optimizer, data)
+
+ model.train()
+ for epoch in range(10):
+ for source, targets in data:
+- source = source.to(device)
+- targets = targets.to(device)
+
+ optimizer.zero_grad()
+
+ output = model(source)
+ loss = F.cross_entropy(output, targets)
+
+- loss.backward()
++ accelerator.backward(loss)
+
+ optimizer.step()
+```
+
+Want to learn more? Check out the [documentation](https://huggingface.co/docs/accelerate) or have a look at our [examples](https://github.com/huggingface/accelerate/tree/main/examples).
+
+## Launching script
+
+🤗 Accelerate also provides an optional CLI tool that allows you to quickly configure and test your training environment before launching the scripts. No need to remember how to use `torch.distributed.run` or to write a specific launcher for TPU training!
+On your machine(s) just run:
+
+```bash
+accelerate config
+```
+
+and answer the questions asked. This will generate a config file that will be used automatically to properly set the default options when doing
+
+```bash
+accelerate launch my_script.py --args_to_my_script
+```
+
+For instance, here is how you would run the GLUE example on the MRPC task (from the root of the repo):
+
+```bash
+accelerate launch examples/nlp_example.py
+```
+
+This CLI tool is **optional**, and you can still use `python my_script.py` or `python -m torchrun my_script.py` at your convenience.
+
+You can also directly pass in the arguments you would to `torchrun` as arguments to `accelerate launch` if you wish to not run` accelerate config`.
+
+For example, here is how to launch on two GPUs:
+
+```bash
+accelerate launch --multi_gpu --num_processes 2 examples/nlp_example.py
+```
+
+To learn more, check the CLI documentation available [here](https://huggingface.co/docs/accelerate/package_reference/cli).
+
+Or view the configuration zoo [here](https://github.com/huggingface/accelerate/blob/main/examples/config_yaml_templates/)
+
+## Launching multi-CPU run using MPI
+
+🤗 Here is another way to launch multi-CPU run using MPI. You can learn how to install Open MPI on [this page](https://www.open-mpi.org/faq/?category=building#easy-build). You can use Intel MPI or MVAPICH as well.
+Once you have MPI setup on your cluster, just run:
+```bash
+accelerate config
+```
+Answer the questions that are asked, selecting to run using multi-CPU, and answer "yes" when asked if you want accelerate to launch mpirun.
+Then, use `accelerate launch` with your script like:
+```bash
+accelerate launch examples/nlp_example.py
+```
+Alternatively, you can use mpirun directly, without using the CLI like:
+```bash
+mpirun -np 2 python examples/nlp_example.py
+```
+
+## Launching training using DeepSpeed
+
+🤗 Accelerate supports training on single/multiple GPUs using DeepSpeed. To use it, you don't need to change anything in your training code; you can set everything using just `accelerate config`. However, if you desire to tweak your DeepSpeed related args from your Python script, we provide you the `DeepSpeedPlugin`.
+
+```python
+from accelerate import Accelerator, DeepSpeedPlugin
+
+# deepspeed needs to know your gradient accumulation steps beforehand, so don't forget to pass it
+# Remember you still need to do gradient accumulation by yourself, just like you would have done without deepspeed
+deepspeed_plugin = DeepSpeedPlugin(zero_stage=2, gradient_accumulation_steps=2)
+accelerator = Accelerator(mixed_precision='fp16', deepspeed_plugin=deepspeed_plugin)
+
+# How to save your 🤗 Transformer?
+accelerator.wait_for_everyone()
+unwrapped_model = accelerator.unwrap_model(model)
+unwrapped_model.save_pretrained(save_dir, save_function=accelerator.save, state_dict=accelerator.get_state_dict(model))
+```
+
+Note: DeepSpeed support is experimental for now. In case you get into some problem, please open an issue.
+
+## Launching your training from a notebook
+
+🤗 Accelerate also provides a `notebook_launcher` function you can use in a notebook to launch a distributed training. This is especially useful for Colab or Kaggle notebooks with a TPU backend. Just define your training loop in a `training_function` then in your last cell, add:
+
+```python
+from accelerate import notebook_launcher
+
+notebook_launcher(training_function)
+```
+
+An example can be found in [this notebook](https://github.com/huggingface/notebooks/blob/main/examples/accelerate_examples/simple_nlp_example.ipynb). [](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/accelerate_examples/simple_nlp_example.ipynb)
+
+## Why should I use 🤗 Accelerate?
+
+You should use 🤗 Accelerate when you want to easily run your training scripts in a distributed environment without having to renounce full control over your training loop. This is not a high-level framework above PyTorch, just a thin wrapper so you don't have to learn a new library. In fact, the whole API of 🤗 Accelerate is in one class, the `Accelerator` object.
+
+## Why shouldn't I use 🤗 Accelerate?
+
+You shouldn't use 🤗 Accelerate if you don't want to write a training loop yourself. There are plenty of high-level libraries above PyTorch that will offer you that, 🤗 Accelerate is not one of them.
+
+## Frameworks using 🤗 Accelerate
+
+If you like the simplicity of 🤗 Accelerate but would prefer a higher-level abstraction around its capabilities, some frameworks and libraries that are built on top of 🤗 Accelerate are listed below:
+
+* [Amphion](https://github.com/open-mmlab/Amphion) is a toolkit for Audio, Music, and Speech Generation. Its purpose is to support reproducible research and help junior researchers and engineers get started in the field of audio, music, and speech generation research and development.
+* [Animus](https://github.com/Scitator/animus) is a minimalistic framework to run machine learning experiments. Animus highlights common "breakpoints" in ML experiments and provides a unified interface for them within [IExperiment](https://github.com/Scitator/animus/blob/main/animus/core.py#L76).
+* [Catalyst](https://github.com/catalyst-team/catalyst#getting-started) is a PyTorch framework for Deep Learning Research and Development. It focuses on reproducibility, rapid experimentation, and codebase reuse so you can create something new rather than write yet another train loop. Catalyst provides a [Runner](https://catalyst-team.github.io/catalyst/api/core.html#runner) to connect all parts of the experiment: hardware backend, data transformations, model training, and inference logic.
+* [fastai](https://github.com/fastai/fastai#installing) is a PyTorch framework for Deep Learning that simplifies training fast and accurate neural nets using modern best practices. fastai provides a [Learner](https://docs.fast.ai/learner.html#Learner) to handle the training, fine-tuning, and inference of deep learning algorithms.
+* [Finetuner](https://github.com/jina-ai/finetuner) is a service that enables models to create higher-quality embeddings for semantic search, visual similarity search, cross-modal text<->image search, recommendation systems, clustering, duplication detection, anomaly detection, or other uses.
+* [InvokeAI](https://github.com/invoke-ai/InvokeAI) is a creative engine for Stable Diffusion models, offering industry-leading WebUI, terminal usage support, and serves as the foundation for many commercial products.
+* [Kornia](https://kornia.readthedocs.io/en/latest/get-started/introduction.html) is a differentiable library that allows classical computer vision to be integrated into deep learning models. Kornia provides a [Trainer](https://kornia.readthedocs.io/en/latest/x.html#kornia.x.Trainer) with the specific purpose to train and fine-tune the supported deep learning algorithms within the library.
+* [Open Assistant](https://projects.laion.ai/Open-Assistant/) is a chat-based assistant that understands tasks, can interact with their party systems, and retrieve information dynamically to do so.
+* [pytorch-accelerated](https://github.com/Chris-hughes10/pytorch-accelerated) is a lightweight training library, with a streamlined feature set centered around a general-purpose [Trainer](https://pytorch-accelerated.readthedocs.io/en/latest/trainer.html), that places a huge emphasis on simplicity and transparency; enabling users to understand exactly what is going on under the hood, but without having to write and maintain the boilerplate themselves!
+* [Stable Diffusion web UI](https://github.com/AUTOMATIC1111/stable-diffusion-webui) is an open-source browser-based easy-to-use interface based on the Gradio library for Stable Diffusion.
+* [torchkeras](https://github.com/lyhue1991/torchkeras) is a simple tool for training pytorch model just in a keras style, a dynamic and beautiful plot is provided in notebook to monitor your loss or metric.
+* [transformers](https://github.com/huggingface/transformers) as a tool for helping train state-of-the-art machine learning models in PyTorch, Tensorflow, and JAX. (Accelerate is the backend for the PyTorch side).
+
+
+## Installation
+
+This repository is tested on Python 3.8+ and PyTorch 1.10.0+
+
+You should install 🤗 Accelerate in a [virtual environment](https://docs.python.org/3/library/venv.html). If you're unfamiliar with Python virtual environments, check out the [user guide](https://packaging.python.org/guides/installing-using-pip-and-virtual-environments/).
+
+First, create a virtual environment with the version of Python you're going to use and activate it.
+
+Then, you will need to install PyTorch: refer to the [official installation page](https://pytorch.org/get-started/locally/#start-locally) regarding the specific install command for your platform. Then 🤗 Accelerate can be installed using pip as follows:
+
+```bash
+pip install accelerate
+```
+
+## Supported integrations
+
+- CPU only
+- multi-CPU on one node (machine)
+- multi-CPU on several nodes (machines)
+- single GPU
+- multi-GPU on one node (machine)
+- multi-GPU on several nodes (machines)
+- TPU
+- FP16/BFloat16 mixed precision
+- FP8 mixed precision with [Transformer Engine](https://github.com/NVIDIA/TransformerEngine) or [MS-AMP](https://github.com/Azure/MS-AMP/)
+- DeepSpeed support (Experimental)
+- PyTorch Fully Sharded Data Parallel (FSDP) support (Experimental)
+- Megatron-LM support (Experimental)
+
+## Citing 🤗 Accelerate
+
+If you use 🤗 Accelerate in your publication, please cite it by using the following BibTeX entry.
+
+```bibtex
+@Misc{accelerate,
+ title = {Accelerate: Training and inference at scale made simple, efficient and adaptable.},
+ author = {Sylvain Gugger and Lysandre Debut and Thomas Wolf and Philipp Schmid and Zachary Mueller and Sourab Mangrulkar and Marc Sun and Benjamin Bossan},
+ howpublished = {\url{https://github.com/huggingface/accelerate}},
+ year = {2022}
+}
+```
diff --git a/venv/lib/python3.10/site-packages/accelerate-1.11.0.dist-info/RECORD b/venv/lib/python3.10/site-packages/accelerate-1.11.0.dist-info/RECORD
new file mode 100644
index 0000000000000000000000000000000000000000..3529c8d563ca38a119d5689549f3ac4794553396
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/accelerate-1.11.0.dist-info/RECORD
@@ -0,0 +1,179 @@
+../../../bin/accelerate,sha256=G_VAPCbL7YjEXtT1Tx3aQGhOzs-uV0SmWu4RdbtXwFc,231
+../../../bin/accelerate-config,sha256=JUWzLYVo9EEHw0kvgd6ZqXR1h6YJ4_th7I_Q-XdA__4,223
+../../../bin/accelerate-estimate-memory,sha256=OwMLazPH_3lMUGRmNYA0KQTpjSvhdTqR1kuuKk1HUxM,225
+../../../bin/accelerate-launch,sha256=SYW-uJX-5fgQooK5Bk2BSUJRAbiIgb2waqCSr1_deug,223
+../../../bin/accelerate-merge-weights,sha256=L4nxyUiqplw2-XyLqntbHZNHIY0BRHG69zoeJsaHRCs,222
+accelerate-1.11.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
+accelerate-1.11.0.dist-info/METADATA,sha256=4a-LH6zZjcoQHBVLMOTq1iEEhB9UnKGaQYBUsrwtWTo,19601
+accelerate-1.11.0.dist-info/RECORD,,
+accelerate-1.11.0.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+accelerate-1.11.0.dist-info/WHEEL,sha256=lTU6B6eIfYoiQJTZNc-fyaR6BpL6ehTzU3xGYxn2n8k,91
+accelerate-1.11.0.dist-info/entry_points.txt,sha256=Vpy8gUGfZ-1VnM2229fb8CpJNLBdMH_wtJ9PQ7b_2tQ,296
+accelerate-1.11.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+accelerate-1.11.0.dist-info/top_level.txt,sha256=esVfdxTidsjQ90zsN_rPpjLFJ4ijRlx4mnLrG09hlt4,11
+accelerate/__init__.py,sha256=zKhkXV3sUdtIkC5HSDDpUd7xUmi_y8a0W94KHHbbHos,1555
+accelerate/__pycache__/__init__.cpython-310.pyc,,
+accelerate/__pycache__/accelerator.cpython-310.pyc,,
+accelerate/__pycache__/big_modeling.cpython-310.pyc,,
+accelerate/__pycache__/checkpointing.cpython-310.pyc,,
+accelerate/__pycache__/data_loader.cpython-310.pyc,,
+accelerate/__pycache__/hooks.cpython-310.pyc,,
+accelerate/__pycache__/inference.cpython-310.pyc,,
+accelerate/__pycache__/launchers.cpython-310.pyc,,
+accelerate/__pycache__/local_sgd.cpython-310.pyc,,
+accelerate/__pycache__/logging.cpython-310.pyc,,
+accelerate/__pycache__/memory_utils.cpython-310.pyc,,
+accelerate/__pycache__/optimizer.cpython-310.pyc,,
+accelerate/__pycache__/parallelism_config.cpython-310.pyc,,
+accelerate/__pycache__/scheduler.cpython-310.pyc,,
+accelerate/__pycache__/state.cpython-310.pyc,,
+accelerate/__pycache__/tracking.cpython-310.pyc,,
+accelerate/accelerator.py,sha256=2NaFKC9r7op9G4rT2kttFxFPCPvj_gHepiG2DITIL8s,191215
+accelerate/big_modeling.py,sha256=s434XS4IHvidURez-iOKApy5V1DkYmhstHSr3LaB6lo,36931
+accelerate/checkpointing.py,sha256=clOiZp0pLht2Rtkjp2UFUSOdow3hVrG8Po8xXtIXqY0,13995
+accelerate/commands/__init__.py,sha256=m1PPTDT4ziIAvM0-FDSgIMIZ69Konn126s6LwuzH6v8,606
+accelerate/commands/__pycache__/__init__.cpython-310.pyc,,
+accelerate/commands/__pycache__/accelerate_cli.cpython-310.pyc,,
+accelerate/commands/__pycache__/env.cpython-310.pyc,,
+accelerate/commands/__pycache__/estimate.cpython-310.pyc,,
+accelerate/commands/__pycache__/launch.cpython-310.pyc,,
+accelerate/commands/__pycache__/merge.cpython-310.pyc,,
+accelerate/commands/__pycache__/test.cpython-310.pyc,,
+accelerate/commands/__pycache__/to_fsdp2.cpython-310.pyc,,
+accelerate/commands/__pycache__/tpu.cpython-310.pyc,,
+accelerate/commands/__pycache__/utils.cpython-310.pyc,,
+accelerate/commands/accelerate_cli.py,sha256=SkwFad6Z1ZsGjtm7TiXFq8je-akshp_0WxX_6rGSBw8,1972
+accelerate/commands/config/__init__.py,sha256=iJK8dgj3pc5Vdr1E7UuGoFu-BlybyXLxYDoTg9gXngE,1645
+accelerate/commands/config/__pycache__/__init__.cpython-310.pyc,,
+accelerate/commands/config/__pycache__/cluster.cpython-310.pyc,,
+accelerate/commands/config/__pycache__/config.cpython-310.pyc,,
+accelerate/commands/config/__pycache__/config_args.cpython-310.pyc,,
+accelerate/commands/config/__pycache__/config_utils.cpython-310.pyc,,
+accelerate/commands/config/__pycache__/default.cpython-310.pyc,,
+accelerate/commands/config/__pycache__/sagemaker.cpython-310.pyc,,
+accelerate/commands/config/__pycache__/update.cpython-310.pyc,,
+accelerate/commands/config/cluster.py,sha256=aHlkL4o2BH3h2w5RCL3IUCKH37fvl9vGWx01up_ltvE,39376
+accelerate/commands/config/config.py,sha256=FuRlQvOjgATEtyqOSsGD-KEtOCvACOHjs2C-krrtldk,3035
+accelerate/commands/config/config_args.py,sha256=b-_GLJKM2_7w84Zjf6PSWV5is-VUHmG0KVE8XeWupBU,10275
+accelerate/commands/config/config_utils.py,sha256=lbV9iKAVSF5bUyQDGpzFduhH5JENYgAilPpRLLlCEq8,3299
+accelerate/commands/config/default.py,sha256=UY0dQ7w1peBkQe8QDCm3XJrTnsRcm_G-_hf44NzNAbU,6213
+accelerate/commands/config/sagemaker.py,sha256=SRLup9XrV_Z-23A3d8gZbkqbonDcscMDDmbpmytCzbY,10636
+accelerate/commands/config/update.py,sha256=0Et2qKKgoeisVQixsJKSqp4dLzSofbZ2jXmAPgJP-DQ,2396
+accelerate/commands/env.py,sha256=gRGwtPhMMWmZ5NTMaMrgHXaOzBV6SvAtP78TuuJvC6g,4336
+accelerate/commands/estimate.py,sha256=4lpX89bh7PmWlhM9uh5LkTGazplZZAWn2v7VvbHOF3k,12650
+accelerate/commands/launch.py,sha256=ZxtKJkcuU7vY8Acptf1iY9SYUcCQ1SbVMM8ZgKl9PhQ,48389
+accelerate/commands/menu/__init__.py,sha256=uqSlBM0TFHBwzdv3p3SXfpAk1lZFp4h1a7mbBdscPHs,645
+accelerate/commands/menu/__pycache__/__init__.cpython-310.pyc,,
+accelerate/commands/menu/__pycache__/cursor.cpython-310.pyc,,
+accelerate/commands/menu/__pycache__/helpers.cpython-310.pyc,,
+accelerate/commands/menu/__pycache__/input.cpython-310.pyc,,
+accelerate/commands/menu/__pycache__/keymap.cpython-310.pyc,,
+accelerate/commands/menu/__pycache__/selection_menu.cpython-310.pyc,,
+accelerate/commands/menu/cursor.py,sha256=-lmpJVAzvNc0c3EOtSuLoKB59zqylVCbYyWLPnrOmvQ,2028
+accelerate/commands/menu/helpers.py,sha256=KrSB5fJjH4MUEUAQJ6bYaN16AYcnl9UalDrPD3DYeeg,1483
+accelerate/commands/menu/input.py,sha256=T8Mdd-Y_OURgqfDV9qZh4Wf6hmT22AneNtJzj4JA1Rk,2512
+accelerate/commands/menu/keymap.py,sha256=eXj-suyYs1m5dEHoUKN4mKAMLc8DWHnwhP6G6JSU0jQ,4086
+accelerate/commands/menu/selection_menu.py,sha256=iylPHUmEemEhn6gwidbgdTGOrCJiR3Il1EjqiYbnues,4959
+accelerate/commands/merge.py,sha256=3Uo14LADvk8RJwZ_i6kIsGwWB3Ul5y8Uq47oHkDYqKU,2387
+accelerate/commands/test.py,sha256=YrPYEaAACOGZ6btn2MV6NbMSEdBUcMWADLbQWaZSHtk,2149
+accelerate/commands/to_fsdp2.py,sha256=TWGc7mC4rzjQmFp2mwUEUiidrPWP9PLdM0Ipm0oEF8o,5992
+accelerate/commands/tpu.py,sha256=KyxDP7IuveidZrbW4rx2s8Ku3o_ptI6tzwr_R7ck0os,5548
+accelerate/commands/utils.py,sha256=aT8xUCe2pCkFII7yZxcfaohEjgBAzMUM7WiD4UuWSOY,4150
+accelerate/data_loader.py,sha256=jev7IMauWO_zpGbP1nJEoSzIuNPeADFqbB1UsDEO5so,65614
+accelerate/hooks.py,sha256=GEGP0GNpkcdcyrqpXTjezMcILLNO7TCABOCAr9fKgO0,33855
+accelerate/inference.py,sha256=8M1LRCfXSstOUhS8O6UlCipQ14cTQgvAqWqDXQEQn50,7684
+accelerate/launchers.py,sha256=gKB8sH5YwV-hOM8ptbzHgm16729X6864vKdLOls4eUY,14176
+accelerate/local_sgd.py,sha256=aCj_yqXK_FhhZRWEpzXIkgXBERH6fC3HyrC3nsOj1uA,4160
+accelerate/logging.py,sha256=rpXi7zO4cKnt72cc_R3WS2xPyL_gZWzd_0vWBp2hTac,5080
+accelerate/memory_utils.py,sha256=3R5LoeHl6GgTZ-IMPrDZMdaEehWarGdPqODushb-6pg,862
+accelerate/optimizer.py,sha256=p5qfCOMi8DAjvx6eECypo4ej3pvmQ_U_bgmrEAKc4hY,8158
+accelerate/parallelism_config.py,sha256=qnyu_RVa0bEqd4Zg25qTMX4yMeVt9GoS_4CfRRFhJMI,13290
+accelerate/scheduler.py,sha256=des_4M_Tt1W8gCYZZbLla0GHBEgJY3Wx2EGBQPTzeiY,4238
+accelerate/state.py,sha256=b0uOBbXyVoC5Pzi_8g4XjSPTYFqy8ZfKeHLmgslnQpI,58276
+accelerate/test_utils/__init__.py,sha256=E347JKLya-_vi4IbWhdsAVQ0FCudtcaD7ErX2bh6RC0,1826
+accelerate/test_utils/__pycache__/__init__.cpython-310.pyc,,
+accelerate/test_utils/__pycache__/examples.cpython-310.pyc,,
+accelerate/test_utils/__pycache__/testing.cpython-310.pyc,,
+accelerate/test_utils/__pycache__/training.cpython-310.pyc,,
+accelerate/test_utils/examples.py,sha256=kSXJrx6OhUghHGzqLnCwqdsDUufMjmB2DPQyQSfZHvQ,7301
+accelerate/test_utils/scripts/__init__.py,sha256=m1PPTDT4ziIAvM0-FDSgIMIZ69Konn126s6LwuzH6v8,606
+accelerate/test_utils/scripts/__pycache__/__init__.cpython-310.pyc,,
+accelerate/test_utils/scripts/__pycache__/test_cli.cpython-310.pyc,,
+accelerate/test_utils/scripts/__pycache__/test_ddp_comm_hook.cpython-310.pyc,,
+accelerate/test_utils/scripts/__pycache__/test_distributed_data_loop.cpython-310.pyc,,
+accelerate/test_utils/scripts/__pycache__/test_merge_weights.cpython-310.pyc,,
+accelerate/test_utils/scripts/__pycache__/test_notebook.cpython-310.pyc,,
+accelerate/test_utils/scripts/__pycache__/test_ops.cpython-310.pyc,,
+accelerate/test_utils/scripts/__pycache__/test_script.cpython-310.pyc,,
+accelerate/test_utils/scripts/__pycache__/test_sync.cpython-310.pyc,,
+accelerate/test_utils/scripts/external_deps/__init__.py,sha256=m1PPTDT4ziIAvM0-FDSgIMIZ69Konn126s6LwuzH6v8,606
+accelerate/test_utils/scripts/external_deps/__pycache__/__init__.cpython-310.pyc,,
+accelerate/test_utils/scripts/external_deps/__pycache__/test_checkpointing.cpython-310.pyc,,
+accelerate/test_utils/scripts/external_deps/__pycache__/test_ds_multiple_model.cpython-310.pyc,,
+accelerate/test_utils/scripts/external_deps/__pycache__/test_metrics.cpython-310.pyc,,
+accelerate/test_utils/scripts/external_deps/__pycache__/test_peak_memory_usage.cpython-310.pyc,,
+accelerate/test_utils/scripts/external_deps/__pycache__/test_performance.cpython-310.pyc,,
+accelerate/test_utils/scripts/external_deps/__pycache__/test_pippy.cpython-310.pyc,,
+accelerate/test_utils/scripts/external_deps/__pycache__/test_zero3_integration.cpython-310.pyc,,
+accelerate/test_utils/scripts/external_deps/test_checkpointing.py,sha256=XHaNRmnrARd1izXFjWGi5UjYGas-4vqayW51jAHBPCA,10699
+accelerate/test_utils/scripts/external_deps/test_ds_multiple_model.py,sha256=hEfb6woEUHBpvMa8cHa-P5MJ2yMi_89ABA45q05P0u4,13765
+accelerate/test_utils/scripts/external_deps/test_metrics.py,sha256=Ev2XKaiwmznoxKujskAAuISGChW646MOiyf0CXEPb9Y,12168
+accelerate/test_utils/scripts/external_deps/test_peak_memory_usage.py,sha256=jlKq-kwd5mQE71k0dGO_Xk61dCen1Qw9Z7deulq8N-4,12495
+accelerate/test_utils/scripts/external_deps/test_performance.py,sha256=y77O2Y9JpcDeP2IBhT0k0FgxrdcYIn68zMPCl7qfo8g,11749
+accelerate/test_utils/scripts/external_deps/test_pippy.py,sha256=rYF5bqSCe_pfylxDv2_2Q7S2mHIb9_6fGns-IcLgE_E,4789
+accelerate/test_utils/scripts/external_deps/test_zero3_integration.py,sha256=-fc3l3PSAplU82FLaeIWCAJdAVBsT72vBQpdJfZ2Hsw,1666
+accelerate/test_utils/scripts/test_cli.py,sha256=-p78NTJgKdeTNDqyrQuGEZZqMpnzdUHPyWZ05FMl5EI,1081
+accelerate/test_utils/scripts/test_ddp_comm_hook.py,sha256=k_-2MBjLKNdMGIcneTbuGd84K05Wp1GEQX6DUVF9UBw,3566
+accelerate/test_utils/scripts/test_distributed_data_loop.py,sha256=RUWTwd7DIpr2fl7JtKOsvTjMiJioTxO8FdSr2Lw_5uI,15137
+accelerate/test_utils/scripts/test_merge_weights.py,sha256=r1oqPOMOqLe5zWenWWTyl9cDbbkFJadpNNiPNBlgR18,5982
+accelerate/test_utils/scripts/test_notebook.py,sha256=qfIy3IvH74-kGn8nadBn_k7qrviqvsxy5ijsnUhuY6o,3894
+accelerate/test_utils/scripts/test_ops.py,sha256=k817w-6h54F1OCIrsarAu8XieWM3H8oZV7bvK-4aJ5k,6268
+accelerate/test_utils/scripts/test_script.py,sha256=dj9zhBhIWHvZ7YSQImzy5B6lWWnV0kAnpENk-k1JkpQ,36259
+accelerate/test_utils/scripts/test_sync.py,sha256=PDe8sYZLCL2LKjj_L9b-Bh2BjAjeii9EZ8sZNfuYx5s,18817
+accelerate/test_utils/testing.py,sha256=0qf5qqTlnzaDqBuJkT4juq5yC1kZgbjRNUUqavbhuPA,30165
+accelerate/test_utils/training.py,sha256=Xn1fIeBUa6QR7SBv_j-JOGL9e3Nl4IZrbtFLL-Zfy6k,5935
+accelerate/tracking.py,sha256=TBIaw_-dyZkA13q-C4GxYF2TFyQHE4ObhQfv6IQQNBE,51520
+accelerate/utils/__init__.py,sha256=Bh39iVjdoVe9OYPslPeMzYyo45edjBZwZc6Mwb8Bqd4,8186
+accelerate/utils/__pycache__/__init__.cpython-310.pyc,,
+accelerate/utils/__pycache__/ao.cpython-310.pyc,,
+accelerate/utils/__pycache__/bnb.cpython-310.pyc,,
+accelerate/utils/__pycache__/constants.cpython-310.pyc,,
+accelerate/utils/__pycache__/dataclasses.cpython-310.pyc,,
+accelerate/utils/__pycache__/deepspeed.cpython-310.pyc,,
+accelerate/utils/__pycache__/environment.cpython-310.pyc,,
+accelerate/utils/__pycache__/fsdp_utils.cpython-310.pyc,,
+accelerate/utils/__pycache__/imports.cpython-310.pyc,,
+accelerate/utils/__pycache__/launch.cpython-310.pyc,,
+accelerate/utils/__pycache__/megatron_lm.cpython-310.pyc,,
+accelerate/utils/__pycache__/memory.cpython-310.pyc,,
+accelerate/utils/__pycache__/modeling.cpython-310.pyc,,
+accelerate/utils/__pycache__/offload.cpython-310.pyc,,
+accelerate/utils/__pycache__/operations.cpython-310.pyc,,
+accelerate/utils/__pycache__/other.cpython-310.pyc,,
+accelerate/utils/__pycache__/random.cpython-310.pyc,,
+accelerate/utils/__pycache__/rich.cpython-310.pyc,,
+accelerate/utils/__pycache__/torch_xla.cpython-310.pyc,,
+accelerate/utils/__pycache__/tqdm.cpython-310.pyc,,
+accelerate/utils/__pycache__/transformer_engine.cpython-310.pyc,,
+accelerate/utils/__pycache__/versions.cpython-310.pyc,,
+accelerate/utils/ao.py,sha256=yfi84ywNpXsKZ6di4jbmtd6bIvRi9bmygr3giN92_Pg,4777
+accelerate/utils/bnb.py,sha256=mlkellhznuPnlJLnv-dLFsWRWaZi_LkY0a609R7SssU,20660
+accelerate/utils/constants.py,sha256=WPw2EojPGov8VFQtF4mplrh_xDSOb4O1riMMWcZSbNI,3561
+accelerate/utils/dataclasses.py,sha256=FrVsDM1N-fGV3R_nFx9_aCgOFcReB7eTPwfoxybIn-A,137691
+accelerate/utils/deepspeed.py,sha256=gt-jwOpOtRs6N4sohuSO2I3x1qLoO6uYYjnPrKp6Zk4,14796
+accelerate/utils/environment.py,sha256=EkjlbfwRXFqEJqA74myv9RIDWKXvgUEBV70-8lezo0w,15256
+accelerate/utils/fsdp_utils.py,sha256=5-hJCZvKrDE7GuZLmwrooBdlJ_-hfj-oV35cUcdrN1k,37843
+accelerate/utils/imports.py,sha256=A7AOE2zlIXrBy3gfq_cL_4I-9B0P4zeuNOwiDtbD1e0,17166
+accelerate/utils/launch.py,sha256=1YNCYE67rFcaeMiMznhAkHSAnrQ33tdRgenzOVw2F4I,34274
+accelerate/utils/megatron_lm.py,sha256=SOze6fUn1T_mPbtwGJSmBDGz87AmgUFRqCZxHuhC5YA,58055
+accelerate/utils/memory.py,sha256=5GtoWvi8H4k8edIHk95Qxy98hFYjbCHh4dN4Iqhc38Q,7270
+accelerate/utils/modeling.py,sha256=QhqI-6JdalB5H7VOpS5-79O-tnjRoTEFUXbAmofoINY,95544
+accelerate/utils/offload.py,sha256=N6ATpOZph7gGrZUjriicD7F6oIhZvmJND66jRyy1nlw,7845
+accelerate/utils/operations.py,sha256=IAdQcwULceQOl1yGpW-FJFEL8FNrpdeiAoa_ONPaTcg,31301
+accelerate/utils/other.py,sha256=ZttbLr-YsGvI2GXLAQH0DhRimh1MQl5dqqhLfi4ShZw,20321
+accelerate/utils/random.py,sha256=Xv_ZJm9eaC2Q7rgZy9OpOunKuTingMiDQCH00qhNVxE,6220
+accelerate/utils/rich.py,sha256=8JZX_uGMQX-BufdXxJpdne7BWd1KyLHSgbiGxrDMYr8,847
+accelerate/utils/torch_xla.py,sha256=Pq1tuqN0X_pWDVza6YgjfO45uoJdoRVRForLeLQzFus,1908
+accelerate/utils/tqdm.py,sha256=k8e9JnieTEQHCCNBaiBys7hPxWlEbyRASdIma-qy_X8,1657
+accelerate/utils/transformer_engine.py,sha256=sVh7oqQL2LepdzAFPHxs2a8VtgukTFbvaNxzLlaq2mo,7521
+accelerate/utils/versions.py,sha256=UgmcbjBm--6CIx1ZamSAMjAK_B_2l48LbeaNygqej8M,2149
diff --git a/venv/lib/python3.10/site-packages/accelerate-1.11.0.dist-info/REQUESTED b/venv/lib/python3.10/site-packages/accelerate-1.11.0.dist-info/REQUESTED
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/venv/lib/python3.10/site-packages/accelerate-1.11.0.dist-info/WHEEL b/venv/lib/python3.10/site-packages/accelerate-1.11.0.dist-info/WHEEL
new file mode 100644
index 0000000000000000000000000000000000000000..370ec5f1aa3524bfd31a19fd21e907335f48a7f6
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/accelerate-1.11.0.dist-info/WHEEL
@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (78.1.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+
diff --git a/venv/lib/python3.10/site-packages/accelerate-1.11.0.dist-info/entry_points.txt b/venv/lib/python3.10/site-packages/accelerate-1.11.0.dist-info/entry_points.txt
new file mode 100644
index 0000000000000000000000000000000000000000..8b9bf6b798b250a47a3febdf0e32c88507fbf86d
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/accelerate-1.11.0.dist-info/entry_points.txt
@@ -0,0 +1,6 @@
+[console_scripts]
+accelerate = accelerate.commands.accelerate_cli:main
+accelerate-config = accelerate.commands.config:main
+accelerate-estimate-memory = accelerate.commands.estimate:main
+accelerate-launch = accelerate.commands.launch:main
+accelerate-merge-weights = accelerate.commands.merge:main
diff --git a/venv/lib/python3.10/site-packages/accelerate-1.11.0.dist-info/licenses/LICENSE b/venv/lib/python3.10/site-packages/accelerate-1.11.0.dist-info/licenses/LICENSE
new file mode 100644
index 0000000000000000000000000000000000000000..261eeb9e9f8b2b4b0d119366dda99c6fd7d35c64
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/accelerate-1.11.0.dist-info/licenses/LICENSE
@@ -0,0 +1,201 @@
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
diff --git a/venv/lib/python3.10/site-packages/accelerate-1.11.0.dist-info/top_level.txt b/venv/lib/python3.10/site-packages/accelerate-1.11.0.dist-info/top_level.txt
new file mode 100644
index 0000000000000000000000000000000000000000..a9368375be0e0e13fdad0eea4b92541bd9e1f594
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/accelerate-1.11.0.dist-info/top_level.txt
@@ -0,0 +1 @@
+accelerate
diff --git a/venv/lib/python3.10/site-packages/charset_normalizer/__init__.py b/venv/lib/python3.10/site-packages/charset_normalizer/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..0d3a37990145e94ad85406166dbaf52f4c311e5e
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/charset_normalizer/__init__.py
@@ -0,0 +1,48 @@
+"""
+Charset-Normalizer
+~~~~~~~~~~~~~~
+The Real First Universal Charset Detector.
+A library that helps you read text from an unknown charset encoding.
+Motivated by chardet, This package is trying to resolve the issue by taking a new approach.
+All IANA character set names for which the Python core library provides codecs are supported.
+
+Basic usage:
+ >>> from charset_normalizer import from_bytes
+ >>> results = from_bytes('Bсеки човек има право на образование. Oбразованието!'.encode('utf_8'))
+ >>> best_guess = results.best()
+ >>> str(best_guess)
+ 'Bсеки човек има право на образование. Oбразованието!'
+
+Others methods and usages are available - see the full documentation
+at .
+:copyright: (c) 2021 by Ahmed TAHRI
+:license: MIT, see LICENSE for more details.
+"""
+
+from __future__ import annotations
+
+import logging
+
+from .api import from_bytes, from_fp, from_path, is_binary
+from .legacy import detect
+from .models import CharsetMatch, CharsetMatches
+from .utils import set_logging_handler
+from .version import VERSION, __version__
+
+__all__ = (
+ "from_fp",
+ "from_path",
+ "from_bytes",
+ "is_binary",
+ "detect",
+ "CharsetMatch",
+ "CharsetMatches",
+ "__version__",
+ "VERSION",
+ "set_logging_handler",
+)
+
+# Attach a NullHandler to the top level logger by default
+# https://docs.python.org/3.3/howto/logging.html#configuring-logging-for-a-library
+
+logging.getLogger("charset_normalizer").addHandler(logging.NullHandler())
diff --git a/venv/lib/python3.10/site-packages/charset_normalizer/__main__.py b/venv/lib/python3.10/site-packages/charset_normalizer/__main__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e0e76f7bfbb411d4424d3a1834b0ea803d80ea7e
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/charset_normalizer/__main__.py
@@ -0,0 +1,6 @@
+from __future__ import annotations
+
+from .cli import cli_detect
+
+if __name__ == "__main__":
+ cli_detect()
diff --git a/venv/lib/python3.10/site-packages/charset_normalizer/__pycache__/__init__.cpython-310.pyc b/venv/lib/python3.10/site-packages/charset_normalizer/__pycache__/__init__.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..8403d0f607e4eff63593ded06b5ec817a260568e
Binary files /dev/null and b/venv/lib/python3.10/site-packages/charset_normalizer/__pycache__/__init__.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/charset_normalizer/__pycache__/__main__.cpython-310.pyc b/venv/lib/python3.10/site-packages/charset_normalizer/__pycache__/__main__.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..22828333881658e4c8c03a0badd258fd0caa1996
Binary files /dev/null and b/venv/lib/python3.10/site-packages/charset_normalizer/__pycache__/__main__.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/charset_normalizer/__pycache__/api.cpython-310.pyc b/venv/lib/python3.10/site-packages/charset_normalizer/__pycache__/api.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..c09a165c7c064b8584c0f09004a81b3964e8ce98
Binary files /dev/null and b/venv/lib/python3.10/site-packages/charset_normalizer/__pycache__/api.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/charset_normalizer/__pycache__/cd.cpython-310.pyc b/venv/lib/python3.10/site-packages/charset_normalizer/__pycache__/cd.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..5d607a84cb760441e3ef03f7bfb576a67c2dd8ad
Binary files /dev/null and b/venv/lib/python3.10/site-packages/charset_normalizer/__pycache__/cd.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/charset_normalizer/__pycache__/constant.cpython-310.pyc b/venv/lib/python3.10/site-packages/charset_normalizer/__pycache__/constant.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..1f35dcacf3445513df89702daec936ec66966fed
Binary files /dev/null and b/venv/lib/python3.10/site-packages/charset_normalizer/__pycache__/constant.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/charset_normalizer/__pycache__/legacy.cpython-310.pyc b/venv/lib/python3.10/site-packages/charset_normalizer/__pycache__/legacy.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..dee0144304ea5be82bb6d28babfd2ee03941b1d0
Binary files /dev/null and b/venv/lib/python3.10/site-packages/charset_normalizer/__pycache__/legacy.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/charset_normalizer/__pycache__/md.cpython-310.pyc b/venv/lib/python3.10/site-packages/charset_normalizer/__pycache__/md.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..31670357708823cc6a57505c177abbd62c67edd7
Binary files /dev/null and b/venv/lib/python3.10/site-packages/charset_normalizer/__pycache__/md.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/charset_normalizer/__pycache__/models.cpython-310.pyc b/venv/lib/python3.10/site-packages/charset_normalizer/__pycache__/models.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..41256cc5973894840a69b4b1ea8deff68b6e9030
Binary files /dev/null and b/venv/lib/python3.10/site-packages/charset_normalizer/__pycache__/models.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/charset_normalizer/__pycache__/utils.cpython-310.pyc b/venv/lib/python3.10/site-packages/charset_normalizer/__pycache__/utils.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..dfddbcc98f011f08d8c788f526e78781b9714e91
Binary files /dev/null and b/venv/lib/python3.10/site-packages/charset_normalizer/__pycache__/utils.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/charset_normalizer/__pycache__/version.cpython-310.pyc b/venv/lib/python3.10/site-packages/charset_normalizer/__pycache__/version.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..80b5a77a5a742697a81eecf6658d9d24f43c2428
Binary files /dev/null and b/venv/lib/python3.10/site-packages/charset_normalizer/__pycache__/version.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/charset_normalizer/api.py b/venv/lib/python3.10/site-packages/charset_normalizer/api.py
new file mode 100644
index 0000000000000000000000000000000000000000..ebd96390507bac07ae21d513b45777011a539892
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/charset_normalizer/api.py
@@ -0,0 +1,669 @@
+from __future__ import annotations
+
+import logging
+from os import PathLike
+from typing import BinaryIO
+
+from .cd import (
+ coherence_ratio,
+ encoding_languages,
+ mb_encoding_languages,
+ merge_coherence_ratios,
+)
+from .constant import IANA_SUPPORTED, TOO_BIG_SEQUENCE, TOO_SMALL_SEQUENCE, TRACE
+from .md import mess_ratio
+from .models import CharsetMatch, CharsetMatches
+from .utils import (
+ any_specified_encoding,
+ cut_sequence_chunks,
+ iana_name,
+ identify_sig_or_bom,
+ is_cp_similar,
+ is_multi_byte_encoding,
+ should_strip_sig_or_bom,
+)
+
+logger = logging.getLogger("charset_normalizer")
+explain_handler = logging.StreamHandler()
+explain_handler.setFormatter(
+ logging.Formatter("%(asctime)s | %(levelname)s | %(message)s")
+)
+
+
+def from_bytes(
+ sequences: bytes | bytearray,
+ steps: int = 5,
+ chunk_size: int = 512,
+ threshold: float = 0.2,
+ cp_isolation: list[str] | None = None,
+ cp_exclusion: list[str] | None = None,
+ preemptive_behaviour: bool = True,
+ explain: bool = False,
+ language_threshold: float = 0.1,
+ enable_fallback: bool = True,
+) -> CharsetMatches:
+ """
+ Given a raw bytes sequence, return the best possibles charset usable to render str objects.
+ If there is no results, it is a strong indicator that the source is binary/not text.
+ By default, the process will extract 5 blocks of 512o each to assess the mess and coherence of a given sequence.
+ And will give up a particular code page after 20% of measured mess. Those criteria are customizable at will.
+
+ The preemptive behavior DOES NOT replace the traditional detection workflow, it prioritize a particular code page
+ but never take it for granted. Can improve the performance.
+
+ You may want to focus your attention to some code page or/and not others, use cp_isolation and cp_exclusion for that
+ purpose.
+
+ This function will strip the SIG in the payload/sequence every time except on UTF-16, UTF-32.
+ By default the library does not setup any handler other than the NullHandler, if you choose to set the 'explain'
+ toggle to True it will alter the logger configuration to add a StreamHandler that is suitable for debugging.
+ Custom logging format and handler can be set manually.
+ """
+
+ if not isinstance(sequences, (bytearray, bytes)):
+ raise TypeError(
+ "Expected object of type bytes or bytearray, got: {}".format(
+ type(sequences)
+ )
+ )
+
+ if explain:
+ previous_logger_level: int = logger.level
+ logger.addHandler(explain_handler)
+ logger.setLevel(TRACE)
+
+ length: int = len(sequences)
+
+ if length == 0:
+ logger.debug("Encoding detection on empty bytes, assuming utf_8 intention.")
+ if explain: # Defensive: ensure exit path clean handler
+ logger.removeHandler(explain_handler)
+ logger.setLevel(previous_logger_level or logging.WARNING)
+ return CharsetMatches([CharsetMatch(sequences, "utf_8", 0.0, False, [], "")])
+
+ if cp_isolation is not None:
+ logger.log(
+ TRACE,
+ "cp_isolation is set. use this flag for debugging purpose. "
+ "limited list of encoding allowed : %s.",
+ ", ".join(cp_isolation),
+ )
+ cp_isolation = [iana_name(cp, False) for cp in cp_isolation]
+ else:
+ cp_isolation = []
+
+ if cp_exclusion is not None:
+ logger.log(
+ TRACE,
+ "cp_exclusion is set. use this flag for debugging purpose. "
+ "limited list of encoding excluded : %s.",
+ ", ".join(cp_exclusion),
+ )
+ cp_exclusion = [iana_name(cp, False) for cp in cp_exclusion]
+ else:
+ cp_exclusion = []
+
+ if length <= (chunk_size * steps):
+ logger.log(
+ TRACE,
+ "override steps (%i) and chunk_size (%i) as content does not fit (%i byte(s) given) parameters.",
+ steps,
+ chunk_size,
+ length,
+ )
+ steps = 1
+ chunk_size = length
+
+ if steps > 1 and length / steps < chunk_size:
+ chunk_size = int(length / steps)
+
+ is_too_small_sequence: bool = len(sequences) < TOO_SMALL_SEQUENCE
+ is_too_large_sequence: bool = len(sequences) >= TOO_BIG_SEQUENCE
+
+ if is_too_small_sequence:
+ logger.log(
+ TRACE,
+ "Trying to detect encoding from a tiny portion of ({}) byte(s).".format(
+ length
+ ),
+ )
+ elif is_too_large_sequence:
+ logger.log(
+ TRACE,
+ "Using lazy str decoding because the payload is quite large, ({}) byte(s).".format(
+ length
+ ),
+ )
+
+ prioritized_encodings: list[str] = []
+
+ specified_encoding: str | None = (
+ any_specified_encoding(sequences) if preemptive_behaviour else None
+ )
+
+ if specified_encoding is not None:
+ prioritized_encodings.append(specified_encoding)
+ logger.log(
+ TRACE,
+ "Detected declarative mark in sequence. Priority +1 given for %s.",
+ specified_encoding,
+ )
+
+ tested: set[str] = set()
+ tested_but_hard_failure: list[str] = []
+ tested_but_soft_failure: list[str] = []
+
+ fallback_ascii: CharsetMatch | None = None
+ fallback_u8: CharsetMatch | None = None
+ fallback_specified: CharsetMatch | None = None
+
+ results: CharsetMatches = CharsetMatches()
+
+ early_stop_results: CharsetMatches = CharsetMatches()
+
+ sig_encoding, sig_payload = identify_sig_or_bom(sequences)
+
+ if sig_encoding is not None:
+ prioritized_encodings.append(sig_encoding)
+ logger.log(
+ TRACE,
+ "Detected a SIG or BOM mark on first %i byte(s). Priority +1 given for %s.",
+ len(sig_payload),
+ sig_encoding,
+ )
+
+ prioritized_encodings.append("ascii")
+
+ if "utf_8" not in prioritized_encodings:
+ prioritized_encodings.append("utf_8")
+
+ for encoding_iana in prioritized_encodings + IANA_SUPPORTED:
+ if cp_isolation and encoding_iana not in cp_isolation:
+ continue
+
+ if cp_exclusion and encoding_iana in cp_exclusion:
+ continue
+
+ if encoding_iana in tested:
+ continue
+
+ tested.add(encoding_iana)
+
+ decoded_payload: str | None = None
+ bom_or_sig_available: bool = sig_encoding == encoding_iana
+ strip_sig_or_bom: bool = bom_or_sig_available and should_strip_sig_or_bom(
+ encoding_iana
+ )
+
+ if encoding_iana in {"utf_16", "utf_32"} and not bom_or_sig_available:
+ logger.log(
+ TRACE,
+ "Encoding %s won't be tested as-is because it require a BOM. Will try some sub-encoder LE/BE.",
+ encoding_iana,
+ )
+ continue
+ if encoding_iana in {"utf_7"} and not bom_or_sig_available:
+ logger.log(
+ TRACE,
+ "Encoding %s won't be tested as-is because detection is unreliable without BOM/SIG.",
+ encoding_iana,
+ )
+ continue
+
+ try:
+ is_multi_byte_decoder: bool = is_multi_byte_encoding(encoding_iana)
+ except (ModuleNotFoundError, ImportError):
+ logger.log(
+ TRACE,
+ "Encoding %s does not provide an IncrementalDecoder",
+ encoding_iana,
+ )
+ continue
+
+ try:
+ if is_too_large_sequence and is_multi_byte_decoder is False:
+ str(
+ (
+ sequences[: int(50e4)]
+ if strip_sig_or_bom is False
+ else sequences[len(sig_payload) : int(50e4)]
+ ),
+ encoding=encoding_iana,
+ )
+ else:
+ decoded_payload = str(
+ (
+ sequences
+ if strip_sig_or_bom is False
+ else sequences[len(sig_payload) :]
+ ),
+ encoding=encoding_iana,
+ )
+ except (UnicodeDecodeError, LookupError) as e:
+ if not isinstance(e, LookupError):
+ logger.log(
+ TRACE,
+ "Code page %s does not fit given bytes sequence at ALL. %s",
+ encoding_iana,
+ str(e),
+ )
+ tested_but_hard_failure.append(encoding_iana)
+ continue
+
+ similar_soft_failure_test: bool = False
+
+ for encoding_soft_failed in tested_but_soft_failure:
+ if is_cp_similar(encoding_iana, encoding_soft_failed):
+ similar_soft_failure_test = True
+ break
+
+ if similar_soft_failure_test:
+ logger.log(
+ TRACE,
+ "%s is deemed too similar to code page %s and was consider unsuited already. Continuing!",
+ encoding_iana,
+ encoding_soft_failed,
+ )
+ continue
+
+ r_ = range(
+ 0 if not bom_or_sig_available else len(sig_payload),
+ length,
+ int(length / steps),
+ )
+
+ multi_byte_bonus: bool = (
+ is_multi_byte_decoder
+ and decoded_payload is not None
+ and len(decoded_payload) < length
+ )
+
+ if multi_byte_bonus:
+ logger.log(
+ TRACE,
+ "Code page %s is a multi byte encoding table and it appear that at least one character "
+ "was encoded using n-bytes.",
+ encoding_iana,
+ )
+
+ max_chunk_gave_up: int = int(len(r_) / 4)
+
+ max_chunk_gave_up = max(max_chunk_gave_up, 2)
+ early_stop_count: int = 0
+ lazy_str_hard_failure = False
+
+ md_chunks: list[str] = []
+ md_ratios = []
+
+ try:
+ for chunk in cut_sequence_chunks(
+ sequences,
+ encoding_iana,
+ r_,
+ chunk_size,
+ bom_or_sig_available,
+ strip_sig_or_bom,
+ sig_payload,
+ is_multi_byte_decoder,
+ decoded_payload,
+ ):
+ md_chunks.append(chunk)
+
+ md_ratios.append(
+ mess_ratio(
+ chunk,
+ threshold,
+ explain is True and 1 <= len(cp_isolation) <= 2,
+ )
+ )
+
+ if md_ratios[-1] >= threshold:
+ early_stop_count += 1
+
+ if (early_stop_count >= max_chunk_gave_up) or (
+ bom_or_sig_available and strip_sig_or_bom is False
+ ):
+ break
+ except (
+ UnicodeDecodeError
+ ) as e: # Lazy str loading may have missed something there
+ logger.log(
+ TRACE,
+ "LazyStr Loading: After MD chunk decode, code page %s does not fit given bytes sequence at ALL. %s",
+ encoding_iana,
+ str(e),
+ )
+ early_stop_count = max_chunk_gave_up
+ lazy_str_hard_failure = True
+
+ # We might want to check the sequence again with the whole content
+ # Only if initial MD tests passes
+ if (
+ not lazy_str_hard_failure
+ and is_too_large_sequence
+ and not is_multi_byte_decoder
+ ):
+ try:
+ sequences[int(50e3) :].decode(encoding_iana, errors="strict")
+ except UnicodeDecodeError as e:
+ logger.log(
+ TRACE,
+ "LazyStr Loading: After final lookup, code page %s does not fit given bytes sequence at ALL. %s",
+ encoding_iana,
+ str(e),
+ )
+ tested_but_hard_failure.append(encoding_iana)
+ continue
+
+ mean_mess_ratio: float = sum(md_ratios) / len(md_ratios) if md_ratios else 0.0
+ if mean_mess_ratio >= threshold or early_stop_count >= max_chunk_gave_up:
+ tested_but_soft_failure.append(encoding_iana)
+ logger.log(
+ TRACE,
+ "%s was excluded because of initial chaos probing. Gave up %i time(s). "
+ "Computed mean chaos is %f %%.",
+ encoding_iana,
+ early_stop_count,
+ round(mean_mess_ratio * 100, ndigits=3),
+ )
+ # Preparing those fallbacks in case we got nothing.
+ if (
+ enable_fallback
+ and encoding_iana
+ in ["ascii", "utf_8", specified_encoding, "utf_16", "utf_32"]
+ and not lazy_str_hard_failure
+ ):
+ fallback_entry = CharsetMatch(
+ sequences,
+ encoding_iana,
+ threshold,
+ bom_or_sig_available,
+ [],
+ decoded_payload,
+ preemptive_declaration=specified_encoding,
+ )
+ if encoding_iana == specified_encoding:
+ fallback_specified = fallback_entry
+ elif encoding_iana == "ascii":
+ fallback_ascii = fallback_entry
+ else:
+ fallback_u8 = fallback_entry
+ continue
+
+ logger.log(
+ TRACE,
+ "%s passed initial chaos probing. Mean measured chaos is %f %%",
+ encoding_iana,
+ round(mean_mess_ratio * 100, ndigits=3),
+ )
+
+ if not is_multi_byte_decoder:
+ target_languages: list[str] = encoding_languages(encoding_iana)
+ else:
+ target_languages = mb_encoding_languages(encoding_iana)
+
+ if target_languages:
+ logger.log(
+ TRACE,
+ "{} should target any language(s) of {}".format(
+ encoding_iana, str(target_languages)
+ ),
+ )
+
+ cd_ratios = []
+
+ # We shall skip the CD when its about ASCII
+ # Most of the time its not relevant to run "language-detection" on it.
+ if encoding_iana != "ascii":
+ for chunk in md_chunks:
+ chunk_languages = coherence_ratio(
+ chunk,
+ language_threshold,
+ ",".join(target_languages) if target_languages else None,
+ )
+
+ cd_ratios.append(chunk_languages)
+
+ cd_ratios_merged = merge_coherence_ratios(cd_ratios)
+
+ if cd_ratios_merged:
+ logger.log(
+ TRACE,
+ "We detected language {} using {}".format(
+ cd_ratios_merged, encoding_iana
+ ),
+ )
+
+ current_match = CharsetMatch(
+ sequences,
+ encoding_iana,
+ mean_mess_ratio,
+ bom_or_sig_available,
+ cd_ratios_merged,
+ (
+ decoded_payload
+ if (
+ is_too_large_sequence is False
+ or encoding_iana in [specified_encoding, "ascii", "utf_8"]
+ )
+ else None
+ ),
+ preemptive_declaration=specified_encoding,
+ )
+
+ results.append(current_match)
+
+ if (
+ encoding_iana in [specified_encoding, "ascii", "utf_8"]
+ and mean_mess_ratio < 0.1
+ ):
+ # If md says nothing to worry about, then... stop immediately!
+ if mean_mess_ratio == 0.0:
+ logger.debug(
+ "Encoding detection: %s is most likely the one.",
+ current_match.encoding,
+ )
+ if explain: # Defensive: ensure exit path clean handler
+ logger.removeHandler(explain_handler)
+ logger.setLevel(previous_logger_level)
+ return CharsetMatches([current_match])
+
+ early_stop_results.append(current_match)
+
+ if (
+ len(early_stop_results)
+ and (specified_encoding is None or specified_encoding in tested)
+ and "ascii" in tested
+ and "utf_8" in tested
+ ):
+ probable_result: CharsetMatch = early_stop_results.best() # type: ignore[assignment]
+ logger.debug(
+ "Encoding detection: %s is most likely the one.",
+ probable_result.encoding,
+ )
+ if explain: # Defensive: ensure exit path clean handler
+ logger.removeHandler(explain_handler)
+ logger.setLevel(previous_logger_level)
+
+ return CharsetMatches([probable_result])
+
+ if encoding_iana == sig_encoding:
+ logger.debug(
+ "Encoding detection: %s is most likely the one as we detected a BOM or SIG within "
+ "the beginning of the sequence.",
+ encoding_iana,
+ )
+ if explain: # Defensive: ensure exit path clean handler
+ logger.removeHandler(explain_handler)
+ logger.setLevel(previous_logger_level)
+ return CharsetMatches([results[encoding_iana]])
+
+ if len(results) == 0:
+ if fallback_u8 or fallback_ascii or fallback_specified:
+ logger.log(
+ TRACE,
+ "Nothing got out of the detection process. Using ASCII/UTF-8/Specified fallback.",
+ )
+
+ if fallback_specified:
+ logger.debug(
+ "Encoding detection: %s will be used as a fallback match",
+ fallback_specified.encoding,
+ )
+ results.append(fallback_specified)
+ elif (
+ (fallback_u8 and fallback_ascii is None)
+ or (
+ fallback_u8
+ and fallback_ascii
+ and fallback_u8.fingerprint != fallback_ascii.fingerprint
+ )
+ or (fallback_u8 is not None)
+ ):
+ logger.debug("Encoding detection: utf_8 will be used as a fallback match")
+ results.append(fallback_u8)
+ elif fallback_ascii:
+ logger.debug("Encoding detection: ascii will be used as a fallback match")
+ results.append(fallback_ascii)
+
+ if results:
+ logger.debug(
+ "Encoding detection: Found %s as plausible (best-candidate) for content. With %i alternatives.",
+ results.best().encoding, # type: ignore
+ len(results) - 1,
+ )
+ else:
+ logger.debug("Encoding detection: Unable to determine any suitable charset.")
+
+ if explain:
+ logger.removeHandler(explain_handler)
+ logger.setLevel(previous_logger_level)
+
+ return results
+
+
+def from_fp(
+ fp: BinaryIO,
+ steps: int = 5,
+ chunk_size: int = 512,
+ threshold: float = 0.20,
+ cp_isolation: list[str] | None = None,
+ cp_exclusion: list[str] | None = None,
+ preemptive_behaviour: bool = True,
+ explain: bool = False,
+ language_threshold: float = 0.1,
+ enable_fallback: bool = True,
+) -> CharsetMatches:
+ """
+ Same thing than the function from_bytes but using a file pointer that is already ready.
+ Will not close the file pointer.
+ """
+ return from_bytes(
+ fp.read(),
+ steps,
+ chunk_size,
+ threshold,
+ cp_isolation,
+ cp_exclusion,
+ preemptive_behaviour,
+ explain,
+ language_threshold,
+ enable_fallback,
+ )
+
+
+def from_path(
+ path: str | bytes | PathLike, # type: ignore[type-arg]
+ steps: int = 5,
+ chunk_size: int = 512,
+ threshold: float = 0.20,
+ cp_isolation: list[str] | None = None,
+ cp_exclusion: list[str] | None = None,
+ preemptive_behaviour: bool = True,
+ explain: bool = False,
+ language_threshold: float = 0.1,
+ enable_fallback: bool = True,
+) -> CharsetMatches:
+ """
+ Same thing than the function from_bytes but with one extra step. Opening and reading given file path in binary mode.
+ Can raise IOError.
+ """
+ with open(path, "rb") as fp:
+ return from_fp(
+ fp,
+ steps,
+ chunk_size,
+ threshold,
+ cp_isolation,
+ cp_exclusion,
+ preemptive_behaviour,
+ explain,
+ language_threshold,
+ enable_fallback,
+ )
+
+
+def is_binary(
+ fp_or_path_or_payload: PathLike | str | BinaryIO | bytes, # type: ignore[type-arg]
+ steps: int = 5,
+ chunk_size: int = 512,
+ threshold: float = 0.20,
+ cp_isolation: list[str] | None = None,
+ cp_exclusion: list[str] | None = None,
+ preemptive_behaviour: bool = True,
+ explain: bool = False,
+ language_threshold: float = 0.1,
+ enable_fallback: bool = False,
+) -> bool:
+ """
+ Detect if the given input (file, bytes, or path) points to a binary file. aka. not a string.
+ Based on the same main heuristic algorithms and default kwargs at the sole exception that fallbacks match
+ are disabled to be stricter around ASCII-compatible but unlikely to be a string.
+ """
+ if isinstance(fp_or_path_or_payload, (str, PathLike)):
+ guesses = from_path(
+ fp_or_path_or_payload,
+ steps=steps,
+ chunk_size=chunk_size,
+ threshold=threshold,
+ cp_isolation=cp_isolation,
+ cp_exclusion=cp_exclusion,
+ preemptive_behaviour=preemptive_behaviour,
+ explain=explain,
+ language_threshold=language_threshold,
+ enable_fallback=enable_fallback,
+ )
+ elif isinstance(
+ fp_or_path_or_payload,
+ (
+ bytes,
+ bytearray,
+ ),
+ ):
+ guesses = from_bytes(
+ fp_or_path_or_payload,
+ steps=steps,
+ chunk_size=chunk_size,
+ threshold=threshold,
+ cp_isolation=cp_isolation,
+ cp_exclusion=cp_exclusion,
+ preemptive_behaviour=preemptive_behaviour,
+ explain=explain,
+ language_threshold=language_threshold,
+ enable_fallback=enable_fallback,
+ )
+ else:
+ guesses = from_fp(
+ fp_or_path_or_payload,
+ steps=steps,
+ chunk_size=chunk_size,
+ threshold=threshold,
+ cp_isolation=cp_isolation,
+ cp_exclusion=cp_exclusion,
+ preemptive_behaviour=preemptive_behaviour,
+ explain=explain,
+ language_threshold=language_threshold,
+ enable_fallback=enable_fallback,
+ )
+
+ return not guesses
diff --git a/venv/lib/python3.10/site-packages/charset_normalizer/cd.py b/venv/lib/python3.10/site-packages/charset_normalizer/cd.py
new file mode 100644
index 0000000000000000000000000000000000000000..71a3ed5197f788135bb700a15a64594780aa6337
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/charset_normalizer/cd.py
@@ -0,0 +1,395 @@
+from __future__ import annotations
+
+import importlib
+from codecs import IncrementalDecoder
+from collections import Counter
+from functools import lru_cache
+from typing import Counter as TypeCounter
+
+from .constant import (
+ FREQUENCIES,
+ KO_NAMES,
+ LANGUAGE_SUPPORTED_COUNT,
+ TOO_SMALL_SEQUENCE,
+ ZH_NAMES,
+)
+from .md import is_suspiciously_successive_range
+from .models import CoherenceMatches
+from .utils import (
+ is_accentuated,
+ is_latin,
+ is_multi_byte_encoding,
+ is_unicode_range_secondary,
+ unicode_range,
+)
+
+
+def encoding_unicode_range(iana_name: str) -> list[str]:
+ """
+ Return associated unicode ranges in a single byte code page.
+ """
+ if is_multi_byte_encoding(iana_name):
+ raise OSError("Function not supported on multi-byte code page")
+
+ decoder = importlib.import_module(f"encodings.{iana_name}").IncrementalDecoder
+
+ p: IncrementalDecoder = decoder(errors="ignore")
+ seen_ranges: dict[str, int] = {}
+ character_count: int = 0
+
+ for i in range(0x40, 0xFF):
+ chunk: str = p.decode(bytes([i]))
+
+ if chunk:
+ character_range: str | None = unicode_range(chunk)
+
+ if character_range is None:
+ continue
+
+ if is_unicode_range_secondary(character_range) is False:
+ if character_range not in seen_ranges:
+ seen_ranges[character_range] = 0
+ seen_ranges[character_range] += 1
+ character_count += 1
+
+ return sorted(
+ [
+ character_range
+ for character_range in seen_ranges
+ if seen_ranges[character_range] / character_count >= 0.15
+ ]
+ )
+
+
+def unicode_range_languages(primary_range: str) -> list[str]:
+ """
+ Return inferred languages used with a unicode range.
+ """
+ languages: list[str] = []
+
+ for language, characters in FREQUENCIES.items():
+ for character in characters:
+ if unicode_range(character) == primary_range:
+ languages.append(language)
+ break
+
+ return languages
+
+
+@lru_cache()
+def encoding_languages(iana_name: str) -> list[str]:
+ """
+ Single-byte encoding language association. Some code page are heavily linked to particular language(s).
+ This function does the correspondence.
+ """
+ unicode_ranges: list[str] = encoding_unicode_range(iana_name)
+ primary_range: str | None = None
+
+ for specified_range in unicode_ranges:
+ if "Latin" not in specified_range:
+ primary_range = specified_range
+ break
+
+ if primary_range is None:
+ return ["Latin Based"]
+
+ return unicode_range_languages(primary_range)
+
+
+@lru_cache()
+def mb_encoding_languages(iana_name: str) -> list[str]:
+ """
+ Multi-byte encoding language association. Some code page are heavily linked to particular language(s).
+ This function does the correspondence.
+ """
+ if (
+ iana_name.startswith("shift_")
+ or iana_name.startswith("iso2022_jp")
+ or iana_name.startswith("euc_j")
+ or iana_name == "cp932"
+ ):
+ return ["Japanese"]
+ if iana_name.startswith("gb") or iana_name in ZH_NAMES:
+ return ["Chinese"]
+ if iana_name.startswith("iso2022_kr") or iana_name in KO_NAMES:
+ return ["Korean"]
+
+ return []
+
+
+@lru_cache(maxsize=LANGUAGE_SUPPORTED_COUNT)
+def get_target_features(language: str) -> tuple[bool, bool]:
+ """
+ Determine main aspects from a supported language if it contains accents and if is pure Latin.
+ """
+ target_have_accents: bool = False
+ target_pure_latin: bool = True
+
+ for character in FREQUENCIES[language]:
+ if not target_have_accents and is_accentuated(character):
+ target_have_accents = True
+ if target_pure_latin and is_latin(character) is False:
+ target_pure_latin = False
+
+ return target_have_accents, target_pure_latin
+
+
+def alphabet_languages(
+ characters: list[str], ignore_non_latin: bool = False
+) -> list[str]:
+ """
+ Return associated languages associated to given characters.
+ """
+ languages: list[tuple[str, float]] = []
+
+ source_have_accents = any(is_accentuated(character) for character in characters)
+
+ for language, language_characters in FREQUENCIES.items():
+ target_have_accents, target_pure_latin = get_target_features(language)
+
+ if ignore_non_latin and target_pure_latin is False:
+ continue
+
+ if target_have_accents is False and source_have_accents:
+ continue
+
+ character_count: int = len(language_characters)
+
+ character_match_count: int = len(
+ [c for c in language_characters if c in characters]
+ )
+
+ ratio: float = character_match_count / character_count
+
+ if ratio >= 0.2:
+ languages.append((language, ratio))
+
+ languages = sorted(languages, key=lambda x: x[1], reverse=True)
+
+ return [compatible_language[0] for compatible_language in languages]
+
+
+def characters_popularity_compare(
+ language: str, ordered_characters: list[str]
+) -> float:
+ """
+ Determine if a ordered characters list (by occurrence from most appearance to rarest) match a particular language.
+ The result is a ratio between 0. (absolutely no correspondence) and 1. (near perfect fit).
+ Beware that is function is not strict on the match in order to ease the detection. (Meaning close match is 1.)
+ """
+ if language not in FREQUENCIES:
+ raise ValueError(f"{language} not available")
+
+ character_approved_count: int = 0
+ FREQUENCIES_language_set = set(FREQUENCIES[language])
+
+ ordered_characters_count: int = len(ordered_characters)
+ target_language_characters_count: int = len(FREQUENCIES[language])
+
+ large_alphabet: bool = target_language_characters_count > 26
+
+ for character, character_rank in zip(
+ ordered_characters, range(0, ordered_characters_count)
+ ):
+ if character not in FREQUENCIES_language_set:
+ continue
+
+ character_rank_in_language: int = FREQUENCIES[language].index(character)
+ expected_projection_ratio: float = (
+ target_language_characters_count / ordered_characters_count
+ )
+ character_rank_projection: int = int(character_rank * expected_projection_ratio)
+
+ if (
+ large_alphabet is False
+ and abs(character_rank_projection - character_rank_in_language) > 4
+ ):
+ continue
+
+ if (
+ large_alphabet is True
+ and abs(character_rank_projection - character_rank_in_language)
+ < target_language_characters_count / 3
+ ):
+ character_approved_count += 1
+ continue
+
+ characters_before_source: list[str] = FREQUENCIES[language][
+ 0:character_rank_in_language
+ ]
+ characters_after_source: list[str] = FREQUENCIES[language][
+ character_rank_in_language:
+ ]
+ characters_before: list[str] = ordered_characters[0:character_rank]
+ characters_after: list[str] = ordered_characters[character_rank:]
+
+ before_match_count: int = len(
+ set(characters_before) & set(characters_before_source)
+ )
+
+ after_match_count: int = len(
+ set(characters_after) & set(characters_after_source)
+ )
+
+ if len(characters_before_source) == 0 and before_match_count <= 4:
+ character_approved_count += 1
+ continue
+
+ if len(characters_after_source) == 0 and after_match_count <= 4:
+ character_approved_count += 1
+ continue
+
+ if (
+ before_match_count / len(characters_before_source) >= 0.4
+ or after_match_count / len(characters_after_source) >= 0.4
+ ):
+ character_approved_count += 1
+ continue
+
+ return character_approved_count / len(ordered_characters)
+
+
+def alpha_unicode_split(decoded_sequence: str) -> list[str]:
+ """
+ Given a decoded text sequence, return a list of str. Unicode range / alphabet separation.
+ Ex. a text containing English/Latin with a bit a Hebrew will return two items in the resulting list;
+ One containing the latin letters and the other hebrew.
+ """
+ layers: dict[str, str] = {}
+
+ for character in decoded_sequence:
+ if character.isalpha() is False:
+ continue
+
+ character_range: str | None = unicode_range(character)
+
+ if character_range is None:
+ continue
+
+ layer_target_range: str | None = None
+
+ for discovered_range in layers:
+ if (
+ is_suspiciously_successive_range(discovered_range, character_range)
+ is False
+ ):
+ layer_target_range = discovered_range
+ break
+
+ if layer_target_range is None:
+ layer_target_range = character_range
+
+ if layer_target_range not in layers:
+ layers[layer_target_range] = character.lower()
+ continue
+
+ layers[layer_target_range] += character.lower()
+
+ return list(layers.values())
+
+
+def merge_coherence_ratios(results: list[CoherenceMatches]) -> CoherenceMatches:
+ """
+ This function merge results previously given by the function coherence_ratio.
+ The return type is the same as coherence_ratio.
+ """
+ per_language_ratios: dict[str, list[float]] = {}
+ for result in results:
+ for sub_result in result:
+ language, ratio = sub_result
+ if language not in per_language_ratios:
+ per_language_ratios[language] = [ratio]
+ continue
+ per_language_ratios[language].append(ratio)
+
+ merge = [
+ (
+ language,
+ round(
+ sum(per_language_ratios[language]) / len(per_language_ratios[language]),
+ 4,
+ ),
+ )
+ for language in per_language_ratios
+ ]
+
+ return sorted(merge, key=lambda x: x[1], reverse=True)
+
+
+def filter_alt_coherence_matches(results: CoherenceMatches) -> CoherenceMatches:
+ """
+ We shall NOT return "English—" in CoherenceMatches because it is an alternative
+ of "English". This function only keeps the best match and remove the em-dash in it.
+ """
+ index_results: dict[str, list[float]] = dict()
+
+ for result in results:
+ language, ratio = result
+ no_em_name: str = language.replace("—", "")
+
+ if no_em_name not in index_results:
+ index_results[no_em_name] = []
+
+ index_results[no_em_name].append(ratio)
+
+ if any(len(index_results[e]) > 1 for e in index_results):
+ filtered_results: CoherenceMatches = []
+
+ for language in index_results:
+ filtered_results.append((language, max(index_results[language])))
+
+ return filtered_results
+
+ return results
+
+
+@lru_cache(maxsize=2048)
+def coherence_ratio(
+ decoded_sequence: str, threshold: float = 0.1, lg_inclusion: str | None = None
+) -> CoherenceMatches:
+ """
+ Detect ANY language that can be identified in given sequence. The sequence will be analysed by layers.
+ A layer = Character extraction by alphabets/ranges.
+ """
+
+ results: list[tuple[str, float]] = []
+ ignore_non_latin: bool = False
+
+ sufficient_match_count: int = 0
+
+ lg_inclusion_list = lg_inclusion.split(",") if lg_inclusion is not None else []
+ if "Latin Based" in lg_inclusion_list:
+ ignore_non_latin = True
+ lg_inclusion_list.remove("Latin Based")
+
+ for layer in alpha_unicode_split(decoded_sequence):
+ sequence_frequencies: TypeCounter[str] = Counter(layer)
+ most_common = sequence_frequencies.most_common()
+
+ character_count: int = sum(o for c, o in most_common)
+
+ if character_count <= TOO_SMALL_SEQUENCE:
+ continue
+
+ popular_character_ordered: list[str] = [c for c, o in most_common]
+
+ for language in lg_inclusion_list or alphabet_languages(
+ popular_character_ordered, ignore_non_latin
+ ):
+ ratio: float = characters_popularity_compare(
+ language, popular_character_ordered
+ )
+
+ if ratio < threshold:
+ continue
+ elif ratio >= 0.8:
+ sufficient_match_count += 1
+
+ results.append((language, round(ratio, 4)))
+
+ if sufficient_match_count >= 3:
+ break
+
+ return sorted(
+ filter_alt_coherence_matches(results), key=lambda x: x[1], reverse=True
+ )
diff --git a/venv/lib/python3.10/site-packages/charset_normalizer/cli/__init__.py b/venv/lib/python3.10/site-packages/charset_normalizer/cli/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..543a5a4de49d07690e73df778aa580589d0789c6
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/charset_normalizer/cli/__init__.py
@@ -0,0 +1,8 @@
+from __future__ import annotations
+
+from .__main__ import cli_detect, query_yes_no
+
+__all__ = (
+ "cli_detect",
+ "query_yes_no",
+)
diff --git a/venv/lib/python3.10/site-packages/charset_normalizer/cli/__main__.py b/venv/lib/python3.10/site-packages/charset_normalizer/cli/__main__.py
new file mode 100644
index 0000000000000000000000000000000000000000..cb64156a0fc164442acc4f4517975a5699d26354
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/charset_normalizer/cli/__main__.py
@@ -0,0 +1,381 @@
+from __future__ import annotations
+
+import argparse
+import sys
+import typing
+from json import dumps
+from os.path import abspath, basename, dirname, join, realpath
+from platform import python_version
+from unicodedata import unidata_version
+
+import charset_normalizer.md as md_module
+from charset_normalizer import from_fp
+from charset_normalizer.models import CliDetectionResult
+from charset_normalizer.version import __version__
+
+
+def query_yes_no(question: str, default: str = "yes") -> bool:
+ """Ask a yes/no question via input() and return their answer.
+
+ "question" is a string that is presented to the user.
+ "default" is the presumed answer if the user just hits .
+ It must be "yes" (the default), "no" or None (meaning
+ an answer is required of the user).
+
+ The "answer" return value is True for "yes" or False for "no".
+
+ Credit goes to (c) https://stackoverflow.com/questions/3041986/apt-command-line-interface-like-yes-no-input
+ """
+ valid = {"yes": True, "y": True, "ye": True, "no": False, "n": False}
+ if default is None:
+ prompt = " [y/n] "
+ elif default == "yes":
+ prompt = " [Y/n] "
+ elif default == "no":
+ prompt = " [y/N] "
+ else:
+ raise ValueError("invalid default answer: '%s'" % default)
+
+ while True:
+ sys.stdout.write(question + prompt)
+ choice = input().lower()
+ if default is not None and choice == "":
+ return valid[default]
+ elif choice in valid:
+ return valid[choice]
+ else:
+ sys.stdout.write("Please respond with 'yes' or 'no' (or 'y' or 'n').\n")
+
+
+class FileType:
+ """Factory for creating file object types
+
+ Instances of FileType are typically passed as type= arguments to the
+ ArgumentParser add_argument() method.
+
+ Keyword Arguments:
+ - mode -- A string indicating how the file is to be opened. Accepts the
+ same values as the builtin open() function.
+ - bufsize -- The file's desired buffer size. Accepts the same values as
+ the builtin open() function.
+ - encoding -- The file's encoding. Accepts the same values as the
+ builtin open() function.
+ - errors -- A string indicating how encoding and decoding errors are to
+ be handled. Accepts the same value as the builtin open() function.
+
+ Backported from CPython 3.12
+ """
+
+ def __init__(
+ self,
+ mode: str = "r",
+ bufsize: int = -1,
+ encoding: str | None = None,
+ errors: str | None = None,
+ ):
+ self._mode = mode
+ self._bufsize = bufsize
+ self._encoding = encoding
+ self._errors = errors
+
+ def __call__(self, string: str) -> typing.IO: # type: ignore[type-arg]
+ # the special argument "-" means sys.std{in,out}
+ if string == "-":
+ if "r" in self._mode:
+ return sys.stdin.buffer if "b" in self._mode else sys.stdin
+ elif any(c in self._mode for c in "wax"):
+ return sys.stdout.buffer if "b" in self._mode else sys.stdout
+ else:
+ msg = f'argument "-" with mode {self._mode}'
+ raise ValueError(msg)
+
+ # all other arguments are used as file names
+ try:
+ return open(string, self._mode, self._bufsize, self._encoding, self._errors)
+ except OSError as e:
+ message = f"can't open '{string}': {e}"
+ raise argparse.ArgumentTypeError(message)
+
+ def __repr__(self) -> str:
+ args = self._mode, self._bufsize
+ kwargs = [("encoding", self._encoding), ("errors", self._errors)]
+ args_str = ", ".join(
+ [repr(arg) for arg in args if arg != -1]
+ + [f"{kw}={arg!r}" for kw, arg in kwargs if arg is not None]
+ )
+ return f"{type(self).__name__}({args_str})"
+
+
+def cli_detect(argv: list[str] | None = None) -> int:
+ """
+ CLI assistant using ARGV and ArgumentParser
+ :param argv:
+ :return: 0 if everything is fine, anything else equal trouble
+ """
+ parser = argparse.ArgumentParser(
+ description="The Real First Universal Charset Detector. "
+ "Discover originating encoding used on text file. "
+ "Normalize text to unicode."
+ )
+
+ parser.add_argument(
+ "files", type=FileType("rb"), nargs="+", help="File(s) to be analysed"
+ )
+ parser.add_argument(
+ "-v",
+ "--verbose",
+ action="store_true",
+ default=False,
+ dest="verbose",
+ help="Display complementary information about file if any. "
+ "Stdout will contain logs about the detection process.",
+ )
+ parser.add_argument(
+ "-a",
+ "--with-alternative",
+ action="store_true",
+ default=False,
+ dest="alternatives",
+ help="Output complementary possibilities if any. Top-level JSON WILL be a list.",
+ )
+ parser.add_argument(
+ "-n",
+ "--normalize",
+ action="store_true",
+ default=False,
+ dest="normalize",
+ help="Permit to normalize input file. If not set, program does not write anything.",
+ )
+ parser.add_argument(
+ "-m",
+ "--minimal",
+ action="store_true",
+ default=False,
+ dest="minimal",
+ help="Only output the charset detected to STDOUT. Disabling JSON output.",
+ )
+ parser.add_argument(
+ "-r",
+ "--replace",
+ action="store_true",
+ default=False,
+ dest="replace",
+ help="Replace file when trying to normalize it instead of creating a new one.",
+ )
+ parser.add_argument(
+ "-f",
+ "--force",
+ action="store_true",
+ default=False,
+ dest="force",
+ help="Replace file without asking if you are sure, use this flag with caution.",
+ )
+ parser.add_argument(
+ "-i",
+ "--no-preemptive",
+ action="store_true",
+ default=False,
+ dest="no_preemptive",
+ help="Disable looking at a charset declaration to hint the detector.",
+ )
+ parser.add_argument(
+ "-t",
+ "--threshold",
+ action="store",
+ default=0.2,
+ type=float,
+ dest="threshold",
+ help="Define a custom maximum amount of noise allowed in decoded content. 0. <= noise <= 1.",
+ )
+ parser.add_argument(
+ "--version",
+ action="version",
+ version="Charset-Normalizer {} - Python {} - Unicode {} - SpeedUp {}".format(
+ __version__,
+ python_version(),
+ unidata_version,
+ "OFF" if md_module.__file__.lower().endswith(".py") else "ON",
+ ),
+ help="Show version information and exit.",
+ )
+
+ args = parser.parse_args(argv)
+
+ if args.replace is True and args.normalize is False:
+ if args.files:
+ for my_file in args.files:
+ my_file.close()
+ print("Use --replace in addition of --normalize only.", file=sys.stderr)
+ return 1
+
+ if args.force is True and args.replace is False:
+ if args.files:
+ for my_file in args.files:
+ my_file.close()
+ print("Use --force in addition of --replace only.", file=sys.stderr)
+ return 1
+
+ if args.threshold < 0.0 or args.threshold > 1.0:
+ if args.files:
+ for my_file in args.files:
+ my_file.close()
+ print("--threshold VALUE should be between 0. AND 1.", file=sys.stderr)
+ return 1
+
+ x_ = []
+
+ for my_file in args.files:
+ matches = from_fp(
+ my_file,
+ threshold=args.threshold,
+ explain=args.verbose,
+ preemptive_behaviour=args.no_preemptive is False,
+ )
+
+ best_guess = matches.best()
+
+ if best_guess is None:
+ print(
+ 'Unable to identify originating encoding for "{}". {}'.format(
+ my_file.name,
+ (
+ "Maybe try increasing maximum amount of chaos."
+ if args.threshold < 1.0
+ else ""
+ ),
+ ),
+ file=sys.stderr,
+ )
+ x_.append(
+ CliDetectionResult(
+ abspath(my_file.name),
+ None,
+ [],
+ [],
+ "Unknown",
+ [],
+ False,
+ 1.0,
+ 0.0,
+ None,
+ True,
+ )
+ )
+ else:
+ x_.append(
+ CliDetectionResult(
+ abspath(my_file.name),
+ best_guess.encoding,
+ best_guess.encoding_aliases,
+ [
+ cp
+ for cp in best_guess.could_be_from_charset
+ if cp != best_guess.encoding
+ ],
+ best_guess.language,
+ best_guess.alphabets,
+ best_guess.bom,
+ best_guess.percent_chaos,
+ best_guess.percent_coherence,
+ None,
+ True,
+ )
+ )
+
+ if len(matches) > 1 and args.alternatives:
+ for el in matches:
+ if el != best_guess:
+ x_.append(
+ CliDetectionResult(
+ abspath(my_file.name),
+ el.encoding,
+ el.encoding_aliases,
+ [
+ cp
+ for cp in el.could_be_from_charset
+ if cp != el.encoding
+ ],
+ el.language,
+ el.alphabets,
+ el.bom,
+ el.percent_chaos,
+ el.percent_coherence,
+ None,
+ False,
+ )
+ )
+
+ if args.normalize is True:
+ if best_guess.encoding.startswith("utf") is True:
+ print(
+ '"{}" file does not need to be normalized, as it already came from unicode.'.format(
+ my_file.name
+ ),
+ file=sys.stderr,
+ )
+ if my_file.closed is False:
+ my_file.close()
+ continue
+
+ dir_path = dirname(realpath(my_file.name))
+ file_name = basename(realpath(my_file.name))
+
+ o_: list[str] = file_name.split(".")
+
+ if args.replace is False:
+ o_.insert(-1, best_guess.encoding)
+ if my_file.closed is False:
+ my_file.close()
+ elif (
+ args.force is False
+ and query_yes_no(
+ 'Are you sure to normalize "{}" by replacing it ?'.format(
+ my_file.name
+ ),
+ "no",
+ )
+ is False
+ ):
+ if my_file.closed is False:
+ my_file.close()
+ continue
+
+ try:
+ x_[0].unicode_path = join(dir_path, ".".join(o_))
+
+ with open(x_[0].unicode_path, "wb") as fp:
+ fp.write(best_guess.output())
+ except OSError as e:
+ print(str(e), file=sys.stderr)
+ if my_file.closed is False:
+ my_file.close()
+ return 2
+
+ if my_file.closed is False:
+ my_file.close()
+
+ if args.minimal is False:
+ print(
+ dumps(
+ [el.__dict__ for el in x_] if len(x_) > 1 else x_[0].__dict__,
+ ensure_ascii=True,
+ indent=4,
+ )
+ )
+ else:
+ for my_file in args.files:
+ print(
+ ", ".join(
+ [
+ el.encoding or "undefined"
+ for el in x_
+ if el.path == abspath(my_file.name)
+ ]
+ )
+ )
+
+ return 0
+
+
+if __name__ == "__main__":
+ cli_detect()
diff --git a/venv/lib/python3.10/site-packages/charset_normalizer/cli/__pycache__/__init__.cpython-310.pyc b/venv/lib/python3.10/site-packages/charset_normalizer/cli/__pycache__/__init__.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..0835dff20464d55e94ddea81e3424c77446a05d8
Binary files /dev/null and b/venv/lib/python3.10/site-packages/charset_normalizer/cli/__pycache__/__init__.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/charset_normalizer/cli/__pycache__/__main__.cpython-310.pyc b/venv/lib/python3.10/site-packages/charset_normalizer/cli/__pycache__/__main__.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..3efb8027ce83cb1c564ae170498ab699a43af23b
Binary files /dev/null and b/venv/lib/python3.10/site-packages/charset_normalizer/cli/__pycache__/__main__.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/charset_normalizer/constant.py b/venv/lib/python3.10/site-packages/charset_normalizer/constant.py
new file mode 100644
index 0000000000000000000000000000000000000000..cc71a019c7fdd7b99c7fdd5b6bb084ab2840ff63
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/charset_normalizer/constant.py
@@ -0,0 +1,2015 @@
+from __future__ import annotations
+
+from codecs import BOM_UTF8, BOM_UTF16_BE, BOM_UTF16_LE, BOM_UTF32_BE, BOM_UTF32_LE
+from encodings.aliases import aliases
+from re import IGNORECASE
+from re import compile as re_compile
+
+# Contain for each eligible encoding a list of/item bytes SIG/BOM
+ENCODING_MARKS: dict[str, bytes | list[bytes]] = {
+ "utf_8": BOM_UTF8,
+ "utf_7": [
+ b"\x2b\x2f\x76\x38",
+ b"\x2b\x2f\x76\x39",
+ b"\x2b\x2f\x76\x2b",
+ b"\x2b\x2f\x76\x2f",
+ b"\x2b\x2f\x76\x38\x2d",
+ ],
+ "gb18030": b"\x84\x31\x95\x33",
+ "utf_32": [BOM_UTF32_BE, BOM_UTF32_LE],
+ "utf_16": [BOM_UTF16_BE, BOM_UTF16_LE],
+}
+
+TOO_SMALL_SEQUENCE: int = 32
+TOO_BIG_SEQUENCE: int = int(10e6)
+
+UTF8_MAXIMAL_ALLOCATION: int = 1_112_064
+
+# Up-to-date Unicode ucd/15.0.0
+UNICODE_RANGES_COMBINED: dict[str, range] = {
+ "Control character": range(32),
+ "Basic Latin": range(32, 128),
+ "Latin-1 Supplement": range(128, 256),
+ "Latin Extended-A": range(256, 384),
+ "Latin Extended-B": range(384, 592),
+ "IPA Extensions": range(592, 688),
+ "Spacing Modifier Letters": range(688, 768),
+ "Combining Diacritical Marks": range(768, 880),
+ "Greek and Coptic": range(880, 1024),
+ "Cyrillic": range(1024, 1280),
+ "Cyrillic Supplement": range(1280, 1328),
+ "Armenian": range(1328, 1424),
+ "Hebrew": range(1424, 1536),
+ "Arabic": range(1536, 1792),
+ "Syriac": range(1792, 1872),
+ "Arabic Supplement": range(1872, 1920),
+ "Thaana": range(1920, 1984),
+ "NKo": range(1984, 2048),
+ "Samaritan": range(2048, 2112),
+ "Mandaic": range(2112, 2144),
+ "Syriac Supplement": range(2144, 2160),
+ "Arabic Extended-B": range(2160, 2208),
+ "Arabic Extended-A": range(2208, 2304),
+ "Devanagari": range(2304, 2432),
+ "Bengali": range(2432, 2560),
+ "Gurmukhi": range(2560, 2688),
+ "Gujarati": range(2688, 2816),
+ "Oriya": range(2816, 2944),
+ "Tamil": range(2944, 3072),
+ "Telugu": range(3072, 3200),
+ "Kannada": range(3200, 3328),
+ "Malayalam": range(3328, 3456),
+ "Sinhala": range(3456, 3584),
+ "Thai": range(3584, 3712),
+ "Lao": range(3712, 3840),
+ "Tibetan": range(3840, 4096),
+ "Myanmar": range(4096, 4256),
+ "Georgian": range(4256, 4352),
+ "Hangul Jamo": range(4352, 4608),
+ "Ethiopic": range(4608, 4992),
+ "Ethiopic Supplement": range(4992, 5024),
+ "Cherokee": range(5024, 5120),
+ "Unified Canadian Aboriginal Syllabics": range(5120, 5760),
+ "Ogham": range(5760, 5792),
+ "Runic": range(5792, 5888),
+ "Tagalog": range(5888, 5920),
+ "Hanunoo": range(5920, 5952),
+ "Buhid": range(5952, 5984),
+ "Tagbanwa": range(5984, 6016),
+ "Khmer": range(6016, 6144),
+ "Mongolian": range(6144, 6320),
+ "Unified Canadian Aboriginal Syllabics Extended": range(6320, 6400),
+ "Limbu": range(6400, 6480),
+ "Tai Le": range(6480, 6528),
+ "New Tai Lue": range(6528, 6624),
+ "Khmer Symbols": range(6624, 6656),
+ "Buginese": range(6656, 6688),
+ "Tai Tham": range(6688, 6832),
+ "Combining Diacritical Marks Extended": range(6832, 6912),
+ "Balinese": range(6912, 7040),
+ "Sundanese": range(7040, 7104),
+ "Batak": range(7104, 7168),
+ "Lepcha": range(7168, 7248),
+ "Ol Chiki": range(7248, 7296),
+ "Cyrillic Extended-C": range(7296, 7312),
+ "Georgian Extended": range(7312, 7360),
+ "Sundanese Supplement": range(7360, 7376),
+ "Vedic Extensions": range(7376, 7424),
+ "Phonetic Extensions": range(7424, 7552),
+ "Phonetic Extensions Supplement": range(7552, 7616),
+ "Combining Diacritical Marks Supplement": range(7616, 7680),
+ "Latin Extended Additional": range(7680, 7936),
+ "Greek Extended": range(7936, 8192),
+ "General Punctuation": range(8192, 8304),
+ "Superscripts and Subscripts": range(8304, 8352),
+ "Currency Symbols": range(8352, 8400),
+ "Combining Diacritical Marks for Symbols": range(8400, 8448),
+ "Letterlike Symbols": range(8448, 8528),
+ "Number Forms": range(8528, 8592),
+ "Arrows": range(8592, 8704),
+ "Mathematical Operators": range(8704, 8960),
+ "Miscellaneous Technical": range(8960, 9216),
+ "Control Pictures": range(9216, 9280),
+ "Optical Character Recognition": range(9280, 9312),
+ "Enclosed Alphanumerics": range(9312, 9472),
+ "Box Drawing": range(9472, 9600),
+ "Block Elements": range(9600, 9632),
+ "Geometric Shapes": range(9632, 9728),
+ "Miscellaneous Symbols": range(9728, 9984),
+ "Dingbats": range(9984, 10176),
+ "Miscellaneous Mathematical Symbols-A": range(10176, 10224),
+ "Supplemental Arrows-A": range(10224, 10240),
+ "Braille Patterns": range(10240, 10496),
+ "Supplemental Arrows-B": range(10496, 10624),
+ "Miscellaneous Mathematical Symbols-B": range(10624, 10752),
+ "Supplemental Mathematical Operators": range(10752, 11008),
+ "Miscellaneous Symbols and Arrows": range(11008, 11264),
+ "Glagolitic": range(11264, 11360),
+ "Latin Extended-C": range(11360, 11392),
+ "Coptic": range(11392, 11520),
+ "Georgian Supplement": range(11520, 11568),
+ "Tifinagh": range(11568, 11648),
+ "Ethiopic Extended": range(11648, 11744),
+ "Cyrillic Extended-A": range(11744, 11776),
+ "Supplemental Punctuation": range(11776, 11904),
+ "CJK Radicals Supplement": range(11904, 12032),
+ "Kangxi Radicals": range(12032, 12256),
+ "Ideographic Description Characters": range(12272, 12288),
+ "CJK Symbols and Punctuation": range(12288, 12352),
+ "Hiragana": range(12352, 12448),
+ "Katakana": range(12448, 12544),
+ "Bopomofo": range(12544, 12592),
+ "Hangul Compatibility Jamo": range(12592, 12688),
+ "Kanbun": range(12688, 12704),
+ "Bopomofo Extended": range(12704, 12736),
+ "CJK Strokes": range(12736, 12784),
+ "Katakana Phonetic Extensions": range(12784, 12800),
+ "Enclosed CJK Letters and Months": range(12800, 13056),
+ "CJK Compatibility": range(13056, 13312),
+ "CJK Unified Ideographs Extension A": range(13312, 19904),
+ "Yijing Hexagram Symbols": range(19904, 19968),
+ "CJK Unified Ideographs": range(19968, 40960),
+ "Yi Syllables": range(40960, 42128),
+ "Yi Radicals": range(42128, 42192),
+ "Lisu": range(42192, 42240),
+ "Vai": range(42240, 42560),
+ "Cyrillic Extended-B": range(42560, 42656),
+ "Bamum": range(42656, 42752),
+ "Modifier Tone Letters": range(42752, 42784),
+ "Latin Extended-D": range(42784, 43008),
+ "Syloti Nagri": range(43008, 43056),
+ "Common Indic Number Forms": range(43056, 43072),
+ "Phags-pa": range(43072, 43136),
+ "Saurashtra": range(43136, 43232),
+ "Devanagari Extended": range(43232, 43264),
+ "Kayah Li": range(43264, 43312),
+ "Rejang": range(43312, 43360),
+ "Hangul Jamo Extended-A": range(43360, 43392),
+ "Javanese": range(43392, 43488),
+ "Myanmar Extended-B": range(43488, 43520),
+ "Cham": range(43520, 43616),
+ "Myanmar Extended-A": range(43616, 43648),
+ "Tai Viet": range(43648, 43744),
+ "Meetei Mayek Extensions": range(43744, 43776),
+ "Ethiopic Extended-A": range(43776, 43824),
+ "Latin Extended-E": range(43824, 43888),
+ "Cherokee Supplement": range(43888, 43968),
+ "Meetei Mayek": range(43968, 44032),
+ "Hangul Syllables": range(44032, 55216),
+ "Hangul Jamo Extended-B": range(55216, 55296),
+ "High Surrogates": range(55296, 56192),
+ "High Private Use Surrogates": range(56192, 56320),
+ "Low Surrogates": range(56320, 57344),
+ "Private Use Area": range(57344, 63744),
+ "CJK Compatibility Ideographs": range(63744, 64256),
+ "Alphabetic Presentation Forms": range(64256, 64336),
+ "Arabic Presentation Forms-A": range(64336, 65024),
+ "Variation Selectors": range(65024, 65040),
+ "Vertical Forms": range(65040, 65056),
+ "Combining Half Marks": range(65056, 65072),
+ "CJK Compatibility Forms": range(65072, 65104),
+ "Small Form Variants": range(65104, 65136),
+ "Arabic Presentation Forms-B": range(65136, 65280),
+ "Halfwidth and Fullwidth Forms": range(65280, 65520),
+ "Specials": range(65520, 65536),
+ "Linear B Syllabary": range(65536, 65664),
+ "Linear B Ideograms": range(65664, 65792),
+ "Aegean Numbers": range(65792, 65856),
+ "Ancient Greek Numbers": range(65856, 65936),
+ "Ancient Symbols": range(65936, 66000),
+ "Phaistos Disc": range(66000, 66048),
+ "Lycian": range(66176, 66208),
+ "Carian": range(66208, 66272),
+ "Coptic Epact Numbers": range(66272, 66304),
+ "Old Italic": range(66304, 66352),
+ "Gothic": range(66352, 66384),
+ "Old Permic": range(66384, 66432),
+ "Ugaritic": range(66432, 66464),
+ "Old Persian": range(66464, 66528),
+ "Deseret": range(66560, 66640),
+ "Shavian": range(66640, 66688),
+ "Osmanya": range(66688, 66736),
+ "Osage": range(66736, 66816),
+ "Elbasan": range(66816, 66864),
+ "Caucasian Albanian": range(66864, 66928),
+ "Vithkuqi": range(66928, 67008),
+ "Linear A": range(67072, 67456),
+ "Latin Extended-F": range(67456, 67520),
+ "Cypriot Syllabary": range(67584, 67648),
+ "Imperial Aramaic": range(67648, 67680),
+ "Palmyrene": range(67680, 67712),
+ "Nabataean": range(67712, 67760),
+ "Hatran": range(67808, 67840),
+ "Phoenician": range(67840, 67872),
+ "Lydian": range(67872, 67904),
+ "Meroitic Hieroglyphs": range(67968, 68000),
+ "Meroitic Cursive": range(68000, 68096),
+ "Kharoshthi": range(68096, 68192),
+ "Old South Arabian": range(68192, 68224),
+ "Old North Arabian": range(68224, 68256),
+ "Manichaean": range(68288, 68352),
+ "Avestan": range(68352, 68416),
+ "Inscriptional Parthian": range(68416, 68448),
+ "Inscriptional Pahlavi": range(68448, 68480),
+ "Psalter Pahlavi": range(68480, 68528),
+ "Old Turkic": range(68608, 68688),
+ "Old Hungarian": range(68736, 68864),
+ "Hanifi Rohingya": range(68864, 68928),
+ "Rumi Numeral Symbols": range(69216, 69248),
+ "Yezidi": range(69248, 69312),
+ "Arabic Extended-C": range(69312, 69376),
+ "Old Sogdian": range(69376, 69424),
+ "Sogdian": range(69424, 69488),
+ "Old Uyghur": range(69488, 69552),
+ "Chorasmian": range(69552, 69600),
+ "Elymaic": range(69600, 69632),
+ "Brahmi": range(69632, 69760),
+ "Kaithi": range(69760, 69840),
+ "Sora Sompeng": range(69840, 69888),
+ "Chakma": range(69888, 69968),
+ "Mahajani": range(69968, 70016),
+ "Sharada": range(70016, 70112),
+ "Sinhala Archaic Numbers": range(70112, 70144),
+ "Khojki": range(70144, 70224),
+ "Multani": range(70272, 70320),
+ "Khudawadi": range(70320, 70400),
+ "Grantha": range(70400, 70528),
+ "Newa": range(70656, 70784),
+ "Tirhuta": range(70784, 70880),
+ "Siddham": range(71040, 71168),
+ "Modi": range(71168, 71264),
+ "Mongolian Supplement": range(71264, 71296),
+ "Takri": range(71296, 71376),
+ "Ahom": range(71424, 71504),
+ "Dogra": range(71680, 71760),
+ "Warang Citi": range(71840, 71936),
+ "Dives Akuru": range(71936, 72032),
+ "Nandinagari": range(72096, 72192),
+ "Zanabazar Square": range(72192, 72272),
+ "Soyombo": range(72272, 72368),
+ "Unified Canadian Aboriginal Syllabics Extended-A": range(72368, 72384),
+ "Pau Cin Hau": range(72384, 72448),
+ "Devanagari Extended-A": range(72448, 72544),
+ "Bhaiksuki": range(72704, 72816),
+ "Marchen": range(72816, 72896),
+ "Masaram Gondi": range(72960, 73056),
+ "Gunjala Gondi": range(73056, 73136),
+ "Makasar": range(73440, 73472),
+ "Kawi": range(73472, 73568),
+ "Lisu Supplement": range(73648, 73664),
+ "Tamil Supplement": range(73664, 73728),
+ "Cuneiform": range(73728, 74752),
+ "Cuneiform Numbers and Punctuation": range(74752, 74880),
+ "Early Dynastic Cuneiform": range(74880, 75088),
+ "Cypro-Minoan": range(77712, 77824),
+ "Egyptian Hieroglyphs": range(77824, 78896),
+ "Egyptian Hieroglyph Format Controls": range(78896, 78944),
+ "Anatolian Hieroglyphs": range(82944, 83584),
+ "Bamum Supplement": range(92160, 92736),
+ "Mro": range(92736, 92784),
+ "Tangsa": range(92784, 92880),
+ "Bassa Vah": range(92880, 92928),
+ "Pahawh Hmong": range(92928, 93072),
+ "Medefaidrin": range(93760, 93856),
+ "Miao": range(93952, 94112),
+ "Ideographic Symbols and Punctuation": range(94176, 94208),
+ "Tangut": range(94208, 100352),
+ "Tangut Components": range(100352, 101120),
+ "Khitan Small Script": range(101120, 101632),
+ "Tangut Supplement": range(101632, 101760),
+ "Kana Extended-B": range(110576, 110592),
+ "Kana Supplement": range(110592, 110848),
+ "Kana Extended-A": range(110848, 110896),
+ "Small Kana Extension": range(110896, 110960),
+ "Nushu": range(110960, 111360),
+ "Duployan": range(113664, 113824),
+ "Shorthand Format Controls": range(113824, 113840),
+ "Znamenny Musical Notation": range(118528, 118736),
+ "Byzantine Musical Symbols": range(118784, 119040),
+ "Musical Symbols": range(119040, 119296),
+ "Ancient Greek Musical Notation": range(119296, 119376),
+ "Kaktovik Numerals": range(119488, 119520),
+ "Mayan Numerals": range(119520, 119552),
+ "Tai Xuan Jing Symbols": range(119552, 119648),
+ "Counting Rod Numerals": range(119648, 119680),
+ "Mathematical Alphanumeric Symbols": range(119808, 120832),
+ "Sutton SignWriting": range(120832, 121520),
+ "Latin Extended-G": range(122624, 122880),
+ "Glagolitic Supplement": range(122880, 122928),
+ "Cyrillic Extended-D": range(122928, 123024),
+ "Nyiakeng Puachue Hmong": range(123136, 123216),
+ "Toto": range(123536, 123584),
+ "Wancho": range(123584, 123648),
+ "Nag Mundari": range(124112, 124160),
+ "Ethiopic Extended-B": range(124896, 124928),
+ "Mende Kikakui": range(124928, 125152),
+ "Adlam": range(125184, 125280),
+ "Indic Siyaq Numbers": range(126064, 126144),
+ "Ottoman Siyaq Numbers": range(126208, 126288),
+ "Arabic Mathematical Alphabetic Symbols": range(126464, 126720),
+ "Mahjong Tiles": range(126976, 127024),
+ "Domino Tiles": range(127024, 127136),
+ "Playing Cards": range(127136, 127232),
+ "Enclosed Alphanumeric Supplement": range(127232, 127488),
+ "Enclosed Ideographic Supplement": range(127488, 127744),
+ "Miscellaneous Symbols and Pictographs": range(127744, 128512),
+ "Emoticons range(Emoji)": range(128512, 128592),
+ "Ornamental Dingbats": range(128592, 128640),
+ "Transport and Map Symbols": range(128640, 128768),
+ "Alchemical Symbols": range(128768, 128896),
+ "Geometric Shapes Extended": range(128896, 129024),
+ "Supplemental Arrows-C": range(129024, 129280),
+ "Supplemental Symbols and Pictographs": range(129280, 129536),
+ "Chess Symbols": range(129536, 129648),
+ "Symbols and Pictographs Extended-A": range(129648, 129792),
+ "Symbols for Legacy Computing": range(129792, 130048),
+ "CJK Unified Ideographs Extension B": range(131072, 173792),
+ "CJK Unified Ideographs Extension C": range(173824, 177984),
+ "CJK Unified Ideographs Extension D": range(177984, 178208),
+ "CJK Unified Ideographs Extension E": range(178208, 183984),
+ "CJK Unified Ideographs Extension F": range(183984, 191472),
+ "CJK Compatibility Ideographs Supplement": range(194560, 195104),
+ "CJK Unified Ideographs Extension G": range(196608, 201552),
+ "CJK Unified Ideographs Extension H": range(201552, 205744),
+ "Tags": range(917504, 917632),
+ "Variation Selectors Supplement": range(917760, 918000),
+ "Supplementary Private Use Area-A": range(983040, 1048576),
+ "Supplementary Private Use Area-B": range(1048576, 1114112),
+}
+
+
+UNICODE_SECONDARY_RANGE_KEYWORD: list[str] = [
+ "Supplement",
+ "Extended",
+ "Extensions",
+ "Modifier",
+ "Marks",
+ "Punctuation",
+ "Symbols",
+ "Forms",
+ "Operators",
+ "Miscellaneous",
+ "Drawing",
+ "Block",
+ "Shapes",
+ "Supplemental",
+ "Tags",
+]
+
+RE_POSSIBLE_ENCODING_INDICATION = re_compile(
+ r"(?:(?:encoding)|(?:charset)|(?:coding))(?:[\:= ]{1,10})(?:[\"\']?)([a-zA-Z0-9\-_]+)(?:[\"\']?)",
+ IGNORECASE,
+)
+
+IANA_NO_ALIASES = [
+ "cp720",
+ "cp737",
+ "cp856",
+ "cp874",
+ "cp875",
+ "cp1006",
+ "koi8_r",
+ "koi8_t",
+ "koi8_u",
+]
+
+IANA_SUPPORTED: list[str] = sorted(
+ filter(
+ lambda x: x.endswith("_codec") is False
+ and x not in {"rot_13", "tactis", "mbcs"},
+ list(set(aliases.values())) + IANA_NO_ALIASES,
+ )
+)
+
+IANA_SUPPORTED_COUNT: int = len(IANA_SUPPORTED)
+
+# pre-computed code page that are similar using the function cp_similarity.
+IANA_SUPPORTED_SIMILAR: dict[str, list[str]] = {
+ "cp037": ["cp1026", "cp1140", "cp273", "cp500"],
+ "cp1026": ["cp037", "cp1140", "cp273", "cp500"],
+ "cp1125": ["cp866"],
+ "cp1140": ["cp037", "cp1026", "cp273", "cp500"],
+ "cp1250": ["iso8859_2"],
+ "cp1251": ["kz1048", "ptcp154"],
+ "cp1252": ["iso8859_15", "iso8859_9", "latin_1"],
+ "cp1253": ["iso8859_7"],
+ "cp1254": ["iso8859_15", "iso8859_9", "latin_1"],
+ "cp1257": ["iso8859_13"],
+ "cp273": ["cp037", "cp1026", "cp1140", "cp500"],
+ "cp437": ["cp850", "cp858", "cp860", "cp861", "cp862", "cp863", "cp865"],
+ "cp500": ["cp037", "cp1026", "cp1140", "cp273"],
+ "cp850": ["cp437", "cp857", "cp858", "cp865"],
+ "cp857": ["cp850", "cp858", "cp865"],
+ "cp858": ["cp437", "cp850", "cp857", "cp865"],
+ "cp860": ["cp437", "cp861", "cp862", "cp863", "cp865"],
+ "cp861": ["cp437", "cp860", "cp862", "cp863", "cp865"],
+ "cp862": ["cp437", "cp860", "cp861", "cp863", "cp865"],
+ "cp863": ["cp437", "cp860", "cp861", "cp862", "cp865"],
+ "cp865": ["cp437", "cp850", "cp857", "cp858", "cp860", "cp861", "cp862", "cp863"],
+ "cp866": ["cp1125"],
+ "iso8859_10": ["iso8859_14", "iso8859_15", "iso8859_4", "iso8859_9", "latin_1"],
+ "iso8859_11": ["tis_620"],
+ "iso8859_13": ["cp1257"],
+ "iso8859_14": [
+ "iso8859_10",
+ "iso8859_15",
+ "iso8859_16",
+ "iso8859_3",
+ "iso8859_9",
+ "latin_1",
+ ],
+ "iso8859_15": [
+ "cp1252",
+ "cp1254",
+ "iso8859_10",
+ "iso8859_14",
+ "iso8859_16",
+ "iso8859_3",
+ "iso8859_9",
+ "latin_1",
+ ],
+ "iso8859_16": [
+ "iso8859_14",
+ "iso8859_15",
+ "iso8859_2",
+ "iso8859_3",
+ "iso8859_9",
+ "latin_1",
+ ],
+ "iso8859_2": ["cp1250", "iso8859_16", "iso8859_4"],
+ "iso8859_3": ["iso8859_14", "iso8859_15", "iso8859_16", "iso8859_9", "latin_1"],
+ "iso8859_4": ["iso8859_10", "iso8859_2", "iso8859_9", "latin_1"],
+ "iso8859_7": ["cp1253"],
+ "iso8859_9": [
+ "cp1252",
+ "cp1254",
+ "cp1258",
+ "iso8859_10",
+ "iso8859_14",
+ "iso8859_15",
+ "iso8859_16",
+ "iso8859_3",
+ "iso8859_4",
+ "latin_1",
+ ],
+ "kz1048": ["cp1251", "ptcp154"],
+ "latin_1": [
+ "cp1252",
+ "cp1254",
+ "cp1258",
+ "iso8859_10",
+ "iso8859_14",
+ "iso8859_15",
+ "iso8859_16",
+ "iso8859_3",
+ "iso8859_4",
+ "iso8859_9",
+ ],
+ "mac_iceland": ["mac_roman", "mac_turkish"],
+ "mac_roman": ["mac_iceland", "mac_turkish"],
+ "mac_turkish": ["mac_iceland", "mac_roman"],
+ "ptcp154": ["cp1251", "kz1048"],
+ "tis_620": ["iso8859_11"],
+}
+
+
+CHARDET_CORRESPONDENCE: dict[str, str] = {
+ "iso2022_kr": "ISO-2022-KR",
+ "iso2022_jp": "ISO-2022-JP",
+ "euc_kr": "EUC-KR",
+ "tis_620": "TIS-620",
+ "utf_32": "UTF-32",
+ "euc_jp": "EUC-JP",
+ "koi8_r": "KOI8-R",
+ "iso8859_1": "ISO-8859-1",
+ "iso8859_2": "ISO-8859-2",
+ "iso8859_5": "ISO-8859-5",
+ "iso8859_6": "ISO-8859-6",
+ "iso8859_7": "ISO-8859-7",
+ "iso8859_8": "ISO-8859-8",
+ "utf_16": "UTF-16",
+ "cp855": "IBM855",
+ "mac_cyrillic": "MacCyrillic",
+ "gb2312": "GB2312",
+ "gb18030": "GB18030",
+ "cp932": "CP932",
+ "cp866": "IBM866",
+ "utf_8": "utf-8",
+ "utf_8_sig": "UTF-8-SIG",
+ "shift_jis": "SHIFT_JIS",
+ "big5": "Big5",
+ "cp1250": "windows-1250",
+ "cp1251": "windows-1251",
+ "cp1252": "Windows-1252",
+ "cp1253": "windows-1253",
+ "cp1255": "windows-1255",
+ "cp1256": "windows-1256",
+ "cp1254": "Windows-1254",
+ "cp949": "CP949",
+}
+
+
+COMMON_SAFE_ASCII_CHARACTERS: set[str] = {
+ "<",
+ ">",
+ "=",
+ ":",
+ "/",
+ "&",
+ ";",
+ "{",
+ "}",
+ "[",
+ "]",
+ ",",
+ "|",
+ '"',
+ "-",
+ "(",
+ ")",
+}
+
+# Sample character sets — replace with full lists if needed
+COMMON_CHINESE_CHARACTERS = "的一是在不了有和人这中大为上个国我以要他时来用们生到作地于出就分对成会可主发年动同工也能下过子说产种面而方后多定行学法所民得经十三之进着等部度家电力里如水化高自二理起小物现实加量都两体制机当使点从业本去把性好应开它合还因由其些然前外天政四日那社义事平形相全表间样与关各重新线内数正心反你明看原又么利比或但质气第向道命此变条只没结解问意建月公无系军很情者最立代想已通并提直题党程展五果料象员革位入常文总次品式活设及管特件长求老头基资边流路级少图山统接知较将组见计别她手角期根论运农指几九区强放决西被干做必战先回则任取据处队南给色光门即保治北造百规热领七海口东导器压志世金增争济阶油思术极交受联什认六共权收证改清己美再采转更单风切打白教速花带安场身车例真务具万每目至达走积示议声报斗完类八离华名确才科张信马节话米整空元况今集温传土许步群广石记需段研界拉林律叫且究观越织装影算低持音众书布复容儿须际商非验连断深难近矿千周委素技备半办青省列习响约支般史感劳便团往酸历市克何除消构府太准精值号率族维划选标写存候毛亲快效斯院查江型眼王按格养易置派层片始却专状育厂京识适属圆包火住调满县局照参红细引听该铁价严龙飞"
+
+COMMON_JAPANESE_CHARACTERS = "日一国年大十二本中長出三時行見月分後前生五間上東四今金九入学高円子外八六下来気小七山話女北午百書先名川千水半男西電校語土木聞食車何南万毎白天母火右読友左休父雨"
+
+COMMON_KOREAN_CHARACTERS = "一二三四五六七八九十百千萬上下左右中人女子大小山川日月火水木金土父母天地國名年時文校學生"
+
+# Combine all into a set
+COMMON_CJK_CHARACTERS = set(
+ "".join(
+ [
+ COMMON_CHINESE_CHARACTERS,
+ COMMON_JAPANESE_CHARACTERS,
+ COMMON_KOREAN_CHARACTERS,
+ ]
+ )
+)
+
+KO_NAMES: set[str] = {"johab", "cp949", "euc_kr"}
+ZH_NAMES: set[str] = {"big5", "cp950", "big5hkscs", "hz"}
+
+# Logging LEVEL below DEBUG
+TRACE: int = 5
+
+
+# Language label that contain the em dash "—"
+# character are to be considered alternative seq to origin
+FREQUENCIES: dict[str, list[str]] = {
+ "English": [
+ "e",
+ "a",
+ "t",
+ "i",
+ "o",
+ "n",
+ "s",
+ "r",
+ "h",
+ "l",
+ "d",
+ "c",
+ "u",
+ "m",
+ "f",
+ "p",
+ "g",
+ "w",
+ "y",
+ "b",
+ "v",
+ "k",
+ "x",
+ "j",
+ "z",
+ "q",
+ ],
+ "English—": [
+ "e",
+ "a",
+ "t",
+ "i",
+ "o",
+ "n",
+ "s",
+ "r",
+ "h",
+ "l",
+ "d",
+ "c",
+ "m",
+ "u",
+ "f",
+ "p",
+ "g",
+ "w",
+ "b",
+ "y",
+ "v",
+ "k",
+ "j",
+ "x",
+ "z",
+ "q",
+ ],
+ "German": [
+ "e",
+ "n",
+ "i",
+ "r",
+ "s",
+ "t",
+ "a",
+ "d",
+ "h",
+ "u",
+ "l",
+ "g",
+ "o",
+ "c",
+ "m",
+ "b",
+ "f",
+ "k",
+ "w",
+ "z",
+ "p",
+ "v",
+ "ü",
+ "ä",
+ "ö",
+ "j",
+ ],
+ "French": [
+ "e",
+ "a",
+ "s",
+ "n",
+ "i",
+ "t",
+ "r",
+ "l",
+ "u",
+ "o",
+ "d",
+ "c",
+ "p",
+ "m",
+ "é",
+ "v",
+ "g",
+ "f",
+ "b",
+ "h",
+ "q",
+ "à",
+ "x",
+ "è",
+ "y",
+ "j",
+ ],
+ "Dutch": [
+ "e",
+ "n",
+ "a",
+ "i",
+ "r",
+ "t",
+ "o",
+ "d",
+ "s",
+ "l",
+ "g",
+ "h",
+ "v",
+ "m",
+ "u",
+ "k",
+ "c",
+ "p",
+ "b",
+ "w",
+ "j",
+ "z",
+ "f",
+ "y",
+ "x",
+ "ë",
+ ],
+ "Italian": [
+ "e",
+ "i",
+ "a",
+ "o",
+ "n",
+ "l",
+ "t",
+ "r",
+ "s",
+ "c",
+ "d",
+ "u",
+ "p",
+ "m",
+ "g",
+ "v",
+ "f",
+ "b",
+ "z",
+ "h",
+ "q",
+ "è",
+ "à",
+ "k",
+ "y",
+ "ò",
+ ],
+ "Polish": [
+ "a",
+ "i",
+ "o",
+ "e",
+ "n",
+ "r",
+ "z",
+ "w",
+ "s",
+ "c",
+ "t",
+ "k",
+ "y",
+ "d",
+ "p",
+ "m",
+ "u",
+ "l",
+ "j",
+ "ł",
+ "g",
+ "b",
+ "h",
+ "ą",
+ "ę",
+ "ó",
+ ],
+ "Spanish": [
+ "e",
+ "a",
+ "o",
+ "n",
+ "s",
+ "r",
+ "i",
+ "l",
+ "d",
+ "t",
+ "c",
+ "u",
+ "m",
+ "p",
+ "b",
+ "g",
+ "v",
+ "f",
+ "y",
+ "ó",
+ "h",
+ "q",
+ "í",
+ "j",
+ "z",
+ "á",
+ ],
+ "Russian": [
+ "о",
+ "а",
+ "е",
+ "и",
+ "н",
+ "с",
+ "т",
+ "р",
+ "в",
+ "л",
+ "к",
+ "м",
+ "д",
+ "п",
+ "у",
+ "г",
+ "я",
+ "ы",
+ "з",
+ "б",
+ "й",
+ "ь",
+ "ч",
+ "х",
+ "ж",
+ "ц",
+ ],
+ # Jap-Kanji
+ "Japanese": [
+ "人",
+ "一",
+ "大",
+ "亅",
+ "丁",
+ "丨",
+ "竹",
+ "笑",
+ "口",
+ "日",
+ "今",
+ "二",
+ "彳",
+ "行",
+ "十",
+ "土",
+ "丶",
+ "寸",
+ "寺",
+ "時",
+ "乙",
+ "丿",
+ "乂",
+ "气",
+ "気",
+ "冂",
+ "巾",
+ "亠",
+ "市",
+ "目",
+ "儿",
+ "見",
+ "八",
+ "小",
+ "凵",
+ "県",
+ "月",
+ "彐",
+ "門",
+ "間",
+ "木",
+ "東",
+ "山",
+ "出",
+ "本",
+ "中",
+ "刀",
+ "分",
+ "耳",
+ "又",
+ "取",
+ "最",
+ "言",
+ "田",
+ "心",
+ "思",
+ "刂",
+ "前",
+ "京",
+ "尹",
+ "事",
+ "生",
+ "厶",
+ "云",
+ "会",
+ "未",
+ "来",
+ "白",
+ "冫",
+ "楽",
+ "灬",
+ "馬",
+ "尸",
+ "尺",
+ "駅",
+ "明",
+ "耂",
+ "者",
+ "了",
+ "阝",
+ "都",
+ "高",
+ "卜",
+ "占",
+ "厂",
+ "广",
+ "店",
+ "子",
+ "申",
+ "奄",
+ "亻",
+ "俺",
+ "上",
+ "方",
+ "冖",
+ "学",
+ "衣",
+ "艮",
+ "食",
+ "自",
+ ],
+ # Jap-Katakana
+ "Japanese—": [
+ "ー",
+ "ン",
+ "ス",
+ "・",
+ "ル",
+ "ト",
+ "リ",
+ "イ",
+ "ア",
+ "ラ",
+ "ッ",
+ "ク",
+ "ド",
+ "シ",
+ "レ",
+ "ジ",
+ "タ",
+ "フ",
+ "ロ",
+ "カ",
+ "テ",
+ "マ",
+ "ィ",
+ "グ",
+ "バ",
+ "ム",
+ "プ",
+ "オ",
+ "コ",
+ "デ",
+ "ニ",
+ "ウ",
+ "メ",
+ "サ",
+ "ビ",
+ "ナ",
+ "ブ",
+ "ャ",
+ "エ",
+ "ュ",
+ "チ",
+ "キ",
+ "ズ",
+ "ダ",
+ "パ",
+ "ミ",
+ "ェ",
+ "ョ",
+ "ハ",
+ "セ",
+ "ベ",
+ "ガ",
+ "モ",
+ "ツ",
+ "ネ",
+ "ボ",
+ "ソ",
+ "ノ",
+ "ァ",
+ "ヴ",
+ "ワ",
+ "ポ",
+ "ペ",
+ "ピ",
+ "ケ",
+ "ゴ",
+ "ギ",
+ "ザ",
+ "ホ",
+ "ゲ",
+ "ォ",
+ "ヤ",
+ "ヒ",
+ "ユ",
+ "ヨ",
+ "ヘ",
+ "ゼ",
+ "ヌ",
+ "ゥ",
+ "ゾ",
+ "ヶ",
+ "ヂ",
+ "ヲ",
+ "ヅ",
+ "ヵ",
+ "ヱ",
+ "ヰ",
+ "ヮ",
+ "ヽ",
+ "゠",
+ "ヾ",
+ "ヷ",
+ "ヿ",
+ "ヸ",
+ "ヹ",
+ "ヺ",
+ ],
+ # Jap-Hiragana
+ "Japanese——": [
+ "の",
+ "に",
+ "る",
+ "た",
+ "と",
+ "は",
+ "し",
+ "い",
+ "を",
+ "で",
+ "て",
+ "が",
+ "な",
+ "れ",
+ "か",
+ "ら",
+ "さ",
+ "っ",
+ "り",
+ "す",
+ "あ",
+ "も",
+ "こ",
+ "ま",
+ "う",
+ "く",
+ "よ",
+ "き",
+ "ん",
+ "め",
+ "お",
+ "け",
+ "そ",
+ "つ",
+ "だ",
+ "や",
+ "え",
+ "ど",
+ "わ",
+ "ち",
+ "み",
+ "せ",
+ "じ",
+ "ば",
+ "へ",
+ "び",
+ "ず",
+ "ろ",
+ "ほ",
+ "げ",
+ "む",
+ "べ",
+ "ひ",
+ "ょ",
+ "ゆ",
+ "ぶ",
+ "ご",
+ "ゃ",
+ "ね",
+ "ふ",
+ "ぐ",
+ "ぎ",
+ "ぼ",
+ "ゅ",
+ "づ",
+ "ざ",
+ "ぞ",
+ "ぬ",
+ "ぜ",
+ "ぱ",
+ "ぽ",
+ "ぷ",
+ "ぴ",
+ "ぃ",
+ "ぁ",
+ "ぇ",
+ "ぺ",
+ "ゞ",
+ "ぢ",
+ "ぉ",
+ "ぅ",
+ "ゐ",
+ "ゝ",
+ "ゑ",
+ "゛",
+ "゜",
+ "ゎ",
+ "ゔ",
+ "゚",
+ "ゟ",
+ "゙",
+ "ゕ",
+ "ゖ",
+ ],
+ "Portuguese": [
+ "a",
+ "e",
+ "o",
+ "s",
+ "i",
+ "r",
+ "d",
+ "n",
+ "t",
+ "m",
+ "u",
+ "c",
+ "l",
+ "p",
+ "g",
+ "v",
+ "b",
+ "f",
+ "h",
+ "ã",
+ "q",
+ "é",
+ "ç",
+ "á",
+ "z",
+ "í",
+ ],
+ "Swedish": [
+ "e",
+ "a",
+ "n",
+ "r",
+ "t",
+ "s",
+ "i",
+ "l",
+ "d",
+ "o",
+ "m",
+ "k",
+ "g",
+ "v",
+ "h",
+ "f",
+ "u",
+ "p",
+ "ä",
+ "c",
+ "b",
+ "ö",
+ "å",
+ "y",
+ "j",
+ "x",
+ ],
+ "Chinese": [
+ "的",
+ "一",
+ "是",
+ "不",
+ "了",
+ "在",
+ "人",
+ "有",
+ "我",
+ "他",
+ "这",
+ "个",
+ "们",
+ "中",
+ "来",
+ "上",
+ "大",
+ "为",
+ "和",
+ "国",
+ "地",
+ "到",
+ "以",
+ "说",
+ "时",
+ "要",
+ "就",
+ "出",
+ "会",
+ "可",
+ "也",
+ "你",
+ "对",
+ "生",
+ "能",
+ "而",
+ "子",
+ "那",
+ "得",
+ "于",
+ "着",
+ "下",
+ "自",
+ "之",
+ "年",
+ "过",
+ "发",
+ "后",
+ "作",
+ "里",
+ "用",
+ "道",
+ "行",
+ "所",
+ "然",
+ "家",
+ "种",
+ "事",
+ "成",
+ "方",
+ "多",
+ "经",
+ "么",
+ "去",
+ "法",
+ "学",
+ "如",
+ "都",
+ "同",
+ "现",
+ "当",
+ "没",
+ "动",
+ "面",
+ "起",
+ "看",
+ "定",
+ "天",
+ "分",
+ "还",
+ "进",
+ "好",
+ "小",
+ "部",
+ "其",
+ "些",
+ "主",
+ "样",
+ "理",
+ "心",
+ "她",
+ "本",
+ "前",
+ "开",
+ "但",
+ "因",
+ "只",
+ "从",
+ "想",
+ "实",
+ ],
+ "Ukrainian": [
+ "о",
+ "а",
+ "н",
+ "і",
+ "и",
+ "р",
+ "в",
+ "т",
+ "е",
+ "с",
+ "к",
+ "л",
+ "у",
+ "д",
+ "м",
+ "п",
+ "з",
+ "я",
+ "ь",
+ "б",
+ "г",
+ "й",
+ "ч",
+ "х",
+ "ц",
+ "ї",
+ ],
+ "Norwegian": [
+ "e",
+ "r",
+ "n",
+ "t",
+ "a",
+ "s",
+ "i",
+ "o",
+ "l",
+ "d",
+ "g",
+ "k",
+ "m",
+ "v",
+ "f",
+ "p",
+ "u",
+ "b",
+ "h",
+ "å",
+ "y",
+ "j",
+ "ø",
+ "c",
+ "æ",
+ "w",
+ ],
+ "Finnish": [
+ "a",
+ "i",
+ "n",
+ "t",
+ "e",
+ "s",
+ "l",
+ "o",
+ "u",
+ "k",
+ "ä",
+ "m",
+ "r",
+ "v",
+ "j",
+ "h",
+ "p",
+ "y",
+ "d",
+ "ö",
+ "g",
+ "c",
+ "b",
+ "f",
+ "w",
+ "z",
+ ],
+ "Vietnamese": [
+ "n",
+ "h",
+ "t",
+ "i",
+ "c",
+ "g",
+ "a",
+ "o",
+ "u",
+ "m",
+ "l",
+ "r",
+ "à",
+ "đ",
+ "s",
+ "e",
+ "v",
+ "p",
+ "b",
+ "y",
+ "ư",
+ "d",
+ "á",
+ "k",
+ "ộ",
+ "ế",
+ ],
+ "Czech": [
+ "o",
+ "e",
+ "a",
+ "n",
+ "t",
+ "s",
+ "i",
+ "l",
+ "v",
+ "r",
+ "k",
+ "d",
+ "u",
+ "m",
+ "p",
+ "í",
+ "c",
+ "h",
+ "z",
+ "á",
+ "y",
+ "j",
+ "b",
+ "ě",
+ "é",
+ "ř",
+ ],
+ "Hungarian": [
+ "e",
+ "a",
+ "t",
+ "l",
+ "s",
+ "n",
+ "k",
+ "r",
+ "i",
+ "o",
+ "z",
+ "á",
+ "é",
+ "g",
+ "m",
+ "b",
+ "y",
+ "v",
+ "d",
+ "h",
+ "u",
+ "p",
+ "j",
+ "ö",
+ "f",
+ "c",
+ ],
+ "Korean": [
+ "이",
+ "다",
+ "에",
+ "의",
+ "는",
+ "로",
+ "하",
+ "을",
+ "가",
+ "고",
+ "지",
+ "서",
+ "한",
+ "은",
+ "기",
+ "으",
+ "년",
+ "대",
+ "사",
+ "시",
+ "를",
+ "리",
+ "도",
+ "인",
+ "스",
+ "일",
+ ],
+ "Indonesian": [
+ "a",
+ "n",
+ "e",
+ "i",
+ "r",
+ "t",
+ "u",
+ "s",
+ "d",
+ "k",
+ "m",
+ "l",
+ "g",
+ "p",
+ "b",
+ "o",
+ "h",
+ "y",
+ "j",
+ "c",
+ "w",
+ "f",
+ "v",
+ "z",
+ "x",
+ "q",
+ ],
+ "Turkish": [
+ "a",
+ "e",
+ "i",
+ "n",
+ "r",
+ "l",
+ "ı",
+ "k",
+ "d",
+ "t",
+ "s",
+ "m",
+ "y",
+ "u",
+ "o",
+ "b",
+ "ü",
+ "ş",
+ "v",
+ "g",
+ "z",
+ "h",
+ "c",
+ "p",
+ "ç",
+ "ğ",
+ ],
+ "Romanian": [
+ "e",
+ "i",
+ "a",
+ "r",
+ "n",
+ "t",
+ "u",
+ "l",
+ "o",
+ "c",
+ "s",
+ "d",
+ "p",
+ "m",
+ "ă",
+ "f",
+ "v",
+ "î",
+ "g",
+ "b",
+ "ș",
+ "ț",
+ "z",
+ "h",
+ "â",
+ "j",
+ ],
+ "Farsi": [
+ "ا",
+ "ی",
+ "ر",
+ "د",
+ "ن",
+ "ه",
+ "و",
+ "م",
+ "ت",
+ "ب",
+ "س",
+ "ل",
+ "ک",
+ "ش",
+ "ز",
+ "ف",
+ "گ",
+ "ع",
+ "خ",
+ "ق",
+ "ج",
+ "آ",
+ "پ",
+ "ح",
+ "ط",
+ "ص",
+ ],
+ "Arabic": [
+ "ا",
+ "ل",
+ "ي",
+ "م",
+ "و",
+ "ن",
+ "ر",
+ "ت",
+ "ب",
+ "ة",
+ "ع",
+ "د",
+ "س",
+ "ف",
+ "ه",
+ "ك",
+ "ق",
+ "أ",
+ "ح",
+ "ج",
+ "ش",
+ "ط",
+ "ص",
+ "ى",
+ "خ",
+ "إ",
+ ],
+ "Danish": [
+ "e",
+ "r",
+ "n",
+ "t",
+ "a",
+ "i",
+ "s",
+ "d",
+ "l",
+ "o",
+ "g",
+ "m",
+ "k",
+ "f",
+ "v",
+ "u",
+ "b",
+ "h",
+ "p",
+ "å",
+ "y",
+ "ø",
+ "æ",
+ "c",
+ "j",
+ "w",
+ ],
+ "Serbian": [
+ "а",
+ "и",
+ "о",
+ "е",
+ "н",
+ "р",
+ "с",
+ "у",
+ "т",
+ "к",
+ "ј",
+ "в",
+ "д",
+ "м",
+ "п",
+ "л",
+ "г",
+ "з",
+ "б",
+ "a",
+ "i",
+ "e",
+ "o",
+ "n",
+ "ц",
+ "ш",
+ ],
+ "Lithuanian": [
+ "i",
+ "a",
+ "s",
+ "o",
+ "r",
+ "e",
+ "t",
+ "n",
+ "u",
+ "k",
+ "m",
+ "l",
+ "p",
+ "v",
+ "d",
+ "j",
+ "g",
+ "ė",
+ "b",
+ "y",
+ "ų",
+ "š",
+ "ž",
+ "c",
+ "ą",
+ "į",
+ ],
+ "Slovene": [
+ "e",
+ "a",
+ "i",
+ "o",
+ "n",
+ "r",
+ "s",
+ "l",
+ "t",
+ "j",
+ "v",
+ "k",
+ "d",
+ "p",
+ "m",
+ "u",
+ "z",
+ "b",
+ "g",
+ "h",
+ "č",
+ "c",
+ "š",
+ "ž",
+ "f",
+ "y",
+ ],
+ "Slovak": [
+ "o",
+ "a",
+ "e",
+ "n",
+ "i",
+ "r",
+ "v",
+ "t",
+ "s",
+ "l",
+ "k",
+ "d",
+ "m",
+ "p",
+ "u",
+ "c",
+ "h",
+ "j",
+ "b",
+ "z",
+ "á",
+ "y",
+ "ý",
+ "í",
+ "č",
+ "é",
+ ],
+ "Hebrew": [
+ "י",
+ "ו",
+ "ה",
+ "ל",
+ "ר",
+ "ב",
+ "ת",
+ "מ",
+ "א",
+ "ש",
+ "נ",
+ "ע",
+ "ם",
+ "ד",
+ "ק",
+ "ח",
+ "פ",
+ "ס",
+ "כ",
+ "ג",
+ "ט",
+ "צ",
+ "ן",
+ "ז",
+ "ך",
+ ],
+ "Bulgarian": [
+ "а",
+ "и",
+ "о",
+ "е",
+ "н",
+ "т",
+ "р",
+ "с",
+ "в",
+ "л",
+ "к",
+ "д",
+ "п",
+ "м",
+ "з",
+ "г",
+ "я",
+ "ъ",
+ "у",
+ "б",
+ "ч",
+ "ц",
+ "й",
+ "ж",
+ "щ",
+ "х",
+ ],
+ "Croatian": [
+ "a",
+ "i",
+ "o",
+ "e",
+ "n",
+ "r",
+ "j",
+ "s",
+ "t",
+ "u",
+ "k",
+ "l",
+ "v",
+ "d",
+ "m",
+ "p",
+ "g",
+ "z",
+ "b",
+ "c",
+ "č",
+ "h",
+ "š",
+ "ž",
+ "ć",
+ "f",
+ ],
+ "Hindi": [
+ "क",
+ "र",
+ "स",
+ "न",
+ "त",
+ "म",
+ "ह",
+ "प",
+ "य",
+ "ल",
+ "व",
+ "ज",
+ "द",
+ "ग",
+ "ब",
+ "श",
+ "ट",
+ "अ",
+ "ए",
+ "थ",
+ "भ",
+ "ड",
+ "च",
+ "ध",
+ "ष",
+ "इ",
+ ],
+ "Estonian": [
+ "a",
+ "i",
+ "e",
+ "s",
+ "t",
+ "l",
+ "u",
+ "n",
+ "o",
+ "k",
+ "r",
+ "d",
+ "m",
+ "v",
+ "g",
+ "p",
+ "j",
+ "h",
+ "ä",
+ "b",
+ "õ",
+ "ü",
+ "f",
+ "c",
+ "ö",
+ "y",
+ ],
+ "Thai": [
+ "า",
+ "น",
+ "ร",
+ "อ",
+ "ก",
+ "เ",
+ "ง",
+ "ม",
+ "ย",
+ "ล",
+ "ว",
+ "ด",
+ "ท",
+ "ส",
+ "ต",
+ "ะ",
+ "ป",
+ "บ",
+ "ค",
+ "ห",
+ "แ",
+ "จ",
+ "พ",
+ "ช",
+ "ข",
+ "ใ",
+ ],
+ "Greek": [
+ "α",
+ "τ",
+ "ο",
+ "ι",
+ "ε",
+ "ν",
+ "ρ",
+ "σ",
+ "κ",
+ "η",
+ "π",
+ "ς",
+ "υ",
+ "μ",
+ "λ",
+ "ί",
+ "ό",
+ "ά",
+ "γ",
+ "έ",
+ "δ",
+ "ή",
+ "ω",
+ "χ",
+ "θ",
+ "ύ",
+ ],
+ "Tamil": [
+ "க",
+ "த",
+ "ப",
+ "ட",
+ "ர",
+ "ம",
+ "ல",
+ "ன",
+ "வ",
+ "ற",
+ "ய",
+ "ள",
+ "ச",
+ "ந",
+ "இ",
+ "ண",
+ "அ",
+ "ஆ",
+ "ழ",
+ "ங",
+ "எ",
+ "உ",
+ "ஒ",
+ "ஸ",
+ ],
+ "Kazakh": [
+ "а",
+ "ы",
+ "е",
+ "н",
+ "т",
+ "р",
+ "л",
+ "і",
+ "д",
+ "с",
+ "м",
+ "қ",
+ "к",
+ "о",
+ "б",
+ "и",
+ "у",
+ "ғ",
+ "ж",
+ "ң",
+ "з",
+ "ш",
+ "й",
+ "п",
+ "г",
+ "ө",
+ ],
+}
+
+LANGUAGE_SUPPORTED_COUNT: int = len(FREQUENCIES)
diff --git a/venv/lib/python3.10/site-packages/charset_normalizer/legacy.py b/venv/lib/python3.10/site-packages/charset_normalizer/legacy.py
new file mode 100644
index 0000000000000000000000000000000000000000..360a3107bdce817594db33de158e71d9fe793a3b
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/charset_normalizer/legacy.py
@@ -0,0 +1,80 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+from warnings import warn
+
+from .api import from_bytes
+from .constant import CHARDET_CORRESPONDENCE, TOO_SMALL_SEQUENCE
+
+# TODO: remove this check when dropping Python 3.7 support
+if TYPE_CHECKING:
+ from typing_extensions import TypedDict
+
+ class ResultDict(TypedDict):
+ encoding: str | None
+ language: str
+ confidence: float | None
+
+
+def detect(
+ byte_str: bytes, should_rename_legacy: bool = False, **kwargs: Any
+) -> ResultDict:
+ """
+ chardet legacy method
+ Detect the encoding of the given byte string. It should be mostly backward-compatible.
+ Encoding name will match Chardet own writing whenever possible. (Not on encoding name unsupported by it)
+ This function is deprecated and should be used to migrate your project easily, consult the documentation for
+ further information. Not planned for removal.
+
+ :param byte_str: The byte sequence to examine.
+ :param should_rename_legacy: Should we rename legacy encodings
+ to their more modern equivalents?
+ """
+ if len(kwargs):
+ warn(
+ f"charset-normalizer disregard arguments '{','.join(list(kwargs.keys()))}' in legacy function detect()"
+ )
+
+ if not isinstance(byte_str, (bytearray, bytes)):
+ raise TypeError( # pragma: nocover
+ f"Expected object of type bytes or bytearray, got: {type(byte_str)}"
+ )
+
+ if isinstance(byte_str, bytearray):
+ byte_str = bytes(byte_str)
+
+ r = from_bytes(byte_str).best()
+
+ encoding = r.encoding if r is not None else None
+ language = r.language if r is not None and r.language != "Unknown" else ""
+ confidence = 1.0 - r.chaos if r is not None else None
+
+ # automatically lower confidence
+ # on small bytes samples.
+ # https://github.com/jawah/charset_normalizer/issues/391
+ if (
+ confidence is not None
+ and confidence >= 0.9
+ and encoding
+ not in {
+ "utf_8",
+ "ascii",
+ }
+ and r.bom is False # type: ignore[union-attr]
+ and len(byte_str) < TOO_SMALL_SEQUENCE
+ ):
+ confidence -= 0.2
+
+ # Note: CharsetNormalizer does not return 'UTF-8-SIG' as the sig get stripped in the detection/normalization process
+ # but chardet does return 'utf-8-sig' and it is a valid codec name.
+ if r is not None and encoding == "utf_8" and r.bom:
+ encoding += "_sig"
+
+ if should_rename_legacy is False and encoding in CHARDET_CORRESPONDENCE:
+ encoding = CHARDET_CORRESPONDENCE[encoding]
+
+ return {
+ "encoding": encoding,
+ "language": language,
+ "confidence": confidence,
+ }
diff --git a/venv/lib/python3.10/site-packages/charset_normalizer/md.cpython-310-x86_64-linux-gnu.so b/venv/lib/python3.10/site-packages/charset_normalizer/md.cpython-310-x86_64-linux-gnu.so
new file mode 100644
index 0000000000000000000000000000000000000000..f71a5afcc365d22b97b203a2781f515882b92ed1
Binary files /dev/null and b/venv/lib/python3.10/site-packages/charset_normalizer/md.cpython-310-x86_64-linux-gnu.so differ
diff --git a/venv/lib/python3.10/site-packages/charset_normalizer/md.py b/venv/lib/python3.10/site-packages/charset_normalizer/md.py
new file mode 100644
index 0000000000000000000000000000000000000000..12ce024bd34c7eee369ddf438f5534e855a59839
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/charset_normalizer/md.py
@@ -0,0 +1,635 @@
+from __future__ import annotations
+
+from functools import lru_cache
+from logging import getLogger
+
+from .constant import (
+ COMMON_SAFE_ASCII_CHARACTERS,
+ TRACE,
+ UNICODE_SECONDARY_RANGE_KEYWORD,
+)
+from .utils import (
+ is_accentuated,
+ is_arabic,
+ is_arabic_isolated_form,
+ is_case_variable,
+ is_cjk,
+ is_emoticon,
+ is_hangul,
+ is_hiragana,
+ is_katakana,
+ is_latin,
+ is_punctuation,
+ is_separator,
+ is_symbol,
+ is_thai,
+ is_unprintable,
+ remove_accent,
+ unicode_range,
+ is_cjk_uncommon,
+)
+
+
+class MessDetectorPlugin:
+ """
+ Base abstract class used for mess detection plugins.
+ All detectors MUST extend and implement given methods.
+ """
+
+ def eligible(self, character: str) -> bool:
+ """
+ Determine if given character should be fed in.
+ """
+ raise NotImplementedError # pragma: nocover
+
+ def feed(self, character: str) -> None:
+ """
+ The main routine to be executed upon character.
+ Insert the logic in witch the text would be considered chaotic.
+ """
+ raise NotImplementedError # pragma: nocover
+
+ def reset(self) -> None: # pragma: no cover
+ """
+ Permit to reset the plugin to the initial state.
+ """
+ raise NotImplementedError
+
+ @property
+ def ratio(self) -> float:
+ """
+ Compute the chaos ratio based on what your feed() has seen.
+ Must NOT be lower than 0.; No restriction gt 0.
+ """
+ raise NotImplementedError # pragma: nocover
+
+
+class TooManySymbolOrPunctuationPlugin(MessDetectorPlugin):
+ def __init__(self) -> None:
+ self._punctuation_count: int = 0
+ self._symbol_count: int = 0
+ self._character_count: int = 0
+
+ self._last_printable_char: str | None = None
+ self._frenzy_symbol_in_word: bool = False
+
+ def eligible(self, character: str) -> bool:
+ return character.isprintable()
+
+ def feed(self, character: str) -> None:
+ self._character_count += 1
+
+ if (
+ character != self._last_printable_char
+ and character not in COMMON_SAFE_ASCII_CHARACTERS
+ ):
+ if is_punctuation(character):
+ self._punctuation_count += 1
+ elif (
+ character.isdigit() is False
+ and is_symbol(character)
+ and is_emoticon(character) is False
+ ):
+ self._symbol_count += 2
+
+ self._last_printable_char = character
+
+ def reset(self) -> None: # Abstract
+ self._punctuation_count = 0
+ self._character_count = 0
+ self._symbol_count = 0
+
+ @property
+ def ratio(self) -> float:
+ if self._character_count == 0:
+ return 0.0
+
+ ratio_of_punctuation: float = (
+ self._punctuation_count + self._symbol_count
+ ) / self._character_count
+
+ return ratio_of_punctuation if ratio_of_punctuation >= 0.3 else 0.0
+
+
+class TooManyAccentuatedPlugin(MessDetectorPlugin):
+ def __init__(self) -> None:
+ self._character_count: int = 0
+ self._accentuated_count: int = 0
+
+ def eligible(self, character: str) -> bool:
+ return character.isalpha()
+
+ def feed(self, character: str) -> None:
+ self._character_count += 1
+
+ if is_accentuated(character):
+ self._accentuated_count += 1
+
+ def reset(self) -> None: # Abstract
+ self._character_count = 0
+ self._accentuated_count = 0
+
+ @property
+ def ratio(self) -> float:
+ if self._character_count < 8:
+ return 0.0
+
+ ratio_of_accentuation: float = self._accentuated_count / self._character_count
+ return ratio_of_accentuation if ratio_of_accentuation >= 0.35 else 0.0
+
+
+class UnprintablePlugin(MessDetectorPlugin):
+ def __init__(self) -> None:
+ self._unprintable_count: int = 0
+ self._character_count: int = 0
+
+ def eligible(self, character: str) -> bool:
+ return True
+
+ def feed(self, character: str) -> None:
+ if is_unprintable(character):
+ self._unprintable_count += 1
+ self._character_count += 1
+
+ def reset(self) -> None: # Abstract
+ self._unprintable_count = 0
+
+ @property
+ def ratio(self) -> float:
+ if self._character_count == 0:
+ return 0.0
+
+ return (self._unprintable_count * 8) / self._character_count
+
+
+class SuspiciousDuplicateAccentPlugin(MessDetectorPlugin):
+ def __init__(self) -> None:
+ self._successive_count: int = 0
+ self._character_count: int = 0
+
+ self._last_latin_character: str | None = None
+
+ def eligible(self, character: str) -> bool:
+ return character.isalpha() and is_latin(character)
+
+ def feed(self, character: str) -> None:
+ self._character_count += 1
+ if (
+ self._last_latin_character is not None
+ and is_accentuated(character)
+ and is_accentuated(self._last_latin_character)
+ ):
+ if character.isupper() and self._last_latin_character.isupper():
+ self._successive_count += 1
+ # Worse if its the same char duplicated with different accent.
+ if remove_accent(character) == remove_accent(self._last_latin_character):
+ self._successive_count += 1
+ self._last_latin_character = character
+
+ def reset(self) -> None: # Abstract
+ self._successive_count = 0
+ self._character_count = 0
+ self._last_latin_character = None
+
+ @property
+ def ratio(self) -> float:
+ if self._character_count == 0:
+ return 0.0
+
+ return (self._successive_count * 2) / self._character_count
+
+
+class SuspiciousRange(MessDetectorPlugin):
+ def __init__(self) -> None:
+ self._suspicious_successive_range_count: int = 0
+ self._character_count: int = 0
+ self._last_printable_seen: str | None = None
+
+ def eligible(self, character: str) -> bool:
+ return character.isprintable()
+
+ def feed(self, character: str) -> None:
+ self._character_count += 1
+
+ if (
+ character.isspace()
+ or is_punctuation(character)
+ or character in COMMON_SAFE_ASCII_CHARACTERS
+ ):
+ self._last_printable_seen = None
+ return
+
+ if self._last_printable_seen is None:
+ self._last_printable_seen = character
+ return
+
+ unicode_range_a: str | None = unicode_range(self._last_printable_seen)
+ unicode_range_b: str | None = unicode_range(character)
+
+ if is_suspiciously_successive_range(unicode_range_a, unicode_range_b):
+ self._suspicious_successive_range_count += 1
+
+ self._last_printable_seen = character
+
+ def reset(self) -> None: # Abstract
+ self._character_count = 0
+ self._suspicious_successive_range_count = 0
+ self._last_printable_seen = None
+
+ @property
+ def ratio(self) -> float:
+ if self._character_count <= 13:
+ return 0.0
+
+ ratio_of_suspicious_range_usage: float = (
+ self._suspicious_successive_range_count * 2
+ ) / self._character_count
+
+ return ratio_of_suspicious_range_usage
+
+
+class SuperWeirdWordPlugin(MessDetectorPlugin):
+ def __init__(self) -> None:
+ self._word_count: int = 0
+ self._bad_word_count: int = 0
+ self._foreign_long_count: int = 0
+
+ self._is_current_word_bad: bool = False
+ self._foreign_long_watch: bool = False
+
+ self._character_count: int = 0
+ self._bad_character_count: int = 0
+
+ self._buffer: str = ""
+ self._buffer_accent_count: int = 0
+ self._buffer_glyph_count: int = 0
+
+ def eligible(self, character: str) -> bool:
+ return True
+
+ def feed(self, character: str) -> None:
+ if character.isalpha():
+ self._buffer += character
+ if is_accentuated(character):
+ self._buffer_accent_count += 1
+ if (
+ self._foreign_long_watch is False
+ and (is_latin(character) is False or is_accentuated(character))
+ and is_cjk(character) is False
+ and is_hangul(character) is False
+ and is_katakana(character) is False
+ and is_hiragana(character) is False
+ and is_thai(character) is False
+ ):
+ self._foreign_long_watch = True
+ if (
+ is_cjk(character)
+ or is_hangul(character)
+ or is_katakana(character)
+ or is_hiragana(character)
+ or is_thai(character)
+ ):
+ self._buffer_glyph_count += 1
+ return
+ if not self._buffer:
+ return
+ if (
+ character.isspace() or is_punctuation(character) or is_separator(character)
+ ) and self._buffer:
+ self._word_count += 1
+ buffer_length: int = len(self._buffer)
+
+ self._character_count += buffer_length
+
+ if buffer_length >= 4:
+ if self._buffer_accent_count / buffer_length >= 0.5:
+ self._is_current_word_bad = True
+ # Word/Buffer ending with an upper case accentuated letter are so rare,
+ # that we will consider them all as suspicious. Same weight as foreign_long suspicious.
+ elif (
+ is_accentuated(self._buffer[-1])
+ and self._buffer[-1].isupper()
+ and all(_.isupper() for _ in self._buffer) is False
+ ):
+ self._foreign_long_count += 1
+ self._is_current_word_bad = True
+ elif self._buffer_glyph_count == 1:
+ self._is_current_word_bad = True
+ self._foreign_long_count += 1
+ if buffer_length >= 24 and self._foreign_long_watch:
+ camel_case_dst = [
+ i
+ for c, i in zip(self._buffer, range(0, buffer_length))
+ if c.isupper()
+ ]
+ probable_camel_cased: bool = False
+
+ if camel_case_dst and (len(camel_case_dst) / buffer_length <= 0.3):
+ probable_camel_cased = True
+
+ if not probable_camel_cased:
+ self._foreign_long_count += 1
+ self._is_current_word_bad = True
+
+ if self._is_current_word_bad:
+ self._bad_word_count += 1
+ self._bad_character_count += len(self._buffer)
+ self._is_current_word_bad = False
+
+ self._foreign_long_watch = False
+ self._buffer = ""
+ self._buffer_accent_count = 0
+ self._buffer_glyph_count = 0
+ elif (
+ character not in {"<", ">", "-", "=", "~", "|", "_"}
+ and character.isdigit() is False
+ and is_symbol(character)
+ ):
+ self._is_current_word_bad = True
+ self._buffer += character
+
+ def reset(self) -> None: # Abstract
+ self._buffer = ""
+ self._is_current_word_bad = False
+ self._foreign_long_watch = False
+ self._bad_word_count = 0
+ self._word_count = 0
+ self._character_count = 0
+ self._bad_character_count = 0
+ self._foreign_long_count = 0
+
+ @property
+ def ratio(self) -> float:
+ if self._word_count <= 10 and self._foreign_long_count == 0:
+ return 0.0
+
+ return self._bad_character_count / self._character_count
+
+
+class CjkUncommonPlugin(MessDetectorPlugin):
+ """
+ Detect messy CJK text that probably means nothing.
+ """
+
+ def __init__(self) -> None:
+ self._character_count: int = 0
+ self._uncommon_count: int = 0
+
+ def eligible(self, character: str) -> bool:
+ return is_cjk(character)
+
+ def feed(self, character: str) -> None:
+ self._character_count += 1
+
+ if is_cjk_uncommon(character):
+ self._uncommon_count += 1
+ return
+
+ def reset(self) -> None: # Abstract
+ self._character_count = 0
+ self._uncommon_count = 0
+
+ @property
+ def ratio(self) -> float:
+ if self._character_count < 8:
+ return 0.0
+
+ uncommon_form_usage: float = self._uncommon_count / self._character_count
+
+ # we can be pretty sure it's garbage when uncommon characters are widely
+ # used. otherwise it could just be traditional chinese for example.
+ return uncommon_form_usage / 10 if uncommon_form_usage > 0.5 else 0.0
+
+
+class ArchaicUpperLowerPlugin(MessDetectorPlugin):
+ def __init__(self) -> None:
+ self._buf: bool = False
+
+ self._character_count_since_last_sep: int = 0
+
+ self._successive_upper_lower_count: int = 0
+ self._successive_upper_lower_count_final: int = 0
+
+ self._character_count: int = 0
+
+ self._last_alpha_seen: str | None = None
+ self._current_ascii_only: bool = True
+
+ def eligible(self, character: str) -> bool:
+ return True
+
+ def feed(self, character: str) -> None:
+ is_concerned = character.isalpha() and is_case_variable(character)
+ chunk_sep = is_concerned is False
+
+ if chunk_sep and self._character_count_since_last_sep > 0:
+ if (
+ self._character_count_since_last_sep <= 64
+ and character.isdigit() is False
+ and self._current_ascii_only is False
+ ):
+ self._successive_upper_lower_count_final += (
+ self._successive_upper_lower_count
+ )
+
+ self._successive_upper_lower_count = 0
+ self._character_count_since_last_sep = 0
+ self._last_alpha_seen = None
+ self._buf = False
+ self._character_count += 1
+ self._current_ascii_only = True
+
+ return
+
+ if self._current_ascii_only is True and character.isascii() is False:
+ self._current_ascii_only = False
+
+ if self._last_alpha_seen is not None:
+ if (character.isupper() and self._last_alpha_seen.islower()) or (
+ character.islower() and self._last_alpha_seen.isupper()
+ ):
+ if self._buf is True:
+ self._successive_upper_lower_count += 2
+ self._buf = False
+ else:
+ self._buf = True
+ else:
+ self._buf = False
+
+ self._character_count += 1
+ self._character_count_since_last_sep += 1
+ self._last_alpha_seen = character
+
+ def reset(self) -> None: # Abstract
+ self._character_count = 0
+ self._character_count_since_last_sep = 0
+ self._successive_upper_lower_count = 0
+ self._successive_upper_lower_count_final = 0
+ self._last_alpha_seen = None
+ self._buf = False
+ self._current_ascii_only = True
+
+ @property
+ def ratio(self) -> float:
+ if self._character_count == 0:
+ return 0.0
+
+ return self._successive_upper_lower_count_final / self._character_count
+
+
+class ArabicIsolatedFormPlugin(MessDetectorPlugin):
+ def __init__(self) -> None:
+ self._character_count: int = 0
+ self._isolated_form_count: int = 0
+
+ def reset(self) -> None: # Abstract
+ self._character_count = 0
+ self._isolated_form_count = 0
+
+ def eligible(self, character: str) -> bool:
+ return is_arabic(character)
+
+ def feed(self, character: str) -> None:
+ self._character_count += 1
+
+ if is_arabic_isolated_form(character):
+ self._isolated_form_count += 1
+
+ @property
+ def ratio(self) -> float:
+ if self._character_count < 8:
+ return 0.0
+
+ isolated_form_usage: float = self._isolated_form_count / self._character_count
+
+ return isolated_form_usage
+
+
+@lru_cache(maxsize=1024)
+def is_suspiciously_successive_range(
+ unicode_range_a: str | None, unicode_range_b: str | None
+) -> bool:
+ """
+ Determine if two Unicode range seen next to each other can be considered as suspicious.
+ """
+ if unicode_range_a is None or unicode_range_b is None:
+ return True
+
+ if unicode_range_a == unicode_range_b:
+ return False
+
+ if "Latin" in unicode_range_a and "Latin" in unicode_range_b:
+ return False
+
+ if "Emoticons" in unicode_range_a or "Emoticons" in unicode_range_b:
+ return False
+
+ # Latin characters can be accompanied with a combining diacritical mark
+ # eg. Vietnamese.
+ if ("Latin" in unicode_range_a or "Latin" in unicode_range_b) and (
+ "Combining" in unicode_range_a or "Combining" in unicode_range_b
+ ):
+ return False
+
+ keywords_range_a, keywords_range_b = (
+ unicode_range_a.split(" "),
+ unicode_range_b.split(" "),
+ )
+
+ for el in keywords_range_a:
+ if el in UNICODE_SECONDARY_RANGE_KEYWORD:
+ continue
+ if el in keywords_range_b:
+ return False
+
+ # Japanese Exception
+ range_a_jp_chars, range_b_jp_chars = (
+ unicode_range_a
+ in (
+ "Hiragana",
+ "Katakana",
+ ),
+ unicode_range_b in ("Hiragana", "Katakana"),
+ )
+ if (range_a_jp_chars or range_b_jp_chars) and (
+ "CJK" in unicode_range_a or "CJK" in unicode_range_b
+ ):
+ return False
+ if range_a_jp_chars and range_b_jp_chars:
+ return False
+
+ if "Hangul" in unicode_range_a or "Hangul" in unicode_range_b:
+ if "CJK" in unicode_range_a or "CJK" in unicode_range_b:
+ return False
+ if unicode_range_a == "Basic Latin" or unicode_range_b == "Basic Latin":
+ return False
+
+ # Chinese/Japanese use dedicated range for punctuation and/or separators.
+ if ("CJK" in unicode_range_a or "CJK" in unicode_range_b) or (
+ unicode_range_a in ["Katakana", "Hiragana"]
+ and unicode_range_b in ["Katakana", "Hiragana"]
+ ):
+ if "Punctuation" in unicode_range_a or "Punctuation" in unicode_range_b:
+ return False
+ if "Forms" in unicode_range_a or "Forms" in unicode_range_b:
+ return False
+ if unicode_range_a == "Basic Latin" or unicode_range_b == "Basic Latin":
+ return False
+
+ return True
+
+
+@lru_cache(maxsize=2048)
+def mess_ratio(
+ decoded_sequence: str, maximum_threshold: float = 0.2, debug: bool = False
+) -> float:
+ """
+ Compute a mess ratio given a decoded bytes sequence. The maximum threshold does stop the computation earlier.
+ """
+
+ detectors: list[MessDetectorPlugin] = [
+ md_class() for md_class in MessDetectorPlugin.__subclasses__()
+ ]
+
+ length: int = len(decoded_sequence) + 1
+
+ mean_mess_ratio: float = 0.0
+
+ if length < 512:
+ intermediary_mean_mess_ratio_calc: int = 32
+ elif length <= 1024:
+ intermediary_mean_mess_ratio_calc = 64
+ else:
+ intermediary_mean_mess_ratio_calc = 128
+
+ for character, index in zip(decoded_sequence + "\n", range(length)):
+ for detector in detectors:
+ if detector.eligible(character):
+ detector.feed(character)
+
+ if (
+ index > 0 and index % intermediary_mean_mess_ratio_calc == 0
+ ) or index == length - 1:
+ mean_mess_ratio = sum(dt.ratio for dt in detectors)
+
+ if mean_mess_ratio >= maximum_threshold:
+ break
+
+ if debug:
+ logger = getLogger("charset_normalizer")
+
+ logger.log(
+ TRACE,
+ "Mess-detector extended-analysis start. "
+ f"intermediary_mean_mess_ratio_calc={intermediary_mean_mess_ratio_calc} mean_mess_ratio={mean_mess_ratio} "
+ f"maximum_threshold={maximum_threshold}",
+ )
+
+ if len(decoded_sequence) > 16:
+ logger.log(TRACE, f"Starting with: {decoded_sequence[:16]}")
+ logger.log(TRACE, f"Ending with: {decoded_sequence[-16::]}")
+
+ for dt in detectors:
+ logger.log(TRACE, f"{dt.__class__}: {dt.ratio}")
+
+ return round(mean_mess_ratio, 3)
diff --git a/venv/lib/python3.10/site-packages/charset_normalizer/models.py b/venv/lib/python3.10/site-packages/charset_normalizer/models.py
new file mode 100644
index 0000000000000000000000000000000000000000..1042758f873d2a54f7078c5411b17ffc11dca4ee
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/charset_normalizer/models.py
@@ -0,0 +1,360 @@
+from __future__ import annotations
+
+from encodings.aliases import aliases
+from hashlib import sha256
+from json import dumps
+from re import sub
+from typing import Any, Iterator, List, Tuple
+
+from .constant import RE_POSSIBLE_ENCODING_INDICATION, TOO_BIG_SEQUENCE
+from .utils import iana_name, is_multi_byte_encoding, unicode_range
+
+
+class CharsetMatch:
+ def __init__(
+ self,
+ payload: bytes,
+ guessed_encoding: str,
+ mean_mess_ratio: float,
+ has_sig_or_bom: bool,
+ languages: CoherenceMatches,
+ decoded_payload: str | None = None,
+ preemptive_declaration: str | None = None,
+ ):
+ self._payload: bytes = payload
+
+ self._encoding: str = guessed_encoding
+ self._mean_mess_ratio: float = mean_mess_ratio
+ self._languages: CoherenceMatches = languages
+ self._has_sig_or_bom: bool = has_sig_or_bom
+ self._unicode_ranges: list[str] | None = None
+
+ self._leaves: list[CharsetMatch] = []
+ self._mean_coherence_ratio: float = 0.0
+
+ self._output_payload: bytes | None = None
+ self._output_encoding: str | None = None
+
+ self._string: str | None = decoded_payload
+
+ self._preemptive_declaration: str | None = preemptive_declaration
+
+ def __eq__(self, other: object) -> bool:
+ if not isinstance(other, CharsetMatch):
+ if isinstance(other, str):
+ return iana_name(other) == self.encoding
+ return False
+ return self.encoding == other.encoding and self.fingerprint == other.fingerprint
+
+ def __lt__(self, other: object) -> bool:
+ """
+ Implemented to make sorted available upon CharsetMatches items.
+ """
+ if not isinstance(other, CharsetMatch):
+ raise ValueError
+
+ chaos_difference: float = abs(self.chaos - other.chaos)
+ coherence_difference: float = abs(self.coherence - other.coherence)
+
+ # Below 1% difference --> Use Coherence
+ if chaos_difference < 0.01 and coherence_difference > 0.02:
+ return self.coherence > other.coherence
+ elif chaos_difference < 0.01 and coherence_difference <= 0.02:
+ # When having a difficult decision, use the result that decoded as many multi-byte as possible.
+ # preserve RAM usage!
+ if len(self._payload) >= TOO_BIG_SEQUENCE:
+ return self.chaos < other.chaos
+ return self.multi_byte_usage > other.multi_byte_usage
+
+ return self.chaos < other.chaos
+
+ @property
+ def multi_byte_usage(self) -> float:
+ return 1.0 - (len(str(self)) / len(self.raw))
+
+ def __str__(self) -> str:
+ # Lazy Str Loading
+ if self._string is None:
+ self._string = str(self._payload, self._encoding, "strict")
+ return self._string
+
+ def __repr__(self) -> str:
+ return f""
+
+ def add_submatch(self, other: CharsetMatch) -> None:
+ if not isinstance(other, CharsetMatch) or other == self:
+ raise ValueError(
+ "Unable to add instance <{}> as a submatch of a CharsetMatch".format(
+ other.__class__
+ )
+ )
+
+ other._string = None # Unload RAM usage; dirty trick.
+ self._leaves.append(other)
+
+ @property
+ def encoding(self) -> str:
+ return self._encoding
+
+ @property
+ def encoding_aliases(self) -> list[str]:
+ """
+ Encoding name are known by many name, using this could help when searching for IBM855 when it's listed as CP855.
+ """
+ also_known_as: list[str] = []
+ for u, p in aliases.items():
+ if self.encoding == u:
+ also_known_as.append(p)
+ elif self.encoding == p:
+ also_known_as.append(u)
+ return also_known_as
+
+ @property
+ def bom(self) -> bool:
+ return self._has_sig_or_bom
+
+ @property
+ def byte_order_mark(self) -> bool:
+ return self._has_sig_or_bom
+
+ @property
+ def languages(self) -> list[str]:
+ """
+ Return the complete list of possible languages found in decoded sequence.
+ Usually not really useful. Returned list may be empty even if 'language' property return something != 'Unknown'.
+ """
+ return [e[0] for e in self._languages]
+
+ @property
+ def language(self) -> str:
+ """
+ Most probable language found in decoded sequence. If none were detected or inferred, the property will return
+ "Unknown".
+ """
+ if not self._languages:
+ # Trying to infer the language based on the given encoding
+ # Its either English or we should not pronounce ourselves in certain cases.
+ if "ascii" in self.could_be_from_charset:
+ return "English"
+
+ # doing it there to avoid circular import
+ from charset_normalizer.cd import encoding_languages, mb_encoding_languages
+
+ languages = (
+ mb_encoding_languages(self.encoding)
+ if is_multi_byte_encoding(self.encoding)
+ else encoding_languages(self.encoding)
+ )
+
+ if len(languages) == 0 or "Latin Based" in languages:
+ return "Unknown"
+
+ return languages[0]
+
+ return self._languages[0][0]
+
+ @property
+ def chaos(self) -> float:
+ return self._mean_mess_ratio
+
+ @property
+ def coherence(self) -> float:
+ if not self._languages:
+ return 0.0
+ return self._languages[0][1]
+
+ @property
+ def percent_chaos(self) -> float:
+ return round(self.chaos * 100, ndigits=3)
+
+ @property
+ def percent_coherence(self) -> float:
+ return round(self.coherence * 100, ndigits=3)
+
+ @property
+ def raw(self) -> bytes:
+ """
+ Original untouched bytes.
+ """
+ return self._payload
+
+ @property
+ def submatch(self) -> list[CharsetMatch]:
+ return self._leaves
+
+ @property
+ def has_submatch(self) -> bool:
+ return len(self._leaves) > 0
+
+ @property
+ def alphabets(self) -> list[str]:
+ if self._unicode_ranges is not None:
+ return self._unicode_ranges
+ # list detected ranges
+ detected_ranges: list[str | None] = [unicode_range(char) for char in str(self)]
+ # filter and sort
+ self._unicode_ranges = sorted(list({r for r in detected_ranges if r}))
+ return self._unicode_ranges
+
+ @property
+ def could_be_from_charset(self) -> list[str]:
+ """
+ The complete list of encoding that output the exact SAME str result and therefore could be the originating
+ encoding.
+ This list does include the encoding available in property 'encoding'.
+ """
+ return [self._encoding] + [m.encoding for m in self._leaves]
+
+ def output(self, encoding: str = "utf_8") -> bytes:
+ """
+ Method to get re-encoded bytes payload using given target encoding. Default to UTF-8.
+ Any errors will be simply ignored by the encoder NOT replaced.
+ """
+ if self._output_encoding is None or self._output_encoding != encoding:
+ self._output_encoding = encoding
+ decoded_string = str(self)
+ if (
+ self._preemptive_declaration is not None
+ and self._preemptive_declaration.lower()
+ not in ["utf-8", "utf8", "utf_8"]
+ ):
+ patched_header = sub(
+ RE_POSSIBLE_ENCODING_INDICATION,
+ lambda m: m.string[m.span()[0] : m.span()[1]].replace(
+ m.groups()[0],
+ iana_name(self._output_encoding).replace("_", "-"), # type: ignore[arg-type]
+ ),
+ decoded_string[:8192],
+ count=1,
+ )
+
+ decoded_string = patched_header + decoded_string[8192:]
+
+ self._output_payload = decoded_string.encode(encoding, "replace")
+
+ return self._output_payload # type: ignore
+
+ @property
+ def fingerprint(self) -> str:
+ """
+ Retrieve the unique SHA256 computed using the transformed (re-encoded) payload. Not the original one.
+ """
+ return sha256(self.output()).hexdigest()
+
+
+class CharsetMatches:
+ """
+ Container with every CharsetMatch items ordered by default from most probable to the less one.
+ Act like a list(iterable) but does not implements all related methods.
+ """
+
+ def __init__(self, results: list[CharsetMatch] | None = None):
+ self._results: list[CharsetMatch] = sorted(results) if results else []
+
+ def __iter__(self) -> Iterator[CharsetMatch]:
+ yield from self._results
+
+ def __getitem__(self, item: int | str) -> CharsetMatch:
+ """
+ Retrieve a single item either by its position or encoding name (alias may be used here).
+ Raise KeyError upon invalid index or encoding not present in results.
+ """
+ if isinstance(item, int):
+ return self._results[item]
+ if isinstance(item, str):
+ item = iana_name(item, False)
+ for result in self._results:
+ if item in result.could_be_from_charset:
+ return result
+ raise KeyError
+
+ def __len__(self) -> int:
+ return len(self._results)
+
+ def __bool__(self) -> bool:
+ return len(self._results) > 0
+
+ def append(self, item: CharsetMatch) -> None:
+ """
+ Insert a single match. Will be inserted accordingly to preserve sort.
+ Can be inserted as a submatch.
+ """
+ if not isinstance(item, CharsetMatch):
+ raise ValueError(
+ "Cannot append instance '{}' to CharsetMatches".format(
+ str(item.__class__)
+ )
+ )
+ # We should disable the submatch factoring when the input file is too heavy (conserve RAM usage)
+ if len(item.raw) < TOO_BIG_SEQUENCE:
+ for match in self._results:
+ if match.fingerprint == item.fingerprint and match.chaos == item.chaos:
+ match.add_submatch(item)
+ return
+ self._results.append(item)
+ self._results = sorted(self._results)
+
+ def best(self) -> CharsetMatch | None:
+ """
+ Simply return the first match. Strict equivalent to matches[0].
+ """
+ if not self._results:
+ return None
+ return self._results[0]
+
+ def first(self) -> CharsetMatch | None:
+ """
+ Redundant method, call the method best(). Kept for BC reasons.
+ """
+ return self.best()
+
+
+CoherenceMatch = Tuple[str, float]
+CoherenceMatches = List[CoherenceMatch]
+
+
+class CliDetectionResult:
+ def __init__(
+ self,
+ path: str,
+ encoding: str | None,
+ encoding_aliases: list[str],
+ alternative_encodings: list[str],
+ language: str,
+ alphabets: list[str],
+ has_sig_or_bom: bool,
+ chaos: float,
+ coherence: float,
+ unicode_path: str | None,
+ is_preferred: bool,
+ ):
+ self.path: str = path
+ self.unicode_path: str | None = unicode_path
+ self.encoding: str | None = encoding
+ self.encoding_aliases: list[str] = encoding_aliases
+ self.alternative_encodings: list[str] = alternative_encodings
+ self.language: str = language
+ self.alphabets: list[str] = alphabets
+ self.has_sig_or_bom: bool = has_sig_or_bom
+ self.chaos: float = chaos
+ self.coherence: float = coherence
+ self.is_preferred: bool = is_preferred
+
+ @property
+ def __dict__(self) -> dict[str, Any]: # type: ignore
+ return {
+ "path": self.path,
+ "encoding": self.encoding,
+ "encoding_aliases": self.encoding_aliases,
+ "alternative_encodings": self.alternative_encodings,
+ "language": self.language,
+ "alphabets": self.alphabets,
+ "has_sig_or_bom": self.has_sig_or_bom,
+ "chaos": self.chaos,
+ "coherence": self.coherence,
+ "unicode_path": self.unicode_path,
+ "is_preferred": self.is_preferred,
+ }
+
+ def to_json(self) -> str:
+ return dumps(self.__dict__, ensure_ascii=True, indent=4)
diff --git a/venv/lib/python3.10/site-packages/charset_normalizer/py.typed b/venv/lib/python3.10/site-packages/charset_normalizer/py.typed
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/venv/lib/python3.10/site-packages/charset_normalizer/utils.py b/venv/lib/python3.10/site-packages/charset_normalizer/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..6bf0384ccf94216f79881674f54182a93e17def0
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/charset_normalizer/utils.py
@@ -0,0 +1,414 @@
+from __future__ import annotations
+
+import importlib
+import logging
+import unicodedata
+from codecs import IncrementalDecoder
+from encodings.aliases import aliases
+from functools import lru_cache
+from re import findall
+from typing import Generator
+
+from _multibytecodec import ( # type: ignore[import-not-found,import]
+ MultibyteIncrementalDecoder,
+)
+
+from .constant import (
+ ENCODING_MARKS,
+ IANA_SUPPORTED_SIMILAR,
+ RE_POSSIBLE_ENCODING_INDICATION,
+ UNICODE_RANGES_COMBINED,
+ UNICODE_SECONDARY_RANGE_KEYWORD,
+ UTF8_MAXIMAL_ALLOCATION,
+ COMMON_CJK_CHARACTERS,
+)
+
+
+@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
+def is_accentuated(character: str) -> bool:
+ try:
+ description: str = unicodedata.name(character)
+ except ValueError: # Defensive: unicode database outdated?
+ return False
+ return (
+ "WITH GRAVE" in description
+ or "WITH ACUTE" in description
+ or "WITH CEDILLA" in description
+ or "WITH DIAERESIS" in description
+ or "WITH CIRCUMFLEX" in description
+ or "WITH TILDE" in description
+ or "WITH MACRON" in description
+ or "WITH RING ABOVE" in description
+ )
+
+
+@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
+def remove_accent(character: str) -> str:
+ decomposed: str = unicodedata.decomposition(character)
+ if not decomposed:
+ return character
+
+ codes: list[str] = decomposed.split(" ")
+
+ return chr(int(codes[0], 16))
+
+
+@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
+def unicode_range(character: str) -> str | None:
+ """
+ Retrieve the Unicode range official name from a single character.
+ """
+ character_ord: int = ord(character)
+
+ for range_name, ord_range in UNICODE_RANGES_COMBINED.items():
+ if character_ord in ord_range:
+ return range_name
+
+ return None
+
+
+@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
+def is_latin(character: str) -> bool:
+ try:
+ description: str = unicodedata.name(character)
+ except ValueError: # Defensive: unicode database outdated?
+ return False
+ return "LATIN" in description
+
+
+@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
+def is_punctuation(character: str) -> bool:
+ character_category: str = unicodedata.category(character)
+
+ if "P" in character_category:
+ return True
+
+ character_range: str | None = unicode_range(character)
+
+ if character_range is None:
+ return False
+
+ return "Punctuation" in character_range
+
+
+@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
+def is_symbol(character: str) -> bool:
+ character_category: str = unicodedata.category(character)
+
+ if "S" in character_category or "N" in character_category:
+ return True
+
+ character_range: str | None = unicode_range(character)
+
+ if character_range is None:
+ return False
+
+ return "Forms" in character_range and character_category != "Lo"
+
+
+@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
+def is_emoticon(character: str) -> bool:
+ character_range: str | None = unicode_range(character)
+
+ if character_range is None:
+ return False
+
+ return "Emoticons" in character_range or "Pictographs" in character_range
+
+
+@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
+def is_separator(character: str) -> bool:
+ if character.isspace() or character in {"｜", "+", "<", ">"}:
+ return True
+
+ character_category: str = unicodedata.category(character)
+
+ return "Z" in character_category or character_category in {"Po", "Pd", "Pc"}
+
+
+@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
+def is_case_variable(character: str) -> bool:
+ return character.islower() != character.isupper()
+
+
+@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
+def is_cjk(character: str) -> bool:
+ try:
+ character_name = unicodedata.name(character)
+ except ValueError: # Defensive: unicode database outdated?
+ return False
+
+ return "CJK" in character_name
+
+
+@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
+def is_hiragana(character: str) -> bool:
+ try:
+ character_name = unicodedata.name(character)
+ except ValueError: # Defensive: unicode database outdated?
+ return False
+
+ return "HIRAGANA" in character_name
+
+
+@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
+def is_katakana(character: str) -> bool:
+ try:
+ character_name = unicodedata.name(character)
+ except ValueError: # Defensive: unicode database outdated?
+ return False
+
+ return "KATAKANA" in character_name
+
+
+@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
+def is_hangul(character: str) -> bool:
+ try:
+ character_name = unicodedata.name(character)
+ except ValueError: # Defensive: unicode database outdated?
+ return False
+
+ return "HANGUL" in character_name
+
+
+@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
+def is_thai(character: str) -> bool:
+ try:
+ character_name = unicodedata.name(character)
+ except ValueError: # Defensive: unicode database outdated?
+ return False
+
+ return "THAI" in character_name
+
+
+@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
+def is_arabic(character: str) -> bool:
+ try:
+ character_name = unicodedata.name(character)
+ except ValueError: # Defensive: unicode database outdated?
+ return False
+
+ return "ARABIC" in character_name
+
+
+@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
+def is_arabic_isolated_form(character: str) -> bool:
+ try:
+ character_name = unicodedata.name(character)
+ except ValueError: # Defensive: unicode database outdated?
+ return False
+
+ return "ARABIC" in character_name and "ISOLATED FORM" in character_name
+
+
+@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
+def is_cjk_uncommon(character: str) -> bool:
+ return character not in COMMON_CJK_CHARACTERS
+
+
+@lru_cache(maxsize=len(UNICODE_RANGES_COMBINED))
+def is_unicode_range_secondary(range_name: str) -> bool:
+ return any(keyword in range_name for keyword in UNICODE_SECONDARY_RANGE_KEYWORD)
+
+
+@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
+def is_unprintable(character: str) -> bool:
+ return (
+ character.isspace() is False # includes \n \t \r \v
+ and character.isprintable() is False
+ and character != "\x1a" # Why? Its the ASCII substitute character.
+ and character != "\ufeff" # bug discovered in Python,
+ # Zero Width No-Break Space located in Arabic Presentation Forms-B, Unicode 1.1 not acknowledged as space.
+ )
+
+
+def any_specified_encoding(sequence: bytes, search_zone: int = 8192) -> str | None:
+ """
+ Extract using ASCII-only decoder any specified encoding in the first n-bytes.
+ """
+ if not isinstance(sequence, bytes):
+ raise TypeError
+
+ seq_len: int = len(sequence)
+
+ results: list[str] = findall(
+ RE_POSSIBLE_ENCODING_INDICATION,
+ sequence[: min(seq_len, search_zone)].decode("ascii", errors="ignore"),
+ )
+
+ if len(results) == 0:
+ return None
+
+ for specified_encoding in results:
+ specified_encoding = specified_encoding.lower().replace("-", "_")
+
+ encoding_alias: str
+ encoding_iana: str
+
+ for encoding_alias, encoding_iana in aliases.items():
+ if encoding_alias == specified_encoding:
+ return encoding_iana
+ if encoding_iana == specified_encoding:
+ return encoding_iana
+
+ return None
+
+
+@lru_cache(maxsize=128)
+def is_multi_byte_encoding(name: str) -> bool:
+ """
+ Verify is a specific encoding is a multi byte one based on it IANA name
+ """
+ return name in {
+ "utf_8",
+ "utf_8_sig",
+ "utf_16",
+ "utf_16_be",
+ "utf_16_le",
+ "utf_32",
+ "utf_32_le",
+ "utf_32_be",
+ "utf_7",
+ } or issubclass(
+ importlib.import_module(f"encodings.{name}").IncrementalDecoder,
+ MultibyteIncrementalDecoder,
+ )
+
+
+def identify_sig_or_bom(sequence: bytes) -> tuple[str | None, bytes]:
+ """
+ Identify and extract SIG/BOM in given sequence.
+ """
+
+ for iana_encoding in ENCODING_MARKS:
+ marks: bytes | list[bytes] = ENCODING_MARKS[iana_encoding]
+
+ if isinstance(marks, bytes):
+ marks = [marks]
+
+ for mark in marks:
+ if sequence.startswith(mark):
+ return iana_encoding, mark
+
+ return None, b""
+
+
+def should_strip_sig_or_bom(iana_encoding: str) -> bool:
+ return iana_encoding not in {"utf_16", "utf_32"}
+
+
+def iana_name(cp_name: str, strict: bool = True) -> str:
+ """Returns the Python normalized encoding name (Not the IANA official name)."""
+ cp_name = cp_name.lower().replace("-", "_")
+
+ encoding_alias: str
+ encoding_iana: str
+
+ for encoding_alias, encoding_iana in aliases.items():
+ if cp_name in [encoding_alias, encoding_iana]:
+ return encoding_iana
+
+ if strict:
+ raise ValueError(f"Unable to retrieve IANA for '{cp_name}'")
+
+ return cp_name
+
+
+def cp_similarity(iana_name_a: str, iana_name_b: str) -> float:
+ if is_multi_byte_encoding(iana_name_a) or is_multi_byte_encoding(iana_name_b):
+ return 0.0
+
+ decoder_a = importlib.import_module(f"encodings.{iana_name_a}").IncrementalDecoder
+ decoder_b = importlib.import_module(f"encodings.{iana_name_b}").IncrementalDecoder
+
+ id_a: IncrementalDecoder = decoder_a(errors="ignore")
+ id_b: IncrementalDecoder = decoder_b(errors="ignore")
+
+ character_match_count: int = 0
+
+ for i in range(255):
+ to_be_decoded: bytes = bytes([i])
+ if id_a.decode(to_be_decoded) == id_b.decode(to_be_decoded):
+ character_match_count += 1
+
+ return character_match_count / 254
+
+
+def is_cp_similar(iana_name_a: str, iana_name_b: str) -> bool:
+ """
+ Determine if two code page are at least 80% similar. IANA_SUPPORTED_SIMILAR dict was generated using
+ the function cp_similarity.
+ """
+ return (
+ iana_name_a in IANA_SUPPORTED_SIMILAR
+ and iana_name_b in IANA_SUPPORTED_SIMILAR[iana_name_a]
+ )
+
+
+def set_logging_handler(
+ name: str = "charset_normalizer",
+ level: int = logging.INFO,
+ format_string: str = "%(asctime)s | %(levelname)s | %(message)s",
+) -> None:
+ logger = logging.getLogger(name)
+ logger.setLevel(level)
+
+ handler = logging.StreamHandler()
+ handler.setFormatter(logging.Formatter(format_string))
+ logger.addHandler(handler)
+
+
+def cut_sequence_chunks(
+ sequences: bytes,
+ encoding_iana: str,
+ offsets: range,
+ chunk_size: int,
+ bom_or_sig_available: bool,
+ strip_sig_or_bom: bool,
+ sig_payload: bytes,
+ is_multi_byte_decoder: bool,
+ decoded_payload: str | None = None,
+) -> Generator[str, None, None]:
+ if decoded_payload and is_multi_byte_decoder is False:
+ for i in offsets:
+ chunk = decoded_payload[i : i + chunk_size]
+ if not chunk:
+ break
+ yield chunk
+ else:
+ for i in offsets:
+ chunk_end = i + chunk_size
+ if chunk_end > len(sequences) + 8:
+ continue
+
+ cut_sequence = sequences[i : i + chunk_size]
+
+ if bom_or_sig_available and strip_sig_or_bom is False:
+ cut_sequence = sig_payload + cut_sequence
+
+ chunk = cut_sequence.decode(
+ encoding_iana,
+ errors="ignore" if is_multi_byte_decoder else "strict",
+ )
+
+ # multi-byte bad cutting detector and adjustment
+ # not the cleanest way to perform that fix but clever enough for now.
+ if is_multi_byte_decoder and i > 0:
+ chunk_partial_size_chk: int = min(chunk_size, 16)
+
+ if (
+ decoded_payload
+ and chunk[:chunk_partial_size_chk] not in decoded_payload
+ ):
+ for j in range(i, i - 4, -1):
+ cut_sequence = sequences[j:chunk_end]
+
+ if bom_or_sig_available and strip_sig_or_bom is False:
+ cut_sequence = sig_payload + cut_sequence
+
+ chunk = cut_sequence.decode(encoding_iana, errors="ignore")
+
+ if chunk[:chunk_partial_size_chk] in decoded_payload:
+ break
+
+ yield chunk
diff --git a/venv/lib/python3.10/site-packages/charset_normalizer/version.py b/venv/lib/python3.10/site-packages/charset_normalizer/version.py
new file mode 100644
index 0000000000000000000000000000000000000000..c843e5331131fe69b5bba96d3f0c1160dcaa943b
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/charset_normalizer/version.py
@@ -0,0 +1,8 @@
+"""
+Expose version
+"""
+
+from __future__ import annotations
+
+__version__ = "3.4.4"
+VERSION = __version__.split(".")
diff --git a/venv/lib/python3.10/site-packages/datasets/__init__.py b/venv/lib/python3.10/site-packages/datasets/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..268d13ad6dca1673680531220d87663d61ecafe6
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/__init__.py
@@ -0,0 +1,47 @@
+# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__version__ = "4.4.1"
+
+from .arrow_dataset import Column, Dataset
+from .arrow_reader import ReadInstruction
+from .builder import ArrowBasedBuilder, BuilderConfig, DatasetBuilder, GeneratorBasedBuilder
+from .combine import concatenate_datasets, interleave_datasets
+from .dataset_dict import DatasetDict, IterableDatasetDict
+from .download import *
+from .features import *
+from .fingerprint import disable_caching, enable_caching, is_caching_enabled
+from .info import DatasetInfo
+from .inspect import (
+ get_dataset_config_info,
+ get_dataset_config_names,
+ get_dataset_default_config_name,
+ get_dataset_infos,
+ get_dataset_split_names,
+)
+from .iterable_dataset import IterableColumn, IterableDataset
+from .load import load_dataset, load_dataset_builder, load_from_disk
+from .splits import (
+ NamedSplit,
+ NamedSplitAll,
+ Split,
+ SplitBase,
+ SplitDict,
+ SplitGenerator,
+ SplitInfo,
+ SubSplitInfo,
+ percent,
+)
+from .utils import *
+from .utils import logging
diff --git a/venv/lib/python3.10/site-packages/datasets/__pycache__/__init__.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/__pycache__/__init__.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..0395a3d6a48ab4126c7664adae7e9fb03b1ebd73
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/__pycache__/__init__.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/__pycache__/arrow_reader.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/__pycache__/arrow_reader.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..b4bed67e995d62c005a82bc02342e4993e0dff51
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/__pycache__/arrow_reader.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/__pycache__/arrow_writer.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/__pycache__/arrow_writer.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..4121a4217891209748831008633198e5ae6b5d81
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/__pycache__/arrow_writer.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/__pycache__/builder.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/__pycache__/builder.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..71a1f1bd22773e9165e977184071c130a93947f5
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/__pycache__/builder.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/__pycache__/combine.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/__pycache__/combine.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..b9b9174f70010d7bb61d957273ef32da8e53d586
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/__pycache__/combine.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/__pycache__/config.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/__pycache__/config.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..05455d9d7ce89f5c2d232e225eeb36f97b13f2da
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/__pycache__/config.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/__pycache__/data_files.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/__pycache__/data_files.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..aba436b89c2de857deae71c7d131f54d20674edd
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/__pycache__/data_files.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/__pycache__/distributed.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/__pycache__/distributed.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..bdce098de7cae6bd93f9e144faa0315c7216197f
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/__pycache__/distributed.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/__pycache__/exceptions.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/__pycache__/exceptions.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..8763bb69004540b166d4461d9fa9b781b7a5029a
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/__pycache__/exceptions.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/__pycache__/fingerprint.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/__pycache__/fingerprint.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..5392384cfc0cdb50b5808be4ff8c912be7a2fb1d
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/__pycache__/fingerprint.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/__pycache__/hub.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/__pycache__/hub.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..6162fc5496be6472855dd333acd083f0920c5cff
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/__pycache__/hub.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/__pycache__/info.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/__pycache__/info.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..478ddd55ee0b0cbd01356be073072aec67cf28c9
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/__pycache__/info.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/__pycache__/inspect.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/__pycache__/inspect.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..99da0cca6b0213c6e44a2967e6648d13b2c6e715
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/__pycache__/inspect.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/__pycache__/keyhash.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/__pycache__/keyhash.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..7085ed36cc41ae32c3be83edec9327cfe57ac16e
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/__pycache__/keyhash.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/__pycache__/load.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/__pycache__/load.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..9369b07cbb23a9b504df1da1ded3a5eeb9d3a12a
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/__pycache__/load.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/__pycache__/naming.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/__pycache__/naming.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..bf3da5c2afe50f8ce99191594f9c74ff764c8002
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/__pycache__/naming.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/__pycache__/search.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/__pycache__/search.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..bf083eac3f966e7759b563cb571c0a6d5c6c166c
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/__pycache__/search.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/__pycache__/splits.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/__pycache__/splits.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..72f3ae8e8635341247903fc94cb11ecda7636d1b
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/__pycache__/splits.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/__pycache__/streaming.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/__pycache__/streaming.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..9249804d3dccc24964010ad7a7eea295fc0b1078
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/__pycache__/streaming.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/__pycache__/table.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/__pycache__/table.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..f8dd3d75737d7a2a22327066d4ec8e551f1c3e8e
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/__pycache__/table.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/arrow_dataset.py b/venv/lib/python3.10/site-packages/datasets/arrow_dataset.py
new file mode 100644
index 0000000000000000000000000000000000000000..250442a0ed8904d13123611d941d7d26bf589db9
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/arrow_dataset.py
@@ -0,0 +1,6832 @@
+# Copyright 2020 The HuggingFace Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""Simple Dataset wrapping an Arrow Table."""
+
+import asyncio
+import contextlib
+import copy
+import fnmatch
+import glob
+import inspect
+import itertools
+import json
+import math
+import os
+import posixpath
+import random
+import re
+import shutil
+import string
+import sys
+import tempfile
+import time
+import warnings
+import weakref
+from collections import Counter, defaultdict
+from collections.abc import Iterable, Iterator, Mapping
+from collections.abc import Sequence as Sequence_
+from copy import deepcopy
+from functools import partial, wraps
+from io import BytesIO
+from math import ceil, floor
+from pathlib import Path
+from random import sample
+from typing import (
+ TYPE_CHECKING,
+ Any,
+ BinaryIO,
+ Callable,
+ Optional,
+ Union,
+ overload,
+)
+
+import fsspec
+import numpy as np
+import pandas as pd
+import pyarrow as pa
+import pyarrow.compute as pc
+from fsspec.core import url_to_fs
+from huggingface_hub import (
+ CommitInfo,
+ CommitOperationAdd,
+ CommitOperationDelete,
+ DatasetCard,
+ DatasetCardData,
+ HfApi,
+)
+from huggingface_hub.hf_api import RepoFile
+from huggingface_hub.utils import HfHubHTTPError, RepositoryNotFoundError
+from multiprocess import Pool
+from tqdm.contrib.concurrent import thread_map
+
+from . import config
+from .arrow_reader import ArrowReader
+from .arrow_writer import ArrowWriter, OptimizedTypedSequence
+from .data_files import sanitize_patterns
+from .download.streaming_download_manager import xgetsize
+from .features import Audio, ClassLabel, Features, Image, List, Value, Video
+from .features.features import (
+ FeatureType,
+ _align_features,
+ _check_if_features_can_be_aligned,
+ _fix_for_backward_compatible_features,
+ generate_from_arrow_type,
+ pandas_types_mapper,
+ require_decoding,
+)
+from .filesystems import is_remote_filesystem
+from .fingerprint import (
+ fingerprint_transform,
+ format_kwargs_for_fingerprint,
+ format_transform_for_fingerprint,
+ generate_fingerprint,
+ generate_random_fingerprint,
+ get_temporary_cache_files_directory,
+ is_caching_enabled,
+ maybe_register_dataset_for_temp_dir_deletion,
+ update_fingerprint,
+ validate_fingerprint,
+)
+from .formatting import format_table, get_format_type_from_alias, get_formatter, query_table
+from .formatting.formatting import LazyDict, _is_range_contiguous
+from .info import DatasetInfo, DatasetInfosDict
+from .naming import _split_re
+from .search import IndexableMixin
+from .splits import NamedSplit, Split, SplitDict, SplitInfo
+from .table import (
+ InMemoryTable,
+ MemoryMappedTable,
+ Table,
+ _memory_mapped_record_batch_reader_from_file,
+ cast_array_to_feature,
+ concat_tables,
+ embed_table_storage,
+ list_table_cache_files,
+ table_cast,
+ table_iter,
+ table_visitor,
+)
+from .utils import logging
+from .utils import tqdm as hf_tqdm
+from .utils.file_utils import estimate_dataset_size
+from .utils.info_utils import is_small_dataset
+from .utils.metadata import MetadataConfigs
+from .utils.py_utils import (
+ Literal,
+ asdict,
+ convert_file_size_to_int,
+ glob_pattern_to_regex,
+ iflatmap_unordered,
+ string_to_dict,
+)
+from .utils.stratify import stratified_shuffle_split_generate_indices
+from .utils.tf_utils import dataset_to_tf, minimal_tf_collate_fn, multiprocess_dataset_to_tf
+from .utils.typing import ListLike, PathLike
+
+
+if TYPE_CHECKING:
+ import sqlite3
+
+ import polars as pl
+ import pyspark
+ import sqlalchemy
+
+ from .dataset_dict import DatasetDict
+ from .iterable_dataset import IterableDataset
+
+logger = logging.get_logger(__name__)
+
+PUSH_TO_HUB_WITHOUT_METADATA_CONFIGS_SPLIT_PATTERN_SHARDED = (
+ "data/{split}-[0-9][0-9][0-9][0-9][0-9]-of-[0-9][0-9][0-9][0-9][0-9]*.parquet"
+)
+
+
+class DatasetInfoMixin:
+ """This base class exposes some attributes of DatasetInfo
+ at the base level of the Dataset for easy access.
+ """
+
+ def __init__(self, info: DatasetInfo, split: Optional[NamedSplit]):
+ self._info = info
+ self._split = split
+
+ @property
+ def info(self):
+ """[`~datasets.DatasetInfo`] object containing all the metadata in the dataset."""
+ return self._info
+
+ @property
+ def split(self):
+ """[`~datasets.NamedSplit`] object corresponding to a named dataset split."""
+ return self._split
+
+ @property
+ def builder_name(self) -> str:
+ return self._info.builder_name
+
+ @property
+ def citation(self) -> str:
+ return self._info.citation
+
+ @property
+ def config_name(self) -> str:
+ return self._info.config_name
+
+ @property
+ def dataset_size(self) -> Optional[int]:
+ return self._info.dataset_size
+
+ @property
+ def description(self) -> str:
+ return self._info.description
+
+ @property
+ def download_checksums(self) -> Optional[dict]:
+ return self._info.download_checksums
+
+ @property
+ def download_size(self) -> Optional[int]:
+ return self._info.download_size
+
+ @property
+ def features(self) -> Optional[Features]:
+ return self._info.features.copy() if self._info.features is not None else None
+
+ @property
+ def homepage(self) -> Optional[str]:
+ return self._info.homepage
+
+ @property
+ def license(self) -> Optional[str]:
+ return self._info.license
+
+ @property
+ def size_in_bytes(self) -> Optional[int]:
+ return self._info.size_in_bytes
+
+ @property
+ def supervised_keys(self):
+ return self._info.supervised_keys
+
+ @property
+ def version(self):
+ return self._info.version
+
+
+class TensorflowDatasetMixin:
+ _TF_DATASET_REFS = set()
+
+ @staticmethod
+ def _get_output_signature(
+ dataset: "Dataset",
+ collate_fn: Callable,
+ collate_fn_args: dict,
+ cols_to_retain: Optional[list[str]] = None,
+ batch_size: Optional[int] = None,
+ num_test_batches: int = 20,
+ ):
+ """Private method used by `to_tf_dataset()` to find the shapes and dtypes of samples from this dataset
+ after being passed through the collate_fn. Tensorflow needs an exact signature for tf.numpy_function, so
+ the only way to do this is to run test batches - the collator may add or rename columns, so we can't figure
+ it out just by inspecting the dataset.
+
+ Args:
+ dataset (`Dataset`): Dataset to load samples from.
+ collate_fn(`bool`): Shuffle the dataset order when loading. Recommended True for training, False for
+ validation/evaluation.
+ collate_fn(`Callable`): A function or callable object (such as a `DataCollator`) that will collate
+ lists of samples into a batch.
+ collate_fn_args (`Dict`): A `dict` of keyword arguments to be passed to the
+ `collate_fn`.
+ batch_size (`int`, optional): The size of batches loaded from the dataset. Used for shape inference.
+ Can be None, which indicates that batch sizes can be variable.
+ num_test_batches (`int`): The number of batches to load from the dataset for shape inference.
+
+ Returns:
+ `dict`: Dict mapping column names to tf.Tensorspec objects
+ `dict`: Dict mapping column names to np.dtype objects
+ """
+ if config.TF_AVAILABLE:
+ import tensorflow as tf
+ else:
+ raise ImportError("Called a Tensorflow-specific function but Tensorflow is not installed.")
+
+ if len(dataset) == 0:
+ raise ValueError("Unable to get the output signature because the dataset is empty.")
+ if batch_size is not None:
+ batch_size = min(len(dataset), batch_size)
+ test_batch_size = 1
+
+ if cols_to_retain is not None:
+ cols_to_retain = list(set(cols_to_retain + ["label_ids", "label", "labels"]))
+
+ test_batches = []
+ for _ in range(num_test_batches):
+ indices = sample(range(len(dataset)), test_batch_size)
+ test_batch = dataset[indices]
+ if cols_to_retain is not None:
+ test_batch = {key: value for key, value in test_batch.items() if key in cols_to_retain}
+ test_batch = [{key: value[i] for key, value in test_batch.items()} for i in range(test_batch_size)]
+ test_batch = collate_fn(test_batch, **collate_fn_args)
+ test_batches.append(test_batch)
+
+ tf_columns_to_signatures = {}
+ np_columns_to_dtypes = {}
+ for column in test_batches[0].keys():
+ raw_arrays = [batch[column] for batch in test_batches]
+ # In case the collate_fn returns something strange
+ np_arrays = []
+ for array in raw_arrays:
+ if isinstance(array, np.ndarray):
+ np_arrays.append(array)
+ elif isinstance(array, tf.Tensor):
+ np_arrays.append(array.numpy())
+ else:
+ np_arrays.append(np.array(array))
+
+ if np.issubdtype(np_arrays[0].dtype, np.integer) or np_arrays[0].dtype == bool:
+ tf_dtype = tf.int64
+ np_dtype = np.int64
+ elif np.issubdtype(np_arrays[0].dtype, np.number):
+ tf_dtype = tf.float32
+ np_dtype = np.float32
+ elif np_arrays[0].dtype.kind == "U": # Unicode strings
+ np_dtype = np.str_
+ tf_dtype = tf.string
+ else:
+ raise RuntimeError(
+ f"Unrecognized array dtype {np_arrays[0].dtype}. \n"
+ "Nested types and image/audio types are not supported yet."
+ )
+ shapes = [array.shape for array in np_arrays]
+ static_shape = []
+ for dim in range(len(shapes[0])):
+ sizes = {shape[dim] for shape in shapes}
+ if dim == 0:
+ static_shape.append(batch_size)
+ continue
+ if len(sizes) == 1: # This dimension looks constant
+ static_shape.append(sizes.pop())
+ else: # Use None for variable dimensions
+ static_shape.append(None)
+ tf_columns_to_signatures[column] = tf.TensorSpec(shape=static_shape, dtype=tf_dtype)
+ np_columns_to_dtypes[column] = np_dtype
+
+ return tf_columns_to_signatures, np_columns_to_dtypes
+
+ def to_tf_dataset(
+ self,
+ batch_size: Optional[int] = None,
+ columns: Optional[Union[str, list[str]]] = None,
+ shuffle: bool = False,
+ collate_fn: Optional[Callable] = None,
+ drop_remainder: bool = False,
+ collate_fn_args: Optional[dict[str, Any]] = None,
+ label_cols: Optional[Union[str, list[str]]] = None,
+ prefetch: bool = True,
+ num_workers: int = 0,
+ num_test_batches: int = 20,
+ ):
+ """Create a `tf.data.Dataset` from the underlying Dataset. This `tf.data.Dataset` will load and collate batches from
+ the Dataset, and is suitable for passing to methods like `model.fit()` or `model.predict()`. The dataset will yield
+ `dicts` for both inputs and labels unless the `dict` would contain only a single key, in which case a raw
+ `tf.Tensor` is yielded instead.
+
+ Args:
+ batch_size (`int`, *optional*):
+ Size of batches to load from the dataset. Defaults to `None`, which implies that the dataset won't be
+ batched, but the returned dataset can be batched later with `tf_dataset.batch(batch_size)`.
+ columns (`List[str]` or `str`, *optional*):
+ Dataset column(s) to load in the `tf.data.Dataset`.
+ Column names that are created by the `collate_fn` and that do not exist in the original dataset can be used.
+ shuffle(`bool`, defaults to `False`):
+ Shuffle the dataset order when loading. Recommended `True` for training, `False` for
+ validation/evaluation.
+ drop_remainder(`bool`, defaults to `False`):
+ Drop the last incomplete batch when loading. Ensures
+ that all batches yielded by the dataset will have the same length on the batch dimension.
+ collate_fn(`Callable`, *optional*):
+ A function or callable object (such as a `DataCollator`) that will collate
+ lists of samples into a batch.
+ collate_fn_args (`Dict`, *optional*):
+ An optional `dict` of keyword arguments to be passed to the
+ `collate_fn`.
+ label_cols (`List[str]` or `str`, defaults to `None`):
+ Dataset column(s) to load as labels.
+ Note that many models compute loss internally rather than letting Keras do it, in which case
+ passing the labels here is optional, as long as they're in the input `columns`.
+ prefetch (`bool`, defaults to `True`):
+ Whether to run the dataloader in a separate thread and maintain
+ a small buffer of batches for training. Improves performance by allowing data to be loaded in the
+ background while the model is training.
+ num_workers (`int`, defaults to `0`):
+ Number of workers to use for loading the dataset.
+ num_test_batches (`int`, defaults to `20`):
+ Number of batches to use to infer the output signature of the dataset.
+ The higher this number, the more accurate the signature will be, but the longer it will take to
+ create the dataset.
+
+ Returns:
+ `tf.data.Dataset`
+
+ Example:
+
+ ```py
+ >>> ds_train = ds["train"].to_tf_dataset(
+ ... columns=['input_ids', 'token_type_ids', 'attention_mask', 'label'],
+ ... shuffle=True,
+ ... batch_size=16,
+ ... collate_fn=data_collator,
+ ... )
+ ```
+ """
+ if config.TF_AVAILABLE:
+ import tensorflow as tf
+ else:
+ raise ImportError("Called a Tensorflow-specific function but Tensorflow is not installed.")
+
+ if (isinstance(columns, list) and len(columns) == 1) or (
+ isinstance(label_cols, list) and len(label_cols) == 1
+ ):
+ warnings.warn(
+ "The output of `to_tf_dataset` will change when a passing single element list for `labels` or "
+ "`columns` in the next datasets version. To return a tuple structure rather than dict, pass a "
+ "single string.\n"
+ "Old behaviour: columns=['a'], labels=['labels'] -> (tf.Tensor, tf.Tensor) \n"
+ " : columns='a', labels='labels' -> (tf.Tensor, tf.Tensor) \n"
+ "New behaviour: columns=['a'],labels=['labels'] -> ({'a': tf.Tensor}, {'labels': tf.Tensor}) \n"
+ " : columns='a', labels='labels' -> (tf.Tensor, tf.Tensor) ",
+ FutureWarning,
+ )
+
+ if isinstance(tf.distribute.get_strategy(), tf.distribute.TPUStrategy):
+ logger.warning(
+ "Note that to_tf_dataset() loads the data with a generator rather than a full tf.data "
+ "pipeline and is not compatible with remote TPU connections. If you encounter errors, please "
+ "try using a TPU VM or, if your data can fit in memory, loading it into memory as a dict of "
+ "Tensors instead of streaming with to_tf_dataset()."
+ )
+
+ if collate_fn is None:
+ # Set a very simple default collator that just stacks things together
+ collate_fn = minimal_tf_collate_fn
+ if collate_fn_args is None:
+ collate_fn_args = {}
+ if label_cols and not columns:
+ raise ValueError("Cannot specify label_cols without specifying columns!")
+ if label_cols is None:
+ label_cols = []
+ elif isinstance(label_cols, str):
+ label_cols = [label_cols]
+ if len(set(label_cols)) < len(label_cols):
+ raise ValueError("List of label_cols contains duplicates.")
+ if columns:
+ if isinstance(columns, str):
+ columns = [columns]
+ if len(set(columns)) < len(columns):
+ raise ValueError("List of columns contains duplicates.")
+ cols_to_retain = list(set(columns + label_cols))
+ else:
+ cols_to_retain = None # Indicates keeping all valid columns
+ columns = []
+
+ if self.format["type"] not in ["custom", "numpy"]:
+ dataset = self.with_format("numpy")
+ else:
+ dataset = self
+
+ # TODO(Matt, QL): deprecate the retention of label_ids and label
+
+ output_signature, columns_to_np_types = dataset._get_output_signature(
+ dataset,
+ collate_fn=collate_fn,
+ collate_fn_args=collate_fn_args,
+ cols_to_retain=cols_to_retain,
+ batch_size=batch_size if drop_remainder else None,
+ num_test_batches=num_test_batches,
+ )
+
+ if "labels" in output_signature:
+ if ("label_ids" in columns or "label" in columns) and "labels" not in columns:
+ columns = [col for col in columns if col not in ["label_ids", "label"]] + ["labels"]
+ if ("label_ids" in label_cols or "label" in label_cols) and "labels" not in label_cols:
+ label_cols = [col for col in label_cols if col not in ["label_ids", "label"]] + ["labels"]
+
+ for col in columns:
+ if col not in output_signature:
+ raise ValueError(f"Column {col} not found in dataset!")
+
+ for col in label_cols:
+ if col not in output_signature:
+ raise ValueError(f"Label column {col} not found in dataset!")
+
+ if num_workers == 0:
+ tf_dataset = dataset_to_tf(
+ dataset=dataset,
+ cols_to_retain=cols_to_retain,
+ collate_fn=collate_fn,
+ collate_fn_args=collate_fn_args,
+ columns_to_np_types=columns_to_np_types,
+ output_signature=output_signature,
+ shuffle=shuffle,
+ batch_size=batch_size,
+ drop_remainder=drop_remainder,
+ )
+ elif num_workers > 0:
+ if batch_size is None:
+ raise NotImplementedError(
+ "`batch_size` must be specified when using multiple workers, as unbatched multiprocessing "
+ "is not supported yet. Please provide a `batch_size` if `num_workers` is greater than 0."
+ )
+ tf_dataset = multiprocess_dataset_to_tf(
+ dataset=dataset,
+ cols_to_retain=cols_to_retain,
+ collate_fn=collate_fn,
+ collate_fn_args=collate_fn_args,
+ columns_to_np_types=columns_to_np_types,
+ output_signature=output_signature,
+ shuffle=shuffle,
+ batch_size=batch_size,
+ drop_remainder=drop_remainder,
+ num_workers=num_workers,
+ )
+ else:
+ raise ValueError("num_workers must be >= 0")
+
+ def split_features_and_labels(input_batch):
+ # TODO(Matt, QL): deprecate returning the dict content when there's only one key
+ features = {key: tensor for key, tensor in input_batch.items() if key in columns}
+ labels = {key: tensor for key, tensor in input_batch.items() if key in label_cols}
+ if len(features) == 1:
+ features = list(features.values())[0]
+ if len(labels) == 1:
+ labels = list(labels.values())[0]
+ if isinstance(labels, dict) and len(labels) == 0:
+ return features
+ else:
+ return features, labels
+
+ if cols_to_retain is not None:
+ tf_dataset = tf_dataset.map(split_features_and_labels)
+
+ if prefetch:
+ tf_dataset = tf_dataset.prefetch(tf.data.experimental.AUTOTUNE)
+
+ # Remove a reference to the open Arrow file on delete
+ def cleanup_callback(ref):
+ dataset.__del__()
+ self._TF_DATASET_REFS.remove(ref)
+
+ self._TF_DATASET_REFS.add(weakref.ref(tf_dataset, cleanup_callback))
+
+ return tf_dataset
+
+
+class DatasetTransformationNotAllowedError(Exception):
+ pass
+
+
+def transmit_format(func):
+ """Wrapper for dataset transforms that recreate a new Dataset to transmit the format of the original dataset to the new dataset"""
+
+ @wraps(func)
+ def wrapper(*args, **kwargs):
+ if args:
+ self: "Dataset" = args[0]
+ args = args[1:]
+ else:
+ self: "Dataset" = kwargs.pop("self")
+ # don't use self.format since it returns a list of columns for 'columns' even if self_format_columns is None
+ unformatted_columns = set(self.column_names) - set(self._format_columns or [])
+ self_format = {
+ "type": self._format_type,
+ "format_kwargs": self._format_kwargs,
+ "columns": self._format_columns,
+ "output_all_columns": self._output_all_columns,
+ }
+ # apply actual function
+ out: Union["Dataset", "DatasetDict"] = func(self, *args, **kwargs)
+ datasets: list["Dataset"] = list(out.values()) if isinstance(out, dict) else [out]
+ # re-apply format to the output
+ for dataset in datasets:
+ new_format = self_format.copy()
+ if new_format["columns"] is not None: # new formatted columns = (columns - previously unformatted columns)
+ # sort the columns to have a deterministic list of columns that we can compare with `out_format`
+ new_format["columns"] = sorted(set(dataset.column_names) - unformatted_columns)
+ out_format = {
+ "type": dataset._format_type,
+ "format_kwargs": dataset._format_kwargs,
+ "columns": sorted(dataset._format_columns) if dataset._format_columns is not None else None,
+ "output_all_columns": dataset._output_all_columns,
+ }
+ if out_format != new_format:
+ fingerprint = dataset._fingerprint
+ dataset.set_format(**new_format)
+ dataset._fingerprint = fingerprint
+ return out
+
+ wrapper._decorator_name_ = "transmit_format"
+ return wrapper
+
+
+def update_metadata_with_features(table: Table, features: Features):
+ """To be used in dataset transforms that modify the features of the dataset, in order to update the features stored in the metadata of its schema."""
+ features = Features({col_name: features[col_name] for col_name in table.column_names})
+ if table.schema.metadata is None or b"huggingface" not in table.schema.metadata:
+ pa_metadata = ArrowWriter._build_metadata(DatasetInfo(features=features))
+ else:
+ metadata = json.loads(table.schema.metadata[b"huggingface"].decode())
+ if "info" not in metadata:
+ metadata["info"] = asdict(DatasetInfo(features=features))
+ else:
+ metadata["info"]["features"] = asdict(DatasetInfo(features=features))["features"]
+ pa_metadata = {"huggingface": json.dumps(metadata)}
+ table = table.replace_schema_metadata(pa_metadata)
+ return table
+
+
+def _check_table(table) -> Table:
+ """We check the table type to make sure it's an instance of :class:`datasets.table.Table`"""
+ if isinstance(table, pa.Table):
+ # for a pyarrow table, we can just consider it as a in-memory table
+ # this is here for backward compatibility
+ return InMemoryTable(table)
+ elif isinstance(table, Table):
+ return table
+ else:
+ raise TypeError(f"Expected a pyarrow.Table or a datasets.table.Table object, but got {table}.")
+
+
+def _check_column_names(column_names: list[str]):
+ """Check the column names to make sure they don't contain duplicates."""
+ counter = Counter(column_names)
+ if not all(count == 1 for count in counter.values()):
+ duplicated_columns = [col for col in counter if counter[col] > 1]
+ raise ValueError(f"The table can't have duplicated columns but columns {duplicated_columns} are duplicated.")
+
+
+def _check_valid_indices_value(index, size):
+ if (index < 0 and index + size < 0) or (index >= size):
+ raise IndexError(f"Index {index} out of range for dataset of size {size}.")
+
+
+class NonExistentDatasetError(Exception):
+ """Used when we expect the existence of a dataset"""
+
+ pass
+
+
+class Column(Sequence_):
+ """
+ An iterable for a specific column of a [`Dataset`].
+
+ Example:
+
+ Iterate on the texts of the "text" column of a dataset:
+
+ ```python
+ for text in dataset["text"]:
+ ...
+ ```
+
+ It also works with nested columns:
+
+ ```python
+ for source in dataset["metadata"]["source"]:
+ ...
+ ```
+ """
+
+ def __init__(self, source: Union["Dataset", "Column"], column_name: str):
+ self.source = source
+ self.column_name = column_name
+ if not isinstance(source.features, dict) or column_name not in source.features:
+ raise ValueError(f"Column '{column_name}' doesn't exist.")
+ self.features = source.features[column_name]
+
+ def __iter__(self) -> Iterator[Any]:
+ if isinstance(self.source, Dataset):
+ if self.source._format_type == "custom":
+ # the formatting transform may require all columns
+ source = self.source
+ else:
+ source = self.source._fast_select_column(self.column_name)
+ else:
+ source = self.source
+ for example in source:
+ yield example[self.column_name]
+
+ def __getitem__(self, key: Union[int, str, list[int]]) -> Any:
+ if isinstance(key, str):
+ return Column(self, key)
+ elif isinstance(self.source, Dataset):
+ if self.source._format_type == "custom":
+ # the formatting transform may require all columns
+ source = self.source
+ else:
+ source = self.source._fast_select_column(self.column_name)
+ return source[key][self.column_name]
+ elif isinstance(key, int):
+ return self.source[key][self.column_name]
+ else:
+ return [item[self.column_name] for item in self.source[key]]
+
+ def __len__(self) -> int:
+ return len(self.source)
+
+ def __repr__(self):
+ return "Column(" + repr(list(self[:5])) + ")"
+
+ def __str__(self):
+ return "Column(" + str(list(self[:5])) + ")"
+
+ def __eq__(self, value):
+ if isinstance(value, Column):
+ return list(self) == list(value)
+ else:
+ return value == list(self)
+
+
+class Dataset(DatasetInfoMixin, IndexableMixin, TensorflowDatasetMixin):
+ """A Dataset backed by an Arrow table."""
+
+ def __init__(
+ self,
+ arrow_table: Table,
+ info: Optional[DatasetInfo] = None,
+ split: Optional[NamedSplit] = None,
+ indices_table: Optional[Table] = None,
+ fingerprint: Optional[str] = None,
+ ):
+ info = info.copy() if info is not None else DatasetInfo()
+ DatasetInfoMixin.__init__(self, info=info, split=split)
+ IndexableMixin.__init__(self)
+
+ self._data: Table = _check_table(arrow_table)
+ self._indices: Optional[Table] = _check_table(indices_table) if indices_table is not None else None
+ maybe_register_dataset_for_temp_dir_deletion(self)
+
+ self._format_type: Optional[str] = None
+ self._format_kwargs: dict = {}
+ self._format_columns: Optional[list] = None
+ self._output_all_columns: bool = False
+ self._fingerprint: str = fingerprint
+
+ # Read metadata
+
+ if self._data.schema.metadata is not None and b"huggingface" in self._data.schema.metadata:
+ metadata = json.loads(self._data.schema.metadata[b"huggingface"].decode())
+ if (
+ "fingerprint" in metadata and self._fingerprint is None
+ ): # try to load fingerprint from the arrow file metadata
+ self._fingerprint = metadata["fingerprint"]
+
+ # Infer features if None
+ inferred_features = Features.from_arrow_schema(arrow_table.schema)
+ if self.info.features is None:
+ self.info.features = inferred_features
+ else: # make sure the nested columns are in the right order
+ try:
+ self.info.features = self.info.features.reorder_fields_as(inferred_features)
+ except ValueError as e:
+ raise ValueError(
+ f"{e}\nThe 'source' features come from dataset_info.json, and the 'target' ones are those of the dataset arrow file."
+ )
+
+ # In case there are types like pa.dictionary that we need to convert to the underlying type
+
+ if self.data.schema != self.info.features.arrow_schema:
+ self._data = self.data.cast(self.info.features.arrow_schema)
+
+ # Infer fingerprint if None
+
+ if self._fingerprint is None:
+ self._fingerprint = generate_fingerprint(self)
+
+ # Sanity checks
+
+ if self._info.features is None:
+ raise ValueError("Features can't be None in a Dataset object")
+ if self._fingerprint is None:
+ raise ValueError("Fingerprint can't be None in a Dataset object")
+ if self.info.features.type != inferred_features.type:
+ raise ValueError(
+ f"External features info don't match the dataset:\nGot\n{self.info.features}\nwith type\n{self.info.features.type}\n\nbut expected something like\n{inferred_features}\nwith type\n{inferred_features.type}"
+ )
+
+ if self._indices is not None:
+ if not pa.types.is_unsigned_integer(self._indices.column(0).type):
+ raise ValueError(
+ f"indices must be an Arrow table of unsigned integers, current type is {self._indices.column(0).type}"
+ )
+ _check_column_names(self._data.column_names)
+
+ self._data = update_metadata_with_features(self._data, self._info.features)
+
+ @property
+ def features(self) -> Features:
+ features = super().features
+ if features is None: # this is already checked in __init__
+ raise ValueError("Features can't be None in a Dataset object")
+ return features
+
+ @classmethod
+ def from_file(
+ cls,
+ filename: str,
+ info: Optional[DatasetInfo] = None,
+ split: Optional[NamedSplit] = None,
+ indices_filename: Optional[str] = None,
+ in_memory: bool = False,
+ ) -> "Dataset":
+ """Instantiate a Dataset backed by an Arrow table at filename.
+
+ Args:
+ filename (`str`):
+ File name of the dataset.
+ info (`DatasetInfo`, *optional*):
+ Dataset information, like description, citation, etc.
+ split (`NamedSplit`, *optional*):
+ Name of the dataset split.
+ indices_filename (`str`, *optional*):
+ File names of the indices.
+ in_memory (`bool`, defaults to `False`):
+ Whether to copy the data in-memory.
+
+ Returns:
+ [`Dataset`]
+ """
+ table = ArrowReader.read_table(filename, in_memory=in_memory)
+
+ if indices_filename is not None:
+ indices_pa_table = ArrowReader.read_table(indices_filename, in_memory=in_memory)
+ else:
+ indices_pa_table = None
+
+ return cls(
+ arrow_table=table,
+ info=info,
+ split=split,
+ indices_table=indices_pa_table,
+ )
+
+ @classmethod
+ def from_buffer(
+ cls,
+ buffer: pa.Buffer,
+ info: Optional[DatasetInfo] = None,
+ split: Optional[NamedSplit] = None,
+ indices_buffer: Optional[pa.Buffer] = None,
+ ) -> "Dataset":
+ """Instantiate a Dataset backed by an Arrow buffer.
+
+ Args:
+ buffer (`pyarrow.Buffer`):
+ Arrow buffer.
+ info (`DatasetInfo`, *optional*):
+ Dataset information, like description, citation, etc.
+ split (`NamedSplit`, *optional*):
+ Name of the dataset split.
+ indices_buffer (`pyarrow.Buffer`, *optional*):
+ Indices Arrow buffer.
+
+ Returns:
+ [`Dataset`]
+ """
+ table = InMemoryTable.from_buffer(buffer)
+
+ if indices_buffer is not None:
+ indices_table = InMemoryTable.from_buffer(buffer)
+ else:
+ indices_table = None
+
+ return cls(table, info=info, split=split, indices_table=indices_table)
+
+ @classmethod
+ def from_pandas(
+ cls,
+ df: pd.DataFrame,
+ features: Optional[Features] = None,
+ info: Optional[DatasetInfo] = None,
+ split: Optional[NamedSplit] = None,
+ preserve_index: Optional[bool] = None,
+ ) -> "Dataset":
+ """
+ Convert `pandas.DataFrame` to a `pyarrow.Table` to create a [`Dataset`].
+
+ The column types in the resulting Arrow Table are inferred from the dtypes of the `pandas.Series` in the
+ DataFrame. In the case of non-object Series, the NumPy dtype is translated to its Arrow equivalent. In the
+ case of `object`, we need to guess the datatype by looking at the Python objects in this Series.
+
+ Be aware that Series of the `object` dtype don't carry enough information to always lead to a meaningful Arrow
+ type. In the case that we cannot infer a type, e.g. because the DataFrame is of length 0 or the Series only
+ contains `None/nan` objects, the type is set to `null`. This behavior can be avoided by constructing explicit
+ features and passing it to this function.
+
+ Important: a dataset created with from_pandas() lives in memory
+ and therefore doesn't have an associated cache directory.
+ This may change in the future, but in the meantime if you
+ want to reduce memory usage you should write it back on disk
+ and reload using e.g. save_to_disk / load_from_disk.
+
+ Args:
+ df (`pandas.DataFrame`):
+ Dataframe that contains the dataset.
+ features ([`Features`], *optional*):
+ Dataset features.
+ info (`DatasetInfo`, *optional*):
+ Dataset information, like description, citation, etc.
+ split (`NamedSplit`, *optional*):
+ Name of the dataset split.
+ preserve_index (`bool`, *optional*):
+ Whether to store the index as an additional column in the resulting Dataset.
+ The default of `None` will store the index as a column, except for `RangeIndex` which is stored as metadata only.
+ Use `preserve_index=True` to force it to be stored as a column.
+
+ Returns:
+ [`Dataset`]
+
+ Example:
+
+ ```py
+ >>> ds = Dataset.from_pandas(df)
+ ```
+ """
+ if info is not None and features is not None and info.features != features:
+ raise ValueError(
+ f"Features specified in `features` and `info.features` can't be different:\n{features}\n{info.features}"
+ )
+ features = features if features is not None else info.features if info is not None else None
+ if features is not None:
+ features = _fix_for_backward_compatible_features(features)
+ if info is None:
+ info = DatasetInfo()
+ info.features = features
+ table = InMemoryTable.from_pandas(
+ df=df,
+ preserve_index=preserve_index,
+ )
+ if features is not None:
+ # more expensive cast than InMemoryTable.from_pandas(..., schema=features.arrow_schema)
+ # needed to support the str to Audio conversion for instance
+ table = table.cast(features.arrow_schema)
+ return cls(table, info=info, split=split)
+
+ @classmethod
+ def from_polars(
+ cls,
+ df: "pl.DataFrame",
+ features: Optional[Features] = None,
+ info: Optional[DatasetInfo] = None,
+ split: Optional[NamedSplit] = None,
+ ) -> "Dataset":
+ """
+ Collect the underlying arrow arrays in an Arrow Table.
+
+ This operation is mostly zero copy.
+
+ Data types that do copy:
+ * CategoricalType
+
+ Args:
+ df (`polars.DataFrame`): DataFrame to convert to Arrow Table
+ features (`Features`, optional): Dataset features.
+ info (`DatasetInfo`, optional): Dataset information, like description, citation, etc.
+ split (`NamedSplit`, optional): Name of the dataset split.
+
+ Examples:
+ ```py
+ >>> ds = Dataset.from_polars(df)
+ ```
+ """
+ if info is not None and features is not None and info.features != features:
+ raise ValueError(
+ f"Features specified in `features` and `info.features` can't be different:\n{features}\n{info.features}"
+ )
+ features = features if features is not None else info.features if info is not None else None
+ if features is not None:
+ features = _fix_for_backward_compatible_features(features)
+ if info is None:
+ info = DatasetInfo()
+ info.features = features
+ table = InMemoryTable(df.to_arrow())
+ if features is not None:
+ # more expensive cast than InMemoryTable.from_polars(..., schema=features.arrow_schema)
+ # needed to support the str to Audio conversion for instance
+ table = table.cast(features.arrow_schema)
+ return cls(table, info=info, split=split)
+
+ @classmethod
+ def from_dict(
+ cls,
+ mapping: dict,
+ features: Optional[Features] = None,
+ info: Optional[DatasetInfo] = None,
+ split: Optional[NamedSplit] = None,
+ ) -> "Dataset":
+ """
+ Convert `dict` to a `pyarrow.Table` to create a [`Dataset`].
+
+ Important: a dataset created with from_dict() lives in memory
+ and therefore doesn't have an associated cache directory.
+ This may change in the future, but in the meantime if you
+ want to reduce memory usage you should write it back on disk
+ and reload using e.g. save_to_disk / load_from_disk.
+
+ Args:
+ mapping (`Mapping`):
+ Mapping of strings to Arrays or Python lists.
+ features ([`Features`], *optional*):
+ Dataset features.
+ info (`DatasetInfo`, *optional*):
+ Dataset information, like description, citation, etc.
+ split (`NamedSplit`, *optional*):
+ Name of the dataset split.
+
+ Returns:
+ [`Dataset`]
+ """
+ if info is not None and features is not None and info.features != features:
+ raise ValueError(
+ f"Features specified in `features` and `info.features` can't be different:\n{features}\n{info.features}"
+ )
+ features = features if features is not None else info.features if info is not None else None
+ if features is not None:
+ features = _fix_for_backward_compatible_features(features)
+ arrow_typed_mapping = {}
+ for col, data in mapping.items():
+ if isinstance(data, (pa.Array, pa.ChunkedArray)):
+ data = cast_array_to_feature(data, features[col]) if features is not None else data
+ else:
+ data = OptimizedTypedSequence(
+ features.encode_column(data, col) if features is not None else data,
+ type=features[col] if features is not None else None,
+ col=col,
+ )
+ arrow_typed_mapping[col] = data
+ mapping = arrow_typed_mapping
+ pa_table = InMemoryTable.from_pydict(mapping=mapping)
+ if info is None:
+ info = DatasetInfo()
+ info.features = features
+ if info.features is None:
+ info.features = Features(
+ {
+ col: generate_from_arrow_type(data.type)
+ if isinstance(data, (pa.Array, pa.ChunkedArray))
+ else data.get_inferred_type()
+ for col, data in mapping.items()
+ }
+ )
+ return cls(pa_table, info=info, split=split)
+
+ @classmethod
+ def from_list(
+ cls,
+ mapping: list[dict],
+ features: Optional[Features] = None,
+ info: Optional[DatasetInfo] = None,
+ split: Optional[NamedSplit] = None,
+ ) -> "Dataset":
+ """
+ Convert a list of dicts to a `pyarrow.Table` to create a [`Dataset`]`.
+
+ Note that the keys of the first entry will be used to determine the dataset columns,
+ regardless of what is passed to features.
+
+ Important: a dataset created with from_list() lives in memory
+ and therefore doesn't have an associated cache directory.
+ This may change in the future, but in the meantime if you
+ want to reduce memory usage you should write it back on disk
+ and reload using e.g. save_to_disk / load_from_disk.
+
+ Args:
+ mapping (`List[dict]`): A list of mappings of strings to row values.
+ features (`Features`, optional): Dataset features.
+ info (`DatasetInfo`, optional): Dataset information, like description, citation, etc.
+ split (`NamedSplit`, optional): Name of the dataset split.
+
+ Returns:
+ [`Dataset`]
+ """
+ # for simplicity and consistency wrt OptimizedTypedSequence we do not use InMemoryTable.from_pylist here
+ mapping = {k: [r.get(k) for r in mapping] for k in mapping[0]} if mapping else {}
+ return cls.from_dict(mapping, features, info, split)
+
+ @staticmethod
+ def from_csv(
+ path_or_paths: Union[PathLike, list[PathLike]],
+ split: Optional[NamedSplit] = None,
+ features: Optional[Features] = None,
+ cache_dir: str = None,
+ keep_in_memory: bool = False,
+ num_proc: Optional[int] = None,
+ **kwargs,
+ ):
+ """Create Dataset from CSV file(s).
+
+ Args:
+ path_or_paths (`path-like` or list of `path-like`):
+ Path(s) of the CSV file(s).
+ split ([`NamedSplit`], *optional*):
+ Split name to be assigned to the dataset.
+ features ([`Features`], *optional*):
+ Dataset features.
+ cache_dir (`str`, *optional*, defaults to `"~/.cache/huggingface/datasets"`):
+ Directory to cache data.
+ keep_in_memory (`bool`, defaults to `False`):
+ Whether to copy the data in-memory.
+ num_proc (`int`, *optional*, defaults to `None`):
+ Number of processes when downloading and generating the dataset locally.
+ This is helpful if the dataset is made of multiple files. Multiprocessing is disabled by default.
+
+
+ **kwargs (additional keyword arguments):
+ Keyword arguments to be passed to [`pandas.read_csv`].
+
+ Returns:
+ [`Dataset`]
+
+ Example:
+
+ ```py
+ >>> ds = Dataset.from_csv('path/to/dataset.csv')
+ ```
+ """
+ # Dynamic import to avoid circular dependency
+ from .io.csv import CsvDatasetReader
+
+ return CsvDatasetReader(
+ path_or_paths,
+ split=split,
+ features=features,
+ cache_dir=cache_dir,
+ keep_in_memory=keep_in_memory,
+ num_proc=num_proc,
+ **kwargs,
+ ).read()
+
+ @staticmethod
+ def from_generator(
+ generator: Callable,
+ features: Optional[Features] = None,
+ cache_dir: str = None,
+ keep_in_memory: bool = False,
+ gen_kwargs: Optional[dict] = None,
+ num_proc: Optional[int] = None,
+ split: NamedSplit = Split.TRAIN,
+ fingerprint: Optional[str] = None,
+ **kwargs,
+ ):
+ """Create a Dataset from a generator.
+
+ Args:
+ generator (:`Callable`):
+ A generator function that `yields` examples.
+ features ([`Features`], *optional*):
+ Dataset features.
+ cache_dir (`str`, *optional*, defaults to `"~/.cache/huggingface/datasets"`):
+ Directory to cache data.
+ keep_in_memory (`bool`, defaults to `False`):
+ Whether to copy the data in-memory.
+ gen_kwargs(`dict`, *optional*):
+ Keyword arguments to be passed to the `generator` callable.
+ You can define a sharded dataset by passing the list of shards in `gen_kwargs` and setting `num_proc` greater than 1.
+ num_proc (`int`, *optional*, defaults to `None`):
+ Number of processes when downloading and generating the dataset locally.
+ This is helpful if the dataset is made of multiple files. Multiprocessing is disabled by default.
+ If `num_proc` is greater than one, then all list values in `gen_kwargs` must be the same length. These values will be split between calls to the generator. The number of shards will be the minimum of the shortest list in `gen_kwargs` and `num_proc`.
+
+
+ split ([`NamedSplit`], defaults to `Split.TRAIN`):
+ Split name to be assigned to the dataset.
+
+
+ fingerprint (`str`, *optional*):
+ Fingerprint that will be used to generate dataset ID.
+ By default `fingerprint` is generated by hashing the generator function and all the args which can be slow
+ if it uses large objects like AI models.
+
+
+ **kwargs (additional keyword arguments):
+ Keyword arguments to be passed to :[`GeneratorConfig`].
+
+ Returns:
+ [`Dataset`]
+
+ Example:
+
+ ```py
+ >>> def gen():
+ ... yield {"text": "Good", "label": 0}
+ ... yield {"text": "Bad", "label": 1}
+ ...
+ >>> ds = Dataset.from_generator(gen)
+ ```
+
+ ```py
+ >>> def gen(shards):
+ ... for shard in shards:
+ ... with open(shard) as f:
+ ... for line in f:
+ ... yield {"line": line}
+ ...
+ >>> shards = [f"data{i}.txt" for i in range(32)]
+ >>> ds = Dataset.from_generator(gen, gen_kwargs={"shards": shards})
+ ```
+ """
+ from .io.generator import GeneratorDatasetInputStream
+
+ return GeneratorDatasetInputStream(
+ generator=generator,
+ features=features,
+ cache_dir=cache_dir,
+ keep_in_memory=keep_in_memory,
+ gen_kwargs=gen_kwargs,
+ num_proc=num_proc,
+ split=split,
+ fingerprint=fingerprint,
+ **kwargs,
+ ).read()
+
+ @staticmethod
+ def from_json(
+ path_or_paths: Union[PathLike, list[PathLike]],
+ split: Optional[NamedSplit] = None,
+ features: Optional[Features] = None,
+ cache_dir: str = None,
+ keep_in_memory: bool = False,
+ field: Optional[str] = None,
+ num_proc: Optional[int] = None,
+ **kwargs,
+ ):
+ """Create Dataset from JSON or JSON Lines file(s).
+
+ Args:
+ path_or_paths (`path-like` or list of `path-like`):
+ Path(s) of the JSON or JSON Lines file(s).
+ split ([`NamedSplit`], *optional*):
+ Split name to be assigned to the dataset.
+ features ([`Features`], *optional*):
+ Dataset features.
+ cache_dir (`str`, *optional*, defaults to `"~/.cache/huggingface/datasets"`):
+ Directory to cache data.
+ keep_in_memory (`bool`, defaults to `False`):
+ Whether to copy the data in-memory.
+ field (`str`, *optional*):
+ Field name of the JSON file where the dataset is contained in.
+ num_proc (`int`, *optional* defaults to `None`):
+ Number of processes when downloading and generating the dataset locally.
+ This is helpful if the dataset is made of multiple files. Multiprocessing is disabled by default.
+
+
+ **kwargs (additional keyword arguments):
+ Keyword arguments to be passed to [`JsonConfig`].
+
+ Returns:
+ [`Dataset`]
+
+ Example:
+
+ ```py
+ >>> ds = Dataset.from_json('path/to/dataset.json')
+ ```
+ """
+ # Dynamic import to avoid circular dependency
+ from .io.json import JsonDatasetReader
+
+ return JsonDatasetReader(
+ path_or_paths,
+ split=split,
+ features=features,
+ cache_dir=cache_dir,
+ keep_in_memory=keep_in_memory,
+ field=field,
+ num_proc=num_proc,
+ **kwargs,
+ ).read()
+
+ @staticmethod
+ def from_parquet(
+ path_or_paths: Union[PathLike, list[PathLike]],
+ split: Optional[NamedSplit] = None,
+ features: Optional[Features] = None,
+ cache_dir: str = None,
+ keep_in_memory: bool = False,
+ columns: Optional[list[str]] = None,
+ num_proc: Optional[int] = None,
+ **kwargs,
+ ):
+ """Create Dataset from Parquet file(s).
+
+ Args:
+ path_or_paths (`path-like` or list of `path-like`):
+ Path(s) of the Parquet file(s).
+ split (`NamedSplit`, *optional*):
+ Split name to be assigned to the dataset.
+ features (`Features`, *optional*):
+ Dataset features.
+ cache_dir (`str`, *optional*, defaults to `"~/.cache/huggingface/datasets"`):
+ Directory to cache data.
+ keep_in_memory (`bool`, defaults to `False`):
+ Whether to copy the data in-memory.
+ columns (`List[str]`, *optional*):
+ If not `None`, only these columns will be read from the file.
+ A column name may be a prefix of a nested field, e.g. 'a' will select
+ 'a.b', 'a.c', and 'a.d.e'.
+ num_proc (`int`, *optional*, defaults to `None`):
+ Number of processes when downloading and generating the dataset locally.
+ This is helpful if the dataset is made of multiple files. Multiprocessing is disabled by default.
+
+
+ **kwargs (additional keyword arguments):
+ Keyword arguments to be passed to [`ParquetConfig`].
+
+ Returns:
+ [`Dataset`]
+
+ Example:
+
+ ```py
+ >>> ds = Dataset.from_parquet('path/to/dataset.parquet')
+ ```
+ """
+ # Dynamic import to avoid circular dependency
+ from .io.parquet import ParquetDatasetReader
+
+ return ParquetDatasetReader(
+ path_or_paths,
+ split=split,
+ features=features,
+ cache_dir=cache_dir,
+ keep_in_memory=keep_in_memory,
+ columns=columns,
+ num_proc=num_proc,
+ **kwargs,
+ ).read()
+
+ @staticmethod
+ def from_text(
+ path_or_paths: Union[PathLike, list[PathLike]],
+ split: Optional[NamedSplit] = None,
+ features: Optional[Features] = None,
+ cache_dir: str = None,
+ keep_in_memory: bool = False,
+ num_proc: Optional[int] = None,
+ **kwargs,
+ ):
+ """Create Dataset from text file(s).
+
+ Args:
+ path_or_paths (`path-like` or list of `path-like`):
+ Path(s) of the text file(s).
+ split (`NamedSplit`, *optional*):
+ Split name to be assigned to the dataset.
+ features (`Features`, *optional*):
+ Dataset features.
+ cache_dir (`str`, *optional*, defaults to `"~/.cache/huggingface/datasets"`):
+ Directory to cache data.
+ keep_in_memory (`bool`, defaults to `False`):
+ Whether to copy the data in-memory.
+ num_proc (`int`, *optional*, defaults to `None`):
+ Number of processes when downloading and generating the dataset locally.
+ This is helpful if the dataset is made of multiple files. Multiprocessing is disabled by default.
+
+
+ **kwargs (additional keyword arguments):
+ Keyword arguments to be passed to [`TextConfig`].
+
+ Returns:
+ [`Dataset`]
+
+ Example:
+
+ ```py
+ >>> ds = Dataset.from_text('path/to/dataset.txt')
+ ```
+ """
+ # Dynamic import to avoid circular dependency
+ from .io.text import TextDatasetReader
+
+ return TextDatasetReader(
+ path_or_paths,
+ split=split,
+ features=features,
+ cache_dir=cache_dir,
+ keep_in_memory=keep_in_memory,
+ num_proc=num_proc,
+ **kwargs,
+ ).read()
+
+ @staticmethod
+ def from_spark(
+ df: "pyspark.sql.DataFrame",
+ split: Optional[NamedSplit] = None,
+ features: Optional[Features] = None,
+ keep_in_memory: bool = False,
+ cache_dir: str = None,
+ working_dir: str = None,
+ load_from_cache_file: bool = True,
+ **kwargs,
+ ):
+ """Create a Dataset from Spark DataFrame. Dataset downloading is distributed over Spark workers.
+
+ Args:
+ df (`pyspark.sql.DataFrame`):
+ The DataFrame containing the desired data.
+ split (`NamedSplit`, *optional*):
+ Split name to be assigned to the dataset.
+ features (`Features`, *optional*):
+ Dataset features.
+ cache_dir (`str`, *optional*, defaults to `"~/.cache/huggingface/datasets"`):
+ Directory to cache data. When using a multi-node Spark cluster, the cache_dir must be accessible to both
+ workers and the driver.
+ keep_in_memory (`bool`):
+ Whether to copy the data in-memory.
+ working_dir (`str`, *optional*)
+ Intermediate directory for each Spark worker to write data to before moving it to `cache_dir`. Setting
+ a non-NFS intermediate directory may improve performance.
+ load_from_cache_file (`bool`):
+ Whether to load the dataset from the cache if possible.
+
+ Returns:
+ [`Dataset`]
+
+ Example:
+
+ ```py
+ >>> df = spark.createDataFrame(
+ >>> data=[[1, "Elia"], [2, "Teo"], [3, "Fang"]],
+ >>> columns=["id", "name"],
+ >>> )
+ >>> ds = Dataset.from_spark(df)
+ ```
+ """
+ # Dynamic import to avoid circular dependency
+ from .io.spark import SparkDatasetReader
+
+ if sys.platform == "win32":
+ raise OSError("Dataset.from_spark is not currently supported on Windows")
+
+ return SparkDatasetReader(
+ df,
+ split=split,
+ features=features,
+ streaming=False,
+ cache_dir=cache_dir,
+ keep_in_memory=keep_in_memory,
+ working_dir=working_dir,
+ load_from_cache_file=load_from_cache_file,
+ **kwargs,
+ ).read()
+
+ @staticmethod
+ def from_sql(
+ sql: Union[str, "sqlalchemy.sql.Selectable"],
+ con: Union[str, "sqlalchemy.engine.Connection", "sqlalchemy.engine.Engine", "sqlite3.Connection"],
+ features: Optional[Features] = None,
+ cache_dir: str = None,
+ keep_in_memory: bool = False,
+ **kwargs,
+ ):
+ """Create Dataset from SQL query or database table.
+
+ Args:
+ sql (`str` or `sqlalchemy.sql.Selectable`):
+ SQL query to be executed or a table name.
+ con (`str` or `sqlite3.Connection` or `sqlalchemy.engine.Connection` or `sqlalchemy.engine.Connection`):
+ A [URI string](https://docs.sqlalchemy.org/en/13/core/engines.html#database-urls) used to instantiate a database connection or a SQLite3/SQLAlchemy connection object.
+ features ([`Features`], *optional*):
+ Dataset features.
+ cache_dir (`str`, *optional*, defaults to `"~/.cache/huggingface/datasets"`):
+ Directory to cache data.
+ keep_in_memory (`bool`, defaults to `False`):
+ Whether to copy the data in-memory.
+ **kwargs (additional keyword arguments):
+ Keyword arguments to be passed to [`SqlConfig`].
+
+ Returns:
+ [`Dataset`]
+
+ Example:
+
+ ```py
+ >>> # Fetch a database table
+ >>> ds = Dataset.from_sql("test_data", "postgres:///db_name")
+ >>> # Execute a SQL query on the table
+ >>> ds = Dataset.from_sql("SELECT sentence FROM test_data", "postgres:///db_name")
+ >>> # Use a Selectable object to specify the query
+ >>> from sqlalchemy import select, text
+ >>> stmt = select([text("sentence")]).select_from(text("test_data"))
+ >>> ds = Dataset.from_sql(stmt, "postgres:///db_name")
+ ```
+
+ > [!TIP]
+ > The returned dataset can only be cached if `con` is specified as URI string.
+ """
+ from .io.sql import SqlDatasetReader
+
+ return SqlDatasetReader(
+ sql,
+ con,
+ features=features,
+ cache_dir=cache_dir,
+ keep_in_memory=keep_in_memory,
+ **kwargs,
+ ).read()
+
+ def __setstate__(self, state):
+ self.__dict__.update(state)
+ maybe_register_dataset_for_temp_dir_deletion(self)
+ return self
+
+ def __del__(self):
+ if hasattr(self, "_data"):
+ del self._data
+ if hasattr(self, "_indices"):
+ del self._indices
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ # Here `del` is used to del the pyarrow tables. This properly closes the files used for memory mapped tables
+ self.__del__()
+
+ def save_to_disk(
+ self,
+ dataset_path: PathLike,
+ max_shard_size: Optional[Union[str, int]] = None,
+ num_shards: Optional[int] = None,
+ num_proc: Optional[int] = None,
+ storage_options: Optional[dict] = None,
+ ):
+ """
+ Saves a dataset to a dataset directory, or in a filesystem using any implementation of `fsspec.spec.AbstractFileSystem`.
+
+ For [`Image`], [`Audio`] and [`Video`] data:
+
+ All the Image(), Audio() and Video() data are stored in the arrow files.
+ If you want to store paths or urls, please use the Value("string") type.
+
+ Args:
+ dataset_path (`path-like`):
+ Path (e.g. `dataset/train`) or remote URI (e.g. `s3://my-bucket/dataset/train`)
+ of the dataset directory where the dataset will be saved to.
+ max_shard_size (`int` or `str`, *optional*, defaults to `"500MB"`):
+ The maximum size of the dataset shards to be saved to the filesystem. If expressed as a string, needs to be digits followed by a unit
+ (like `"50MB"`).
+ num_shards (`int`, *optional*):
+ Number of shards to write. By default the number of shards depends on `max_shard_size` and `num_proc`.
+
+
+ num_proc (`int`, *optional*):
+ Number of processes when downloading and generating the dataset locally.
+ Multiprocessing is disabled by default.
+
+
+ storage_options (`dict`, *optional*):
+ Key/value pairs to be passed on to the file-system backend, if any.
+
+
+
+ Example:
+
+ ```py
+ >>> ds.save_to_disk("path/to/dataset/directory")
+ >>> ds.save_to_disk("path/to/dataset/directory", max_shard_size="1GB")
+ >>> ds.save_to_disk("path/to/dataset/directory", num_shards=1024)
+ ```
+ """
+ if max_shard_size is not None and num_shards is not None:
+ raise ValueError(
+ "Failed to push_to_hub: please specify either max_shard_size or num_shards, but not both."
+ )
+ if self.list_indexes():
+ raise ValueError("please remove all the indexes using `dataset.drop_index` before saving a dataset")
+
+ if num_shards is None:
+ dataset_nbytes = self._estimate_nbytes()
+ max_shard_size = convert_file_size_to_int(max_shard_size or config.MAX_SHARD_SIZE)
+ num_shards = int(dataset_nbytes / max_shard_size) + 1
+ num_shards = max(num_shards, num_proc or 1)
+
+ fs: fsspec.AbstractFileSystem
+ fs, _ = url_to_fs(dataset_path, **(storage_options or {}))
+
+ if not is_remote_filesystem(fs):
+ parent_cache_files_paths = {
+ Path(cache_filename["filename"]).resolve().parent for cache_filename in self.cache_files
+ }
+ # Check that the dataset doesn't overwrite itself. It can cause a permission error on Windows and a segfault on linux.
+ if Path(dataset_path).expanduser().resolve() in parent_cache_files_paths:
+ raise PermissionError(
+ f"Tried to overwrite {Path(dataset_path).expanduser().resolve()} but a dataset can't overwrite itself."
+ )
+
+ fs.makedirs(dataset_path, exist_ok=True)
+
+ # Get json serializable state
+ state = {
+ key: self.__dict__[key]
+ for key in [
+ "_fingerprint",
+ "_format_columns",
+ "_format_kwargs",
+ "_format_type",
+ "_output_all_columns",
+ ]
+ }
+ state["_split"] = str(self.split) if self.split is not None else self.split
+ state["_data_files"] = [
+ {"filename": f"data-{shard_idx:05d}-of-{num_shards:05d}.arrow"} for shard_idx in range(num_shards)
+ ]
+ for k in state["_format_kwargs"].keys():
+ try:
+ json.dumps(state["_format_kwargs"][k])
+ except TypeError as e:
+ raise TypeError(
+ str(e) + f"\nThe format kwargs must be JSON serializable, but key '{k}' isn't."
+ ) from None
+ # Get json serializable dataset info
+ dataset_info = asdict(self._info)
+
+ shards_done = 0
+ pbar = hf_tqdm(
+ unit=" examples",
+ total=len(self),
+ desc=f"Saving the dataset ({shards_done}/{num_shards} shards)",
+ )
+ kwargs_per_job = (
+ {
+ "job_id": shard_idx,
+ "shard": self.shard(num_shards=num_shards, index=shard_idx, contiguous=True),
+ "fpath": posixpath.join(dataset_path, f"data-{shard_idx:05d}-of-{num_shards:05d}.arrow"),
+ "storage_options": storage_options,
+ }
+ for shard_idx in range(num_shards)
+ )
+ shard_lengths = [None] * num_shards
+ shard_sizes = [None] * num_shards
+ if num_proc is not None and num_proc >= 1:
+ with Pool(num_proc) as pool:
+ with pbar:
+ for job_id, done, content in iflatmap_unordered(
+ pool, Dataset._save_to_disk_single, kwargs_iterable=kwargs_per_job
+ ):
+ if done:
+ shards_done += 1
+ pbar.set_description(f"Saving the dataset ({shards_done}/{num_shards} shards)")
+ logger.debug(f"Finished writing shard number {job_id} of {num_shards}.")
+ shard_lengths[job_id], shard_sizes[job_id] = content
+ else:
+ pbar.update(content)
+ else:
+ with pbar:
+ for kwargs in kwargs_per_job:
+ for job_id, done, content in Dataset._save_to_disk_single(**kwargs):
+ if done:
+ shards_done += 1
+ pbar.set_description(f"Saving the dataset ({shards_done}/{num_shards} shards)")
+ logger.debug(f"Finished writing shard number {job_id} of {num_shards}.")
+ shard_lengths[job_id], shard_sizes[job_id] = content
+ else:
+ pbar.update(content)
+ with fs.open(
+ posixpath.join(dataset_path, config.DATASET_STATE_JSON_FILENAME), "w", encoding="utf-8"
+ ) as state_file:
+ json.dump(state, state_file, indent=2, sort_keys=True)
+ with fs.open(
+ posixpath.join(dataset_path, config.DATASET_INFO_FILENAME), "w", encoding="utf-8"
+ ) as dataset_info_file:
+ # Sort only the first level of keys, or we might shuffle fields of nested features if we use sort_keys=True
+ sorted_keys_dataset_info = {key: dataset_info[key] for key in sorted(dataset_info)}
+ json.dump(sorted_keys_dataset_info, dataset_info_file, indent=2)
+
+ @staticmethod
+ def _save_to_disk_single(job_id: int, shard: "Dataset", fpath: str, storage_options: Optional[dict]):
+ batch_size = config.DEFAULT_MAX_BATCH_SIZE
+
+ num_examples_progress_update = 0
+ writer = ArrowWriter(
+ features=shard.features,
+ path=fpath,
+ storage_options=storage_options,
+ embed_local_files=True,
+ )
+ try:
+ _time = time.time()
+ for pa_table in shard.with_format("arrow").iter(batch_size):
+ writer.write_table(pa_table)
+ num_examples_progress_update += len(pa_table)
+ if time.time() > _time + config.PBAR_REFRESH_TIME_INTERVAL:
+ _time = time.time()
+ yield job_id, False, num_examples_progress_update
+ num_examples_progress_update = 0
+ finally:
+ yield job_id, False, num_examples_progress_update
+ num_examples, num_bytes = writer.finalize()
+ writer.close()
+
+ yield job_id, True, (num_examples, num_bytes)
+
+ @staticmethod
+ def _build_local_temp_path(uri_or_path: str) -> Path:
+ """
+ Builds and returns a Path concatenating a local temporary dir with the dir path (or absolute/relative
+ path extracted from the uri) passed.
+
+ Args:
+ uri_or_path (`str`): Path (e.g. `"dataset/train"`) or remote URI (e.g.
+ `"s3://my-bucket/dataset/train"`) to concatenate.
+
+ Returns:
+ :class:`Path`: the concatenated path (temp dir + path)
+ """
+ src_dataset_path = Path(uri_or_path)
+ tmp_dir = get_temporary_cache_files_directory()
+ return Path(tmp_dir, src_dataset_path.relative_to(src_dataset_path.anchor))
+
+ @staticmethod
+ def load_from_disk(
+ dataset_path: PathLike,
+ keep_in_memory: Optional[bool] = None,
+ storage_options: Optional[dict] = None,
+ ) -> "Dataset":
+ """
+ Loads a dataset that was previously saved using [`save_to_disk`] from a dataset directory, or from a
+ filesystem using any implementation of `fsspec.spec.AbstractFileSystem`.
+
+ Args:
+ dataset_path (`path-like`):
+ Path (e.g. `"dataset/train"`) or remote URI (e.g. `"s3//my-bucket/dataset/train"`)
+ of the dataset directory where the dataset will be loaded from.
+ keep_in_memory (`bool`, defaults to `None`):
+ Whether to copy the dataset in-memory. If `None`, the
+ dataset will not be copied in-memory unless explicitly enabled by setting
+ `datasets.config.IN_MEMORY_MAX_SIZE` to nonzero. See more details in the
+ [improve performance](../cache#improve-performance) section.
+ storage_options (`dict`, *optional*):
+ Key/value pairs to be passed on to the file-system backend, if any.
+
+
+
+ Returns:
+ [`Dataset`] or [`DatasetDict`]:
+ - If `dataset_path` is a path of a dataset directory, the dataset requested.
+ - If `dataset_path` is a path of a dataset dict directory, a `datasets.DatasetDict` with each split.
+
+ Example:
+
+ ```py
+ >>> ds = load_from_disk("path/to/dataset/directory")
+ ```
+ """
+ fs: fsspec.AbstractFileSystem
+ fs, dataset_path = url_to_fs(dataset_path, **(storage_options or {}))
+
+ dest_dataset_path = dataset_path
+ dataset_dict_json_path = posixpath.join(dest_dataset_path, config.DATASETDICT_JSON_FILENAME)
+ dataset_state_json_path = posixpath.join(dest_dataset_path, config.DATASET_STATE_JSON_FILENAME)
+ dataset_info_path = posixpath.join(dest_dataset_path, config.DATASET_INFO_FILENAME)
+
+ dataset_dict_is_file = fs.isfile(dataset_dict_json_path)
+ dataset_info_is_file = fs.isfile(dataset_info_path)
+ dataset_state_is_file = fs.isfile(dataset_state_json_path)
+ if not dataset_info_is_file and not dataset_state_is_file:
+ if dataset_dict_is_file:
+ raise FileNotFoundError(
+ f"No such files: '{dataset_info_path}', nor '{dataset_state_json_path}' found. Expected to load a `Dataset` object, but got a `DatasetDict`. Please use either `datasets.load_from_disk` or `DatasetDict.load_from_disk` instead."
+ )
+ raise FileNotFoundError(
+ f"No such files: '{dataset_info_path}', nor '{dataset_state_json_path}' found. Expected to load a `Dataset` object but provided path is not a `Dataset`."
+ )
+ if not dataset_info_is_file:
+ if dataset_dict_is_file:
+ raise FileNotFoundError(
+ f"No such file: '{dataset_info_path}' found. Expected to load a `Dataset` object, but got a `DatasetDict`. Please use either `datasets.load_from_disk` or `DatasetDict.load_from_disk` instead."
+ )
+ raise FileNotFoundError(
+ f"No such file: '{dataset_info_path}'. Expected to load a `Dataset` object but provided path is not a `Dataset`."
+ )
+ if not dataset_state_is_file:
+ if dataset_dict_is_file:
+ raise FileNotFoundError(
+ f"No such file: '{dataset_state_json_path}' found. Expected to load a `Dataset` object, but got a `DatasetDict`. Please use either `datasets.load_from_disk` or `DatasetDict.load_from_disk` instead."
+ )
+ raise FileNotFoundError(
+ f"No such file: '{dataset_state_json_path}'. Expected to load a `Dataset` object but provided path is not a `Dataset`."
+ )
+
+ # copies file from filesystem if it is remote filesystem to local filesystem and modifies dataset_path to temp directory containing local copies
+ if is_remote_filesystem(fs):
+ src_dataset_path = dest_dataset_path
+ dest_dataset_path = Dataset._build_local_temp_path(src_dataset_path)
+ fs.download(src_dataset_path, dest_dataset_path.as_posix(), recursive=True)
+ dataset_state_json_path = posixpath.join(dest_dataset_path, config.DATASET_STATE_JSON_FILENAME)
+ dataset_info_path = posixpath.join(dest_dataset_path, config.DATASET_INFO_FILENAME)
+
+ with open(dataset_state_json_path, encoding="utf-8") as state_file:
+ state = json.load(state_file)
+ with open(dataset_info_path, encoding="utf-8") as dataset_info_file:
+ dataset_info = DatasetInfo.from_dict(json.load(dataset_info_file))
+
+ dataset_size = estimate_dataset_size(
+ Path(dest_dataset_path, data_file["filename"]) for data_file in state["_data_files"]
+ )
+ keep_in_memory = keep_in_memory if keep_in_memory is not None else is_small_dataset(dataset_size)
+ table_cls = InMemoryTable if keep_in_memory else MemoryMappedTable
+
+ arrow_table = concat_tables(
+ thread_map(
+ table_cls.from_file,
+ [posixpath.join(dest_dataset_path, data_file["filename"]) for data_file in state["_data_files"]],
+ tqdm_class=hf_tqdm,
+ desc="Loading dataset from disk",
+ # set `disable=None` rather than `disable=False` by default to disable progress bar when no TTY attached
+ disable=len(state["_data_files"]) <= 16 or None,
+ )
+ )
+
+ split = state["_split"]
+ split = Split(split) if split is not None else split
+
+ dataset = Dataset(
+ arrow_table=arrow_table,
+ info=dataset_info,
+ split=split,
+ fingerprint=state["_fingerprint"],
+ )
+
+ format = {
+ "type": state["_format_type"],
+ "format_kwargs": state["_format_kwargs"],
+ "columns": state["_format_columns"],
+ "output_all_columns": state["_output_all_columns"],
+ }
+ dataset = dataset.with_format(**format)
+
+ return dataset
+
+ @property
+ def data(self) -> Table:
+ """The Apache Arrow table backing the dataset.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> ds.data
+ MemoryMappedTable
+ text: string
+ label: int64
+ ----
+ text: [["compassionately explores the seemingly irreconcilable situation between conservative christian parents and their estranged gay and lesbian children .","the soundtrack alone is worth the price of admission .","rodriguez does a splendid job of racial profiling hollywood style--casting excellent latin actors of all ages--a trend long overdue .","beneath the film's obvious determination to shock at any cost lies considerable skill and determination , backed by sheer nerve .","bielinsky is a filmmaker of impressive talent .","so beautifully acted and directed , it's clear that washington most certainly has a new career ahead of him if he so chooses .","a visual spectacle full of stunning images and effects .","a gentle and engrossing character study .","it's enough to watch huppert scheming , with her small , intelligent eyes as steady as any noir villain , and to enjoy the perfectly pitched web of tension that chabrol spins .","an engrossing portrait of uncompromising artists trying to create something original against the backdrop of a corporate music industry that only seems to care about the bottom line .",...,"ultimately , jane learns her place as a girl , softens up and loses some of the intensity that made her an interesting character to begin with .","ah-nuld's action hero days might be over .","it's clear why deuces wild , which was shot two years ago , has been gathering dust on mgm's shelf .","feels like nothing quite so much as a middle-aged moviemaker's attempt to surround himself with beautiful , half-naked women .","when the precise nature of matthew's predicament finally comes into sharp focus , the revelation fails to justify the build-up .","this picture is murder by numbers , and as easy to be bored by as your abc's , despite a few whopping shootouts .","hilarious musical comedy though stymied by accents thick as mud .","if you are into splatter movies , then you will probably have a reasonably good time with the salton sea .","a dull , simple-minded and stereotypical tale of drugs , death and mind-numbing indifference on the inner-city streets .","the feature-length stretch . . . strains the show's concept ."]]
+ label: [[1,1,1,1,1,1,1,1,1,1,...,0,0,0,0,0,0,0,0,0,0]]
+ ```
+ """
+ return self._data
+
+ @property
+ def cache_files(self) -> list[dict]:
+ """The cache files containing the Apache Arrow table backing the dataset.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> ds.cache_files
+ [{'filename': '/root/.cache/huggingface/datasets/rotten_tomatoes_movie_review/default/1.0.0/40d411e45a6ce3484deed7cc15b82a53dad9a72aafd9f86f8f227134bec5ca46/rotten_tomatoes_movie_review-validation.arrow'}]
+ ```
+ """
+ cache_files = list_table_cache_files(self._data)
+ if self._indices is not None:
+ cache_files += list_table_cache_files(self._indices)
+ return [{"filename": cache_filename} for cache_filename in cache_files]
+
+ @property
+ def num_columns(self) -> int:
+ """Number of columns in the dataset.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> ds.num_columns
+ 2
+ ```
+ """
+ return self._data.num_columns
+
+ @property
+ def num_rows(self) -> int:
+ """Number of rows in the dataset (same as [`Dataset.__len__`]).
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> ds.num_rows
+ 1066
+ ```
+ """
+ if self._indices is not None:
+ return self._indices.num_rows
+ return self._data.num_rows
+
+ @property
+ def column_names(self) -> list[str]:
+ """Names of the columns in the dataset.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> ds.column_names
+ ['text', 'label']
+ ```
+ """
+ return self._data.column_names
+
+ @property
+ def shape(self) -> tuple[int, int]:
+ """Shape of the dataset (number of columns, number of rows).
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> ds.shape
+ (1066, 2)
+ ```
+ """
+ if self._indices is not None:
+ return (self._indices.num_rows, self._data.num_columns)
+ return self._data.shape
+
+ def unique(self, column: str) -> list:
+ """Return a list of the unique elements in a column.
+
+ This is implemented in the low-level backend and as such, very fast.
+
+ Args:
+ column (`str`):
+ Column name (list all the column names with [`~datasets.Dataset.column_names`]).
+
+ Returns:
+ `list`: List of unique elements in the given column.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> ds.unique('label')
+ [1, 0]
+ ```
+ """
+ if column not in self._data.column_names:
+ raise ValueError(f"Column ({column}) not in table columns ({self._data.column_names}).")
+
+ if self._indices is not None and self._indices.num_rows != self._data.num_rows:
+ dataset = self.flatten_indices()
+ else:
+ dataset = self
+
+ return dataset._data.column(column).unique().to_pylist()
+
+ def class_encode_column(self, column: str, include_nulls: bool = False) -> "Dataset":
+ """Casts the given column as [`~datasets.features.ClassLabel`] and updates the table.
+
+ Args:
+ column (`str`):
+ The name of the column to cast (list all the column names with [`~datasets.Dataset.column_names`])
+ include_nulls (`bool`, defaults to `False`):
+ Whether to include null values in the class labels. If `True`, the null values will be encoded as the `"None"` class label.
+
+
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("boolq", split="validation")
+ >>> ds.features
+ {'answer': Value('bool'),
+ 'passage': Value('string'),
+ 'question': Value('string')}
+ >>> ds = ds.class_encode_column('answer')
+ >>> ds.features
+ {'answer': ClassLabel(num_classes=2, names=['False', 'True']),
+ 'passage': Value('string'),
+ 'question': Value('string')}
+ ```
+ """
+ # Sanity checks
+ if column not in self._data.column_names:
+ raise ValueError(f"Column ({column}) not in table columns ({self._data.column_names}).")
+ src_feat = self._info.features[column]
+ if not isinstance(src_feat, Value):
+ raise ValueError(
+ f"Class encoding is only supported for {Value.__name__} column, and column {column} is {type(src_feat).__name__}."
+ )
+
+ if src_feat.dtype != "string" or (include_nulls and None in self.unique(column)):
+
+ def stringify_column(batch):
+ batch[column] = [
+ str(sample) if include_nulls or sample is not None else None for sample in batch[column]
+ ]
+ return batch
+
+ dset = self.map(
+ stringify_column,
+ batched=True,
+ desc="Stringifying the column",
+ )
+ else:
+ dset = self
+
+ # Create the new feature
+ class_names = sorted(str(sample) for sample in dset.unique(column) if include_nulls or sample is not None)
+ dst_feat = ClassLabel(names=class_names)
+
+ def cast_to_class_labels(batch):
+ batch[column] = [
+ dst_feat.str2int(str(sample)) if include_nulls or sample is not None else None
+ for sample in batch[column]
+ ]
+ return batch
+
+ new_features = dset.features.copy()
+ new_features[column] = dst_feat
+
+ dset = dset.map(
+ cast_to_class_labels,
+ batched=True,
+ features=new_features,
+ desc="Casting to class labels",
+ )
+
+ return dset
+
+ @fingerprint_transform(inplace=False)
+ def flatten(self, new_fingerprint: Optional[str] = None, max_depth=16) -> "Dataset":
+ """Flatten the table.
+ Each column with a struct type is flattened into one column per struct field.
+ Other columns are left unchanged.
+
+ Args:
+ new_fingerprint (`str`, *optional*):
+ The new fingerprint of the dataset after transform.
+ If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments.
+
+ Returns:
+ [`Dataset`]: A copy of the dataset with flattened columns.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("rajpurkar/squad", split="train")
+ >>> ds.features
+ {'id': Value('string'),
+ 'title': Value('string'),
+ 'context': Value('string'),
+ 'question': Value('string'),
+ 'answers': {'text': List(Value('string')),
+ 'answer_start': List(Value('int32'))}}
+ >>> ds = ds.flatten()
+ >>> ds
+ Dataset({
+ features: ['id', 'title', 'context', 'question', 'answers.text', 'answers.answer_start'],
+ num_rows: 87599
+ })
+ ```
+ """
+ dataset = copy.deepcopy(self)
+ for depth in range(1, max_depth):
+ if any(isinstance(field.type, pa.StructType) for field in dataset._data.schema):
+ dataset._data = dataset._data.flatten()
+ else:
+ break
+ dataset.info.features = self._info.features.flatten(max_depth=max_depth)
+ dataset.info.features = Features({col: dataset.info.features[col] for col in dataset.data.column_names})
+ dataset._data = update_metadata_with_features(dataset._data, dataset.features)
+ logger.info(f"Flattened dataset from depth {depth} to depth {1 if depth + 1 < max_depth else 'unknown'}.")
+ dataset._fingerprint = new_fingerprint
+ return dataset
+
+ def cast(
+ self,
+ features: Features,
+ batch_size: Optional[int] = 1000,
+ keep_in_memory: bool = False,
+ load_from_cache_file: Optional[bool] = None,
+ cache_file_name: Optional[str] = None,
+ writer_batch_size: Optional[int] = 1000,
+ num_proc: Optional[int] = None,
+ ) -> "Dataset":
+ """
+ Cast the dataset to a new set of features.
+
+ Args:
+ features ([`Features`]):
+ New features to cast the dataset to.
+ The name of the fields in the features must match the current column names.
+ The type of the data must also be convertible from one type to the other.
+ For non-trivial conversion, e.g. `str` <-> `ClassLabel` you should use [`~datasets.Dataset.map`] to update the Dataset.
+ batch_size (`int`, defaults to `1000`):
+ Number of examples per batch provided to cast.
+ If `batch_size <= 0` or `batch_size == None` then provide the full dataset as a single batch to cast.
+ keep_in_memory (`bool`, defaults to `False`):
+ Whether to copy the data in-memory.
+ load_from_cache_file (`bool`, defaults to `True` if caching is enabled):
+ If a cache file storing the current computation from `function`
+ can be identified, use it instead of recomputing.
+ cache_file_name (`str`, *optional*, defaults to `None`):
+ Provide the name of a path for the cache file. It is used to store the
+ results of the computation instead of the automatically generated cache file name.
+ writer_batch_size (`int`, defaults to `1000`):
+ Number of rows per write operation for the cache file writer.
+ This value is a good trade-off between memory usage during the processing, and processing speed.
+ Higher value makes the processing do fewer lookups, lower value consume less temporary memory while running [`~datasets.Dataset.map`].
+ num_proc (`int`, *optional*, defaults to `None`):
+ Number of processes for multiprocessing. By default it doesn't
+ use multiprocessing.
+
+ Returns:
+ [`Dataset`]: A copy of the dataset with casted features.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset, ClassLabel, Value
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> ds.features
+ {'label': ClassLabel(names=['neg', 'pos']),
+ 'text': Value('string')}
+ >>> new_features = ds.features.copy()
+ >>> new_features['label'] = ClassLabel(names=['bad', 'good'])
+ >>> new_features['text'] = Value('large_string')
+ >>> ds = ds.cast(new_features)
+ >>> ds.features
+ {'label': ClassLabel(names=['bad', 'good']),
+ 'text': Value('large_string')}
+ ```
+ """
+ if sorted(features) != sorted(self._data.column_names):
+ raise ValueError(
+ f"The columns in features ({list(features)}) must be identical "
+ f"as the columns in the dataset: {self._data.column_names}"
+ )
+
+ features = _fix_for_backward_compatible_features(features)
+ schema = features.arrow_schema
+ format = self.format
+ dataset = self.with_format("arrow")
+ # capture the PyArrow version here to make the lambda serializable on Windows
+ dataset = dataset.map(
+ partial(table_cast, schema=schema),
+ batched=True,
+ batch_size=batch_size,
+ keep_in_memory=keep_in_memory,
+ load_from_cache_file=load_from_cache_file,
+ cache_file_name=cache_file_name,
+ writer_batch_size=writer_batch_size,
+ num_proc=num_proc,
+ features=features,
+ desc="Casting the dataset",
+ )
+ dataset = dataset.with_format(**format)
+ return dataset
+
+ @fingerprint_transform(inplace=False)
+ def cast_column(self, column: str, feature: FeatureType, new_fingerprint: Optional[str] = None) -> "Dataset":
+ """Cast column to feature for decoding.
+
+ Args:
+ column (`str`):
+ Column name.
+ feature (`FeatureType`):
+ Target feature.
+ new_fingerprint (`str`, *optional*):
+ The new fingerprint of the dataset after transform.
+ If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments.
+
+ Returns:
+ [`Dataset`]
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset, ClassLabel
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> ds.features
+ {'label': ClassLabel(names=['neg', 'pos']),
+ 'text': Value('string')}
+ >>> ds = ds.cast_column('label', ClassLabel(names=['bad', 'good']))
+ >>> ds.features
+ {'label': ClassLabel(names=['bad', 'good']),
+ 'text': Value('string')}
+ ```
+ """
+ feature = _fix_for_backward_compatible_features(feature)
+ if hasattr(feature, "decode_example"):
+ dataset = copy.deepcopy(self)
+ dataset._info.features[column] = feature
+ dataset._fingerprint = new_fingerprint
+ dataset._data = dataset._data.cast(dataset.features.arrow_schema)
+ dataset._data = update_metadata_with_features(dataset._data, dataset.features)
+ return dataset
+ else:
+ features = self.features
+ features[column] = feature
+ return self.cast(features)
+
+ @transmit_format
+ @fingerprint_transform(inplace=False)
+ def remove_columns(self, column_names: Union[str, list[str]], new_fingerprint: Optional[str] = None) -> "Dataset":
+ """
+ Remove one or several column(s) in the dataset and the features associated to them.
+
+ You can also remove a column using [`~datasets.Dataset.map`] with `remove_columns` but the present method
+ doesn't copy the data of the remaining columns and is thus faster.
+
+ Args:
+ column_names (`Union[str, List[str]]`):
+ Name of the column(s) to remove.
+ new_fingerprint (`str`, *optional*):
+ The new fingerprint of the dataset after transform.
+ If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments.
+
+ Returns:
+ [`Dataset`]: A copy of the dataset object without the columns to remove.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> ds = ds.remove_columns('label')
+ Dataset({
+ features: ['text'],
+ num_rows: 1066
+ })
+ >>> ds = ds.remove_columns(column_names=ds.column_names) # Removing all the columns returns an empty dataset with the `num_rows` property set to 0
+ Dataset({
+ features: [],
+ num_rows: 0
+ })
+ ```
+ """
+ dataset = copy.deepcopy(self)
+ if isinstance(column_names, str):
+ column_names = [column_names]
+
+ missing_columns = set(column_names) - set(self._data.column_names)
+ if missing_columns:
+ raise ValueError(
+ f"Column name {list(missing_columns)} not in the dataset. "
+ f"Current columns in the dataset: {dataset._data.column_names}"
+ )
+
+ for column_name in column_names:
+ del dataset._info.features[column_name]
+
+ dataset._data = dataset._data.drop(column_names)
+ dataset._data = update_metadata_with_features(dataset._data, dataset.features)
+ dataset._fingerprint = new_fingerprint
+ return dataset
+
+ @fingerprint_transform(inplace=False)
+ def rename_column(
+ self, original_column_name: str, new_column_name: str, new_fingerprint: Optional[str] = None
+ ) -> "Dataset":
+ """
+ Rename a column in the dataset, and move the features associated to the original column under the new column
+ name.
+
+ Args:
+ original_column_name (`str`):
+ Name of the column to rename.
+ new_column_name (`str`):
+ New name for the column.
+ new_fingerprint (`str`, *optional*):
+ The new fingerprint of the dataset after transform.
+ If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments.
+
+ Returns:
+ [`Dataset`]: A copy of the dataset with a renamed column.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> ds = ds.rename_column('label', 'label_new')
+ Dataset({
+ features: ['text', 'label_new'],
+ num_rows: 1066
+ })
+ ```
+ """
+ dataset = copy.deepcopy(self)
+ if original_column_name not in dataset._data.column_names:
+ raise ValueError(
+ f"Original column name {original_column_name} not in the dataset. "
+ f"Current columns in the dataset: {dataset._data.column_names}"
+ )
+ if new_column_name in dataset._data.column_names:
+ raise ValueError(
+ f"New column name {new_column_name} already in the dataset. "
+ f"Please choose a column name which is not already in the dataset. "
+ f"Current columns in the dataset: {dataset._data.column_names}"
+ )
+ if not new_column_name:
+ raise ValueError("New column name is empty.")
+
+ def rename(columns):
+ return [new_column_name if col == original_column_name else col for col in columns]
+
+ new_column_names = rename(self._data.column_names)
+ if self._format_columns is not None:
+ dataset._format_columns = rename(self._format_columns)
+
+ dataset._info.features = Features(
+ {
+ new_column_name if col == original_column_name else col: feature
+ for col, feature in self._info.features.items()
+ }
+ )
+
+ dataset._data = dataset._data.rename_columns(new_column_names)
+ dataset._data = update_metadata_with_features(dataset._data, dataset.features)
+ dataset._fingerprint = new_fingerprint
+ return dataset
+
+ @fingerprint_transform(inplace=False)
+ def rename_columns(self, column_mapping: dict[str, str], new_fingerprint: Optional[str] = None) -> "Dataset":
+ """
+ Rename several columns in the dataset, and move the features associated to the original columns under
+ the new column names.
+
+ Args:
+ column_mapping (`Dict[str, str]`):
+ A mapping of columns to rename to their new names
+ new_fingerprint (`str`, *optional*):
+ The new fingerprint of the dataset after transform.
+ If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments.
+
+ Returns:
+ [`Dataset`]: A copy of the dataset with renamed columns
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> ds = ds.rename_columns({'text': 'text_new', 'label': 'label_new'})
+ Dataset({
+ features: ['text_new', 'label_new'],
+ num_rows: 1066
+ })
+ ```
+ """
+ dataset = copy.deepcopy(self)
+
+ extra_columns = set(column_mapping.keys()) - set(dataset.column_names)
+ if extra_columns:
+ raise ValueError(
+ f"Original column names {extra_columns} not in the dataset. "
+ f"Current columns in the dataset: {dataset._data.column_names}"
+ )
+
+ number_of_duplicates_in_new_columns = len(column_mapping.values()) - len(set(column_mapping.values()))
+ if number_of_duplicates_in_new_columns != 0:
+ raise ValueError(
+ "New column names must all be different, but this column mapping "
+ f"has {number_of_duplicates_in_new_columns} duplicates"
+ )
+
+ empty_new_columns = [new_col for new_col in column_mapping.values() if not new_col]
+ if empty_new_columns:
+ raise ValueError(f"New column names {empty_new_columns} are empty.")
+
+ def rename(columns):
+ return [column_mapping[col] if col in column_mapping else col for col in columns]
+
+ new_column_names = rename(self._data.column_names)
+ if self._format_columns is not None:
+ dataset._format_columns = rename(self._format_columns)
+
+ dataset._info.features = Features(
+ {
+ column_mapping[col] if col in column_mapping else col: feature
+ for col, feature in (self._info.features or {}).items()
+ }
+ )
+
+ dataset._data = dataset._data.rename_columns(new_column_names)
+ dataset._data = update_metadata_with_features(dataset._data, dataset.features)
+ dataset._fingerprint = new_fingerprint
+ return dataset
+
+ @transmit_format
+ @fingerprint_transform(inplace=False)
+ def select_columns(self, column_names: Union[str, list[str]], new_fingerprint: Optional[str] = None) -> "Dataset":
+ """Select one or several column(s) in the dataset and the features
+ associated to them.
+
+ Args:
+ column_names (`Union[str, List[str]]`):
+ Name of the column(s) to keep.
+ new_fingerprint (`str`, *optional*):
+ The new fingerprint of the dataset after transform. If `None`,
+ the new fingerprint is computed using a hash of the previous
+ fingerprint, and the transform arguments.
+
+ Returns:
+ [`Dataset`]: A copy of the dataset object which only consists of
+ selected columns.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> ds = ds.select_columns(['text'])
+ >>> ds
+ Dataset({
+ features: ['text'],
+ num_rows: 1066
+ })
+ ```
+ """
+ if isinstance(column_names, str):
+ column_names = [column_names]
+
+ missing_columns = set(column_names) - set(self._data.column_names)
+ if missing_columns:
+ raise ValueError(
+ f"Column name {list(missing_columns)} not in the "
+ "dataset. Current columns in the dataset: "
+ f"{self._data.column_names}."
+ )
+
+ dataset = copy.deepcopy(self)
+ dataset._data = dataset._data.select(column_names)
+ dataset._info.features = Features({col: self._info.features[col] for col in dataset._data.column_names})
+ dataset._data = update_metadata_with_features(dataset._data, dataset.features)
+ dataset._fingerprint = new_fingerprint
+ return dataset
+
+ @transmit_format
+ def _fast_select_column(self, column_name: str) -> "Dataset":
+ dataset = copy.copy(self)
+ dataset._data = dataset._data.select([column_name])
+ dataset._info = DatasetInfo(features=Features({column_name: self._info.features[column_name]}))
+ return dataset
+
+ def __len__(self):
+ """Number of rows in the dataset.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> ds.__len__
+
+ ```
+ """
+ return self.num_rows
+
+ def __iter__(self):
+ """Iterate through the examples.
+
+ If a formatting is set with [`Dataset.set_format`] rows will be returned with the
+ selected format.
+ """
+ if self._indices is None:
+ # Fast iteration
+ # Benchmark: https://gist.github.com/mariosasko/0248288a2e3a7556873969717c1fe52b (fast_iter_batch)
+ format_kwargs = self._format_kwargs if self._format_kwargs is not None else {}
+ formatter = get_formatter(self._format_type, features=self._info.features, **format_kwargs)
+ batch_size = config.ARROW_READER_BATCH_SIZE_IN_DATASET_ITER
+ for pa_subtable in table_iter(self.data, batch_size=batch_size):
+ for i in range(pa_subtable.num_rows):
+ pa_subtable_ex = pa_subtable.slice(i, 1)
+ formatted_output = format_table(
+ pa_subtable_ex,
+ 0,
+ formatter=formatter,
+ format_columns=self._format_columns,
+ output_all_columns=self._output_all_columns,
+ )
+ yield formatted_output
+ else:
+ for i in range(self.num_rows):
+ yield self._getitem(
+ i,
+ )
+
+ def iter(self, batch_size: int, drop_last_batch: bool = False):
+ """Iterate through the batches of size `batch_size`.
+
+ If a formatting is set with [`~datasets.Dataset.set_format`] rows will be returned with the
+ selected format.
+
+ Args:
+ batch_size (:obj:`int`): size of each batch to yield.
+ drop_last_batch (:obj:`bool`, default `False`): Whether a last batch smaller than the batch_size should be
+ dropped
+ """
+ if self._indices is None:
+ # Fast iteration
+ # Benchmark: https://gist.github.com/mariosasko/0248288a2e3a7556873969717c1fe52b (fast_iter_batch)
+ format_kwargs = self._format_kwargs if self._format_kwargs is not None else {}
+ formatter = get_formatter(self._format_type, features=self._info.features, **format_kwargs)
+ for pa_subtable in table_iter(self.data, batch_size=batch_size, drop_last_batch=drop_last_batch):
+ formatted_batch = format_table(
+ pa_subtable,
+ range(pa_subtable.num_rows),
+ formatter=formatter,
+ format_columns=self._format_columns,
+ output_all_columns=self._output_all_columns,
+ )
+ yield formatted_batch
+ else:
+ num_rows = self.num_rows if not drop_last_batch else self.num_rows // batch_size * batch_size
+ for i in range(0, num_rows, batch_size):
+ yield self._getitem(
+ slice(i, i + batch_size),
+ )
+
+ def __repr__(self):
+ return f"Dataset({{\n features: {list(self._info.features.keys())},\n num_rows: {self.num_rows}\n}})"
+
+ @property
+ def format(self):
+ return {
+ "type": self._format_type,
+ "format_kwargs": self._format_kwargs,
+ "columns": self.column_names if self._format_columns is None else self._format_columns,
+ "output_all_columns": self._output_all_columns,
+ }
+
+ @contextlib.contextmanager
+ def formatted_as(
+ self,
+ type: Optional[str] = None,
+ columns: Optional[list] = None,
+ output_all_columns: bool = False,
+ **format_kwargs,
+ ):
+ """To be used in a `with` statement. Set `__getitem__` return format (type and columns).
+
+ Args:
+ type (`str`, *optional*):
+ Either output type selected in `[None, 'numpy', 'torch', 'tensorflow', 'jax', 'arrow', 'pandas', 'polars']`.
+ `None` means `__getitem__`` returns python objects (default).
+ columns (`List[str]`, *optional*):
+ Columns to format in the output.
+ `None` means `__getitem__` returns all columns (default).
+ output_all_columns (`bool`, defaults to `False`):
+ Keep un-formatted columns as well in the output (as python objects).
+ **format_kwargs (additional keyword arguments):
+ Keywords arguments passed to the convert function like `np.array`, `torch.tensor` or `tensorflow.ragged.constant`.
+ """
+ old_format_type = self._format_type
+ old_format_kwargs = self._format_kwargs
+ old_format_columns = self._format_columns
+ old_output_all_columns = self._output_all_columns
+ try:
+ self.set_format(type, columns, output_all_columns, **format_kwargs)
+ yield
+ finally:
+ self.set_format(old_format_type, old_format_columns, old_output_all_columns, **old_format_kwargs)
+
+ @fingerprint_transform(inplace=True)
+ def set_format(
+ self,
+ type: Optional[str] = None,
+ columns: Optional[list] = None,
+ output_all_columns: bool = False,
+ **format_kwargs,
+ ):
+ """Set `__getitem__` return format (type and columns). The data formatting is applied on-the-fly.
+ The format `type` (for example "numpy") is used to format batches when using `__getitem__`.
+ It's also possible to use custom transforms for formatting using [`~datasets.Dataset.set_transform`].
+
+ Args:
+ type (`str`, *optional*):
+ Either output type selected in `[None, 'numpy', 'torch', 'tensorflow', 'jax', 'arrow', 'pandas', 'polars']`.
+ `None` means `__getitem__` returns python objects (default).
+ columns (`List[str]`, *optional*):
+ Columns to format in the output.
+ `None` means `__getitem__` returns all columns (default).
+ output_all_columns (`bool`, defaults to `False`):
+ Keep un-formatted columns as well in the output (as python objects).
+ **format_kwargs (additional keyword arguments):
+ Keywords arguments passed to the convert function like `np.array`, `torch.tensor` or `tensorflow.ragged.constant`.
+
+ It is possible to call [`~datasets.Dataset.map`] after calling `set_format`. Since `map` may add new columns, then the list of formatted columns
+ gets updated. In this case, if you apply `map` on a dataset to add a new column, then this column will be formatted as:
+
+ ```
+ new formatted columns = (all columns - previously unformatted columns)
+ ```
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> from transformers import AutoTokenizer
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
+ >>> ds = ds.map(lambda x: tokenizer(x['text'], truncation=True, padding=True), batched=True)
+ >>> ds.set_format(type='numpy', columns=['text', 'label'])
+ >>> ds.format
+ {'type': 'numpy',
+ 'format_kwargs': {},
+ 'columns': ['text', 'label'],
+ 'output_all_columns': False}
+ ```
+ """
+ format_kwargs.update(format_kwargs.pop("format_kwargs", {})) # allow to use self.set_format(**self.format)
+
+ # Check that the format_type and format_kwargs are valid and make it possible to have a Formatter
+ type = get_format_type_from_alias(type)
+ get_formatter(type, features=self._info.features, **format_kwargs)
+
+ # Check filter column
+ if isinstance(columns, str):
+ columns = [columns]
+ if isinstance(columns, tuple):
+ columns = list(columns)
+ if columns is not None:
+ missing_columns = set(columns) - set(self._data.column_names)
+ if missing_columns:
+ raise ValueError(
+ f"Columns {list(missing_columns)} not in the dataset. Current columns in the dataset: {self._data.column_names}"
+ )
+ if columns is not None:
+ columns = columns.copy() # Ensures modifications made to the list after this call don't cause bugs
+
+ self._format_type = type
+ self._format_kwargs = format_kwargs
+ self._format_columns = columns
+ self._output_all_columns = output_all_columns
+ logger.debug(
+ "Set __getitem__(key) output type to %s for %s columns "
+ " (when key is int or slice) and %s output other (un-formatted) columns.",
+ "python objects" if type is None else type,
+ "no" if columns is None else str(columns),
+ "do" if output_all_columns else "don't",
+ )
+
+ def reset_format(self):
+ """Reset `__getitem__` return format to python objects and all columns.
+
+ Same as `self.set_format()`
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> from transformers import AutoTokenizer
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
+ >>> ds = ds.map(lambda x: tokenizer(x['text'], truncation=True, padding=True), batched=True)
+ >>> ds.set_format(type='numpy', columns=['input_ids', 'token_type_ids', 'attention_mask', 'label'])
+ >>> ds.format
+ {'columns': ['input_ids', 'token_type_ids', 'attention_mask', 'label'],
+ 'format_kwargs': {},
+ 'output_all_columns': False,
+ 'type': 'numpy'}
+ >>> ds.reset_format()
+ >>> ds.format
+ {'columns': ['text', 'label', 'input_ids', 'token_type_ids', 'attention_mask'],
+ 'format_kwargs': {},
+ 'output_all_columns': False,
+ 'type': None}
+ ```
+ """
+ self.set_format()
+
+ def set_transform(
+ self,
+ transform: Optional[Callable],
+ columns: Optional[list] = None,
+ output_all_columns: bool = False,
+ ):
+ """Set `__getitem__` return format using this transform. The transform is applied on-the-fly on batches when `__getitem__` is called.
+ As [`~datasets.Dataset.set_format`], this can be reset using [`~datasets.Dataset.reset_format`].
+
+ Args:
+ transform (`Callable`, *optional*):
+ User-defined formatting transform, replaces the format defined by [`~datasets.Dataset.set_format`].
+ A formatting function is a callable that takes a batch (as a `dict`) as input and returns a batch.
+ This function is applied right before returning the objects in `__getitem__`.
+ columns (`List[str]`, *optional*):
+ Columns to format in the output.
+ If specified, then the input batch of the transform only contains those columns.
+ output_all_columns (`bool`, defaults to `False`):
+ Keep un-formatted columns as well in the output (as python objects).
+ If set to True, then the other un-formatted columns are kept with the output of the transform.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> from transformers import AutoTokenizer
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> tokenizer = AutoTokenizer.from_pretrained('bert-base-uncased')
+ >>> def encode(batch):
+ ... return tokenizer(batch['text'], padding=True, truncation=True, return_tensors='pt')
+ >>> ds.set_transform(encode)
+ >>> ds[0]
+ {'attention_mask': tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1]),
+ 'input_ids': tensor([ 101, 29353, 2135, 15102, 1996, 9428, 20868, 2890, 8663, 6895,
+ 20470, 2571, 3663, 2090, 4603, 3017, 3008, 1998, 2037, 24211,
+ 5637, 1998, 11690, 2336, 1012, 102]),
+ 'token_type_ids': tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0])}
+ ```
+ """
+ self.set_format("custom", columns=columns, output_all_columns=output_all_columns, transform=transform)
+
+ def with_format(
+ self,
+ type: Optional[str] = None,
+ columns: Optional[list] = None,
+ output_all_columns: bool = False,
+ **format_kwargs,
+ ):
+ """Set `__getitem__` return format (type and columns). The data formatting is applied on-the-fly.
+ The format `type` (for example "numpy") is used to format batches when using `__getitem__`.
+
+ It's also possible to use custom transforms for formatting using [`~datasets.Dataset.with_transform`].
+
+ Contrary to [`~datasets.Dataset.set_format`], `with_format` returns a new [`Dataset`] object.
+
+ Args:
+ type (`str`, *optional*):
+ Either output type selected in `[None, 'numpy', 'torch', 'tensorflow', 'jax', 'arrow', 'pandas', 'polars']`.
+ `None` means `__getitem__` returns python objects (default).
+ columns (`List[str]`, *optional*):
+ Columns to format in the output.
+ `None` means `__getitem__` returns all columns (default).
+ output_all_columns (`bool`, defaults to `False`):
+ Keep un-formatted columns as well in the output (as python objects).
+ **format_kwargs (additional keyword arguments):
+ Keywords arguments passed to the convert function like `np.array`, `torch.tensor` or `tensorflow.ragged.constant`.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> from transformers import AutoTokenizer
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
+ >>> ds = ds.map(lambda x: tokenizer(x['text'], truncation=True, padding=True), batched=True)
+ >>> ds.format
+ {'columns': ['text', 'label', 'input_ids', 'token_type_ids', 'attention_mask'],
+ 'format_kwargs': {},
+ 'output_all_columns': False,
+ 'type': None}
+ >>> ds = ds.with_format("torch")
+ >>> ds.format
+ {'columns': ['text', 'label', 'input_ids', 'token_type_ids', 'attention_mask'],
+ 'format_kwargs': {},
+ 'output_all_columns': False,
+ 'type': 'torch'}
+ >>> ds[0]
+ {'text': 'compassionately explores the seemingly irreconcilable situation between conservative christian parents and their estranged gay and lesbian children .',
+ 'label': tensor(1),
+ 'input_ids': tensor([ 101, 18027, 16310, 16001, 1103, 9321, 178, 11604, 7235, 6617,
+ 1742, 2165, 2820, 1206, 6588, 22572, 12937, 1811, 2153, 1105,
+ 1147, 12890, 19587, 6463, 1105, 15026, 1482, 119, 102, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0]),
+ 'token_type_ids': tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]),
+ 'attention_mask': tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])}
+ ```
+ """
+ dataset = copy.deepcopy(self)
+ dataset.set_format(type=type, columns=columns, output_all_columns=output_all_columns, **format_kwargs)
+ return dataset
+
+ def with_transform(
+ self,
+ transform: Optional[Callable],
+ columns: Optional[list] = None,
+ output_all_columns: bool = False,
+ ):
+ """Set `__getitem__` return format using this transform. The transform is applied on-the-fly on batches when `__getitem__` is called.
+
+ As [`~datasets.Dataset.set_format`], this can be reset using [`~datasets.Dataset.reset_format`].
+
+ Contrary to [`~datasets.Dataset.set_transform`], `with_transform` returns a new [`Dataset`] object.
+
+ Args:
+ transform (`Callable`, `optional`):
+ User-defined formatting transform, replaces the format defined by [`~datasets.Dataset.set_format`].
+ A formatting function is a callable that takes a batch (as a `dict`) as input and returns a batch.
+ This function is applied right before returning the objects in `__getitem__`.
+ columns (`List[str]`, `optional`):
+ Columns to format in the output.
+ If specified, then the input batch of the transform only contains those columns.
+ output_all_columns (`bool`, defaults to `False`):
+ Keep un-formatted columns as well in the output (as python objects).
+ If set to `True`, then the other un-formatted columns are kept with the output of the transform.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> from transformers import AutoTokenizer
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
+ >>> def encode(example):
+ ... return tokenizer(example["text"], padding=True, truncation=True, return_tensors='pt')
+ >>> ds = ds.with_transform(encode)
+ >>> ds[0]
+ {'attention_mask': tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1]),
+ 'input_ids': tensor([ 101, 18027, 16310, 16001, 1103, 9321, 178, 11604, 7235, 6617,
+ 1742, 2165, 2820, 1206, 6588, 22572, 12937, 1811, 2153, 1105,
+ 1147, 12890, 19587, 6463, 1105, 15026, 1482, 119, 102]),
+ 'token_type_ids': tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0])}
+ ```
+ """
+ dataset = copy.deepcopy(self)
+ dataset.set_transform(transform=transform, columns=columns, output_all_columns=output_all_columns)
+ return dataset
+
+ def _getitem(self, key: Union[int, slice, str, ListLike[int]], **kwargs) -> Union[dict, list]:
+ """
+ Can be used to index columns (by string names) or rows (by integer, slice, or list-like of integer indices)
+ """
+ if isinstance(key, bool):
+ raise TypeError("dataset index must be int, str, slice or collection of int, not bool")
+ format_type = kwargs["format_type"] if "format_type" in kwargs else self._format_type
+ format_columns = kwargs["format_columns"] if "format_columns" in kwargs else self._format_columns
+ output_all_columns = (
+ kwargs["output_all_columns"] if "output_all_columns" in kwargs else self._output_all_columns
+ )
+ format_kwargs = kwargs["format_kwargs"] if "format_kwargs" in kwargs else self._format_kwargs
+ format_kwargs = format_kwargs if format_kwargs is not None else {}
+ formatter = get_formatter(format_type, features=self._info.features, **format_kwargs)
+ pa_subtable = query_table(self._data, key, indices=self._indices)
+ formatted_output = format_table(
+ pa_subtable, key, formatter=formatter, format_columns=format_columns, output_all_columns=output_all_columns
+ )
+ return formatted_output
+
+ @overload
+ def __getitem__(self, key: Union[int, slice, Iterable[int]]) -> dict: # noqa: F811
+ ...
+
+ @overload
+ def __getitem__(self, key: str) -> list: # noqa: F811
+ ...
+
+ def __getitem__(self, key): # noqa: F811
+ """Can be used to index columns (by string names) or rows (by integer index or iterable of indices or bools)."""
+ if isinstance(key, str):
+ if self._format_type is None or self._format_type not in ("arrow", "pandas", "polars"):
+ return Column(self, key)
+ return self._getitem(key)
+
+ def __getitems__(self, keys: list) -> list:
+ """Can be used to get a batch using a list of integers indices."""
+ batch = self.__getitem__(keys)
+ n_examples = len(batch[next(iter(batch))])
+ return [{col: array[i] for col, array in batch.items()} for i in range(n_examples)]
+
+ def cleanup_cache_files(self) -> int:
+ """Clean up all cache files in the dataset cache directory, excepted the currently used cache file if there is
+ one.
+
+ Be careful when running this command that no other process is currently using other cache files.
+
+ Returns:
+ `int`: Number of removed files.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> ds.cleanup_cache_files()
+ 10
+ ```
+ """
+ current_cache_files = [os.path.abspath(cache_file["filename"]) for cache_file in self.cache_files]
+ if not current_cache_files:
+ return 0
+ cache_directory = os.path.dirname(current_cache_files[0])
+ logger.info(f"Listing files in {cache_directory}")
+ files: list[str] = os.listdir(cache_directory)
+ files_to_remove = []
+ for f_name in files:
+ full_name = os.path.abspath(os.path.join(cache_directory, f_name))
+ if f_name.startswith("cache-") and f_name.endswith(".arrow"):
+ if full_name in current_cache_files:
+ logger.info(f"Keeping currently used cache file at {full_name}")
+ continue
+ files_to_remove.append(full_name)
+ for file_path in files_to_remove:
+ logger.info(f"Removing {file_path}")
+ os.remove(file_path)
+ return len(files_to_remove)
+
+ def _get_cache_file_path(self, fingerprint):
+ if is_caching_enabled() and self.cache_files:
+ cache_file_name = "cache-" + fingerprint + ".arrow"
+ cache_directory = os.path.dirname(self.cache_files[0]["filename"])
+ else:
+ cache_file_name = "cache-" + generate_random_fingerprint() + ".arrow"
+ cache_directory = get_temporary_cache_files_directory()
+ cache_file_path = os.path.join(cache_directory, cache_file_name)
+ return cache_file_path
+
+ @transmit_format
+ def map(
+ self,
+ function: Optional[Callable] = None,
+ with_indices: bool = False,
+ with_rank: bool = False,
+ input_columns: Optional[Union[str, list[str]]] = None,
+ batched: bool = False,
+ batch_size: Optional[int] = 1000,
+ drop_last_batch: bool = False,
+ remove_columns: Optional[Union[str, list[str]]] = None,
+ keep_in_memory: bool = False,
+ load_from_cache_file: Optional[bool] = None,
+ cache_file_name: Optional[str] = None,
+ writer_batch_size: Optional[int] = 1000,
+ features: Optional[Features] = None,
+ disable_nullable: bool = False,
+ fn_kwargs: Optional[dict] = None,
+ num_proc: Optional[int] = None,
+ suffix_template: str = "_{rank:05d}_of_{num_proc:05d}",
+ new_fingerprint: Optional[str] = None,
+ desc: Optional[str] = None,
+ try_original_type: Optional[bool] = True,
+ ) -> "Dataset":
+ """
+ Apply a function to all the examples in the table (individually or in batches) and update the table.
+ If your function returns a column that already exists, then it overwrites it.
+
+ You can specify whether the function should be batched or not with the `batched` parameter:
+
+ - If batched is `False`, then the function takes 1 example in and should return 1 example.
+ An example is a dictionary, e.g. `{"text": "Hello there !"}`.
+ - If batched is `True` and `batch_size` is 1, then the function takes a batch of 1 example as input and can return a batch with 1 or more examples.
+ A batch is a dictionary, e.g. a batch of 1 example is `{"text": ["Hello there !"]}`.
+ - If batched is `True` and `batch_size` is `n > 1`, then the function takes a batch of `n` examples as input and can return a batch with `n` examples, or with an arbitrary number of examples.
+ Note that the last batch may have less than `n` examples.
+ A batch is a dictionary, e.g. a batch of `n` examples is `{"text": ["Hello there !"] * n}`.
+
+ If the function is asynchronous, then `map` will run your function in parallel, with up to one thousand simultaneous calls.
+ It is recommended to use a `asyncio.Semaphore` in your function if you want to set a maximum number of operations that can run at the same time.
+
+ Args:
+ function (`Callable`): Function with one of the following signatures:
+
+ - `function(example: Dict[str, Any]) -> Dict[str, Any]` if `batched=False` and `with_indices=False` and `with_rank=False`
+ - `function(example: Dict[str, Any], *extra_args) -> Dict[str, Any]` if `batched=False` and `with_indices=True` and/or `with_rank=True` (one extra arg for each)
+ - `function(batch: Dict[str, List]) -> Dict[str, List]` if `batched=True` and `with_indices=False` and `with_rank=False`
+ - `function(batch: Dict[str, List], *extra_args) -> Dict[str, List]` if `batched=True` and `with_indices=True` and/or `with_rank=True` (one extra arg for each)
+
+ For advanced usage, the function can also return a `pyarrow.Table`.
+ If the function is asynchronous, then `map` will run your function in parallel.
+ Moreover if your function returns nothing (`None`), then `map` will run your function and return the dataset unchanged.
+ If no function is provided, default to identity function: `lambda x: x`.
+ with_indices (`bool`, defaults to `False`):
+ Provide example indices to `function`. Note that in this case the
+ signature of `function` should be `def function(example, idx[, rank]): ...`.
+ with_rank (`bool`, defaults to `False`):
+ Provide process rank to `function`. Note that in this case the
+ signature of `function` should be `def function(example[, idx], rank): ...`.
+ input_columns (`Optional[Union[str, List[str]]]`, defaults to `None`):
+ The columns to be passed into `function`
+ as positional arguments. If `None`, a `dict` mapping to all formatted columns is passed as one argument.
+ batched (`bool`, defaults to `False`):
+ Provide batch of examples to `function`.
+ batch_size (`int`, *optional*, defaults to `1000`):
+ Number of examples per batch provided to `function` if `batched=True`.
+ If `batch_size <= 0` or `batch_size == None`, provide the full dataset as a single batch to `function`.
+ drop_last_batch (`bool`, defaults to `False`):
+ Whether a last batch smaller than the batch_size should be
+ dropped instead of being processed by the function.
+ remove_columns (`Optional[Union[str, List[str]]]`, defaults to `None`):
+ Remove a selection of columns while doing the mapping.
+ Columns will be removed before updating the examples with the output of `function`, i.e. if `function` is adding
+ columns with names in `remove_columns`, these columns will be kept.
+ keep_in_memory (`bool`, defaults to `False`):
+ Keep the dataset in memory instead of writing it to a cache file.
+ load_from_cache_file (`Optional[bool]`, defaults to `True` if caching is enabled):
+ If a cache file storing the current computation from `function`
+ can be identified, use it instead of recomputing.
+ cache_file_name (`str`, *optional*, defaults to `None`):
+ Provide the name of a path for the cache file. It is used to store the
+ results of the computation instead of the automatically generated cache file name.
+ writer_batch_size (`int`, defaults to `1000`):
+ Number of rows per write operation for the cache file writer.
+ This value is a good trade-off between memory usage during the processing, and processing speed.
+ Higher value makes the processing do fewer lookups, lower value consume less temporary memory while running `map`.
+ features (`Optional[datasets.Features]`, defaults to `None`):
+ Use a specific Features to store the cache file
+ instead of the automatically generated one.
+ disable_nullable (`bool`, defaults to `False`):
+ Disallow null values in the table.
+ fn_kwargs (`Dict`, *optional*, defaults to `None`):
+ Keyword arguments to be passed to `function`.
+ num_proc (`int`, *optional*, defaults to `None`):
+ The number of processes to use for multiprocessing.
+ - If `None` or `0`, no multiprocessing is used and the operation runs in the main process.
+ - If greater than `1`, one or multiple worker processes are used to process data in parallel.
+ Note: The function passed to `map()` must be picklable for multiprocessing to work correctly
+ (i.e., prefer functions defined at the top level of a module, not inside another function or class).
+ suffix_template (`str`):
+ If `cache_file_name` is specified, then this suffix
+ will be added at the end of the base name of each. Defaults to `"_{rank:05d}_of_{num_proc:05d}"`. For example, if `cache_file_name` is "processed.arrow", then for
+ `rank=1` and `num_proc=4`, the resulting file would be `"processed_00001_of_00004.arrow"` for the default suffix.
+ new_fingerprint (`str`, *optional*, defaults to `None`):
+ The new fingerprint of the dataset after transform.
+ If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments.
+ desc (`str`, *optional*, defaults to `None`):
+ Meaningful description to be displayed alongside with the progress bar while mapping examples.
+ try_original_type (`Optional[bool]`, defaults to `True`):
+ Try to keep the types of the original columns (e.g. int32 -> int32).
+ Set to False if you want to always infer new types.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> def add_prefix(example):
+ ... example["text"] = "Review: " + example["text"]
+ ... return example
+ >>> ds = ds.map(add_prefix)
+ >>> ds[0:3]["text"]
+ ['Review: compassionately explores the seemingly irreconcilable situation between conservative christian parents and their estranged gay and lesbian children .',
+ 'Review: the soundtrack alone is worth the price of admission .',
+ 'Review: rodriguez does a splendid job of racial profiling hollywood style--casting excellent latin actors of all ages--a trend long overdue .']
+
+ # process a batch of examples
+ >>> ds = ds.map(lambda example: tokenizer(example["text"]), batched=True)
+ # set number of processors
+ >>> ds = ds.map(add_prefix, num_proc=4)
+ ```
+ """
+ if keep_in_memory and cache_file_name is not None:
+ raise ValueError("Please use either `keep_in_memory` or `cache_file_name` but not both.")
+
+ if num_proc == 0:
+ num_proc = None
+ elif num_proc is not None and num_proc < 0:
+ raise ValueError("num_proc must be >= 0 or None.")
+
+ string_formatter = string.Formatter()
+ fields = {field_name for _, field_name, _, _ in string_formatter.parse(suffix_template) if field_name}
+ if fields != {"rank", "num_proc"}:
+ raise ValueError(f"suffix_template must contain exactly the fields 'rank' and 'num_proc', got: {fields}")
+
+ # If the array is empty we do nothing (but we make sure to handle an empty indices mapping and remove the requested columns anyway)
+ if len(self) == 0:
+ if self._indices is not None: # empty indices mapping
+ self = Dataset(
+ self.data.slice(0, 0),
+ info=self.info.copy(),
+ split=self.split,
+ fingerprint=new_fingerprint,
+ )
+ if remove_columns:
+ return self.remove_columns(remove_columns)
+ else:
+ return self
+
+ if function is None:
+ function = lambda x: x # noqa: E731
+
+ if isinstance(input_columns, str):
+ input_columns = [input_columns]
+
+ if input_columns is not None:
+ missing_columns = set(input_columns) - set(self._data.column_names)
+ if missing_columns:
+ raise ValueError(
+ f"Input column {list(missing_columns)} not in the dataset. Current columns in the dataset: {self._data.column_names}"
+ )
+
+ if isinstance(remove_columns, str):
+ remove_columns = [remove_columns]
+
+ if remove_columns is not None:
+ missing_columns = set(remove_columns) - set(self._data.column_names)
+ if missing_columns:
+ raise ValueError(
+ f"Column to remove {list(missing_columns)} not in the dataset. Current columns in the dataset: {self._data.column_names}"
+ )
+
+ load_from_cache_file = load_from_cache_file if load_from_cache_file is not None else is_caching_enabled()
+
+ if fn_kwargs is None:
+ fn_kwargs = {}
+
+ if features is not None:
+ features = _fix_for_backward_compatible_features(features)
+
+ if num_proc is not None and num_proc > len(self):
+ num_proc = len(self)
+ logger.warning(
+ f"num_proc must be <= {len(self)}. Reducing num_proc to {num_proc} for dataset of size {len(self)}."
+ )
+
+ dataset_kwargs = {
+ "shard": self,
+ "function": function,
+ "with_indices": with_indices,
+ "with_rank": with_rank,
+ "input_columns": input_columns,
+ "batched": batched,
+ "batch_size": batch_size,
+ "drop_last_batch": drop_last_batch,
+ "remove_columns": remove_columns,
+ "keep_in_memory": keep_in_memory,
+ "writer_batch_size": writer_batch_size,
+ "features": features,
+ "disable_nullable": disable_nullable,
+ "fn_kwargs": fn_kwargs,
+ "try_original_type": try_original_type,
+ }
+
+ if new_fingerprint is None:
+ # we create a unique hash from the function,
+ # current dataset file and the mapping args
+ transform = format_transform_for_fingerprint(Dataset._map_single)
+ kwargs_for_fingerprint = format_kwargs_for_fingerprint(Dataset._map_single, (), dataset_kwargs)
+ kwargs_for_fingerprint["fingerprint_name"] = "new_fingerprint"
+ new_fingerprint = update_fingerprint(self._fingerprint, transform, kwargs_for_fingerprint)
+ else:
+ validate_fingerprint(new_fingerprint)
+ dataset_kwargs["new_fingerprint"] = new_fingerprint
+
+ if self.cache_files:
+ if cache_file_name is None:
+ cache_file_name = self._get_cache_file_path(new_fingerprint)
+ dataset_kwargs["cache_file_name"] = cache_file_name
+
+ if cache_file_name is not None:
+ cache_file_prefix, cache_file_ext = os.path.splitext(cache_file_name)
+ if not cache_file_ext:
+ raise ValueError(f"Expected cache_file_name to have an extension, but got: {cache_file_name}")
+ else:
+ cache_file_prefix = cache_file_ext = None
+
+ def load_processed_shard_from_cache(shard_kwargs: dict[str, Any]) -> Dataset:
+ """Load a processed shard from cache if it exists, otherwise throw an error."""
+ shard = shard_kwargs["shard"]
+ # Check if we've already cached this computation (indexed by a hash)
+ if shard_kwargs["cache_file_name"] is not None:
+ if os.path.exists(shard_kwargs["cache_file_name"]) and load_from_cache_file:
+ info = shard.info.copy()
+ info.features = features
+ return Dataset.from_file(shard_kwargs["cache_file_name"], info=info, split=shard.split)
+ raise NonExistentDatasetError
+
+ existing_cache_file_map: dict[int, list[str]] = defaultdict(list)
+ if cache_file_name is not None:
+ if os.path.exists(cache_file_name):
+ existing_cache_file_map[1] = [cache_file_name]
+
+ assert cache_file_prefix is not None and cache_file_ext is not None
+ cache_file_with_suffix_pattern = cache_file_prefix + suffix_template + cache_file_ext
+
+ for cache_file in glob.iglob(f"{cache_file_prefix}*{cache_file_ext}"):
+ suffix_variable_map = string_to_dict(
+ Path(cache_file).as_posix(), Path(cache_file_with_suffix_pattern).as_posix()
+ )
+ if suffix_variable_map is not None:
+ file_num_proc = int(suffix_variable_map["num_proc"])
+ existing_cache_file_map[file_num_proc].append(cache_file)
+
+ num_shards = num_proc or 1
+ if existing_cache_file_map:
+ # to avoid remapping when a different num_proc is given than when originally cached, update num_shards to
+ # what was used originally
+
+ def select_existing_cache_files(mapped_num_proc: int) -> tuple[float, ...]:
+ percent_missing = (mapped_num_proc - len(existing_cache_file_map[mapped_num_proc])) / mapped_num_proc
+ num_shards_diff = abs(mapped_num_proc - num_shards)
+ return (
+ percent_missing, # choose the most complete set of existing cache files
+ num_shards_diff, # then choose the mapped_num_proc closest to the current num_proc
+ mapped_num_proc, # finally, choose whichever mapped_num_proc is lower
+ )
+
+ num_shards = min(existing_cache_file_map, key=select_existing_cache_files)
+
+ existing_cache_files = existing_cache_file_map[num_shards]
+
+ def format_cache_file_name(
+ cache_file_name: Optional[str],
+ rank: Union[int, Literal["*"]], # noqa: F722
+ ) -> Optional[str]:
+ if not cache_file_name:
+ return cache_file_name
+
+ assert cache_file_prefix is not None and cache_file_ext is not None
+
+ if isinstance(rank, int):
+ cache_file_name = (
+ cache_file_prefix + suffix_template.format(rank=rank, num_proc=num_shards) + cache_file_ext
+ )
+ if not os.path.exists(cache_file_name):
+ process_name = (
+ "Main process" if num_proc is None or num_proc == 1 else f"Process #{rank % num_shards + 1}"
+ )
+ logger.info(f"{process_name} will write at {cache_file_name}")
+ else:
+ # TODO: this assumes the format_spec of rank in suffix_template
+ cache_file_name = (
+ cache_file_prefix
+ + suffix_template.replace("{rank:05d}", "{rank}").format(rank=rank, num_proc=num_shards)
+ + cache_file_ext
+ )
+ return cache_file_name
+
+ def format_new_fingerprint(new_fingerprint: str, rank: int) -> str:
+ new_fingerprint = new_fingerprint + suffix_template.format(rank=rank, num_proc=num_shards)
+ validate_fingerprint(new_fingerprint)
+ return new_fingerprint
+
+ if num_proc is not None and num_proc >= 1:
+ prev_env = deepcopy(os.environ)
+ # check if parallelism if off
+ # from https://github.com/huggingface/tokenizers/blob/bb668bc439dc34389b71dbb8ce0c597f15707b53/tokenizers/src/utils/parallelism.rs#L22
+ if prev_env.get("TOKENIZERS_PARALLELISM", "false").lower() not in (
+ "",
+ "off",
+ "false",
+ "f",
+ "no",
+ "n",
+ "0",
+ ):
+ logger.warning("Setting TOKENIZERS_PARALLELISM=false for forked processes.")
+ os.environ["TOKENIZERS_PARALLELISM"] = "false"
+ else:
+ prev_env = os.environ
+
+ kwargs_per_job: list[Optional[dict[str, Any]]]
+ if num_shards == 1:
+ shards = [self]
+ kwargs_per_job = [dataset_kwargs]
+ else:
+ shards = [
+ self.shard(num_shards=num_shards, index=rank, contiguous=True, keep_in_memory=keep_in_memory)
+ for rank in range(num_shards)
+ ]
+ kwargs_per_job = [
+ {
+ **dataset_kwargs,
+ "shard": shards[rank],
+ "cache_file_name": format_cache_file_name(cache_file_name, rank),
+ "rank": rank,
+ "offset": sum(len(s) for s in shards[:rank]),
+ "new_fingerprint": format_new_fingerprint(new_fingerprint, rank),
+ }
+ for rank in range(num_shards)
+ ]
+
+ transformed_shards: list[Optional[Dataset]] = [None] * num_shards
+ for rank in range(num_shards):
+ try:
+ job_kwargs = kwargs_per_job[rank]
+ assert job_kwargs is not None
+ transformed_shards[rank] = load_processed_shard_from_cache(job_kwargs)
+ kwargs_per_job[rank] = None
+ except NonExistentDatasetError:
+ pass
+
+ if unprocessed_kwargs_per_job := [kwargs for kwargs in kwargs_per_job if kwargs is not None]:
+ if len(unprocessed_kwargs_per_job) != num_shards:
+ logger.info(
+ f"Reprocessing {len(unprocessed_kwargs_per_job)}/{num_shards} shards because some of them were "
+ "missing from the cache."
+ )
+
+ pbar_total = len(self)
+ pbar_initial = len(existing_cache_files) * pbar_total // num_shards
+ if batched and drop_last_batch:
+ batch_size = batch_size or 1
+ pbar_initial = pbar_initial // num_shards // batch_size * num_shards * batch_size
+ pbar_total = pbar_total // num_shards // batch_size * num_shards * batch_size
+
+ with hf_tqdm(
+ unit=" examples",
+ initial=pbar_initial,
+ total=pbar_total,
+ desc=(desc or "Map") + (f" (num_proc={num_proc})" if num_proc is not None and num_proc >= 1 else ""),
+ ) as pbar:
+ shards_done = 0
+
+ def check_if_shard_done(rank: Optional[int], done: bool, content: Union[Dataset, int]) -> None:
+ nonlocal shards_done
+ if done:
+ shards_done += 1
+ logger.debug(f"Finished processing shard number {rank} of {num_shards}.")
+ assert isinstance(content, Dataset)
+ transformed_shards[rank or 0] = content
+ else:
+ assert isinstance(content, int)
+ pbar.update(content)
+
+ if num_proc is not None and num_proc >= 1:
+ with Pool(num_proc) as pool:
+ os.environ = prev_env
+ logger.info(f"Spawning {num_proc} processes")
+
+ for rank, done, content in iflatmap_unordered(
+ pool, Dataset._map_single, kwargs_iterable=unprocessed_kwargs_per_job
+ ):
+ check_if_shard_done(rank, done, content)
+
+ pool.close()
+ pool.join()
+ else:
+ for unprocessed_kwargs in unprocessed_kwargs_per_job:
+ for rank, done, content in Dataset._map_single(**unprocessed_kwargs):
+ check_if_shard_done(rank, done, content)
+
+ # Avoids PermissionError on Windows (the error: https://github.com/huggingface/datasets/actions/runs/4026734820/jobs/6921621805)
+ for job_kwargs in unprocessed_kwargs_per_job:
+ if "shard" in job_kwargs:
+ del job_kwargs["shard"]
+ else:
+ logger.info(f"Loading cached processed dataset at {format_cache_file_name(cache_file_name, '*')}")
+
+ all_transformed_shards = [shard for shard in transformed_shards if shard is not None]
+ if len(transformed_shards) != len(all_transformed_shards):
+ raise ValueError(
+ f"Failed to retrieve results from map: result list {transformed_shards} still contains None - "
+ "at least one worker failed to return its results"
+ )
+
+ if num_shards == 1:
+ result = all_transformed_shards[0]
+ else:
+ logger.info(f"Concatenating {num_shards} shards")
+ result = _concatenate_map_style_datasets(all_transformed_shards)
+
+ # update fingerprint if the dataset changed
+ result._fingerprint = (
+ new_fingerprint
+ if any(
+ transformed_shard._fingerprint != shard._fingerprint
+ for transformed_shard, shard in zip(all_transformed_shards, shards)
+ )
+ else self._fingerprint
+ )
+
+ return result
+
+ @staticmethod
+ def _map_single(
+ shard: "Dataset",
+ function: Optional[Callable] = None,
+ with_indices: bool = False,
+ with_rank: bool = False,
+ input_columns: Optional[list[str]] = None,
+ batched: bool = False,
+ batch_size: Optional[int] = 1000,
+ drop_last_batch: bool = False,
+ remove_columns: Optional[list[str]] = None,
+ keep_in_memory: bool = False,
+ cache_file_name: Optional[str] = None,
+ writer_batch_size: Optional[int] = 1000,
+ features: Optional[Features] = None,
+ disable_nullable: bool = False,
+ fn_kwargs: Optional[dict] = None,
+ new_fingerprint: Optional[str] = None,
+ rank: Optional[int] = None,
+ offset: int = 0,
+ try_original_type: Optional[bool] = True,
+ ) -> Iterable[tuple[Optional[int], bool, Union[int, "Dataset"]]]:
+ """Apply a function to all the elements in the table (individually or in batches)
+ and update the table (if function does update examples).
+
+ Args:
+ shard (`datasets.Dataset`): Dataset to map the transform on.
+ function (`Callable`): with one of the following signature:
+ - `function(example: Dict[str, Any]) -> Dict[str, Any]` if `batched=False` and `with_indices=False` and `with_rank=False`
+ - `function(example: Dict[str, Any], *extra_args) -> Dict[str, Any]` if `batched=False` and `with_indices=True` and/or `with_rank=True` (one extra arg for each)
+ - `function(batch: Dict[str, List]) -> Dict[str, List]` if `batched=True` and `with_indices=False` and `with_rank=False`
+ - `function(batch: Dict[str, List], *extra_args) -> Dict[str, List]` if `batched=True` and `with_indices=True` and/or `with_rank=True` (one extra arg for each)
+
+ For advanced usage, the function can also return a `pyarrow.Table`.
+ Moreover if your function returns nothing (`None`), then `map` will run your function and return the dataset unchanged.
+ If no function is provided, default to identity function: lambda x: x
+ with_indices (`bool`, defaults to `False`): Provide example indices to `function`. Note that in this case the signature of `function` should be `def function(example, idx[, rank]): ...`.
+ with_rank (`bool`, default `False`): Provide process rank to `function`. Note that in this case the signature of `function` should be `def function(example[, idx], rank): ...`.
+ input_columns (`Optional[List[str]]`, defaults to `None`): The columns to be passed into `function` as
+ positional arguments. If `None`, a dict mapping to all formatted columns is passed as one argument.
+ batched (`bool`, defaults to `False`): Provide batch of examples to `function`
+ batch_size (`int`, optional, defaults to `1000`): Number of examples per batch provided to `function` if `batched=True`
+ `batch_size <= 0` or `batch_size == None`: Provide the full dataset as a single batch to `function`
+ drop_last_batch (`bool`, default: `False`): Whether a last batch smaller than the batch_size should be
+ dropped instead of being processed by the function.
+ remove_columns (`Optional[List[str]]`, defaults to `None`): Remove a selection of columns while doing the mapping.
+ Columns will be removed before updating the examples with the output of `function`, i.e. if `function` is adding
+ columns with names in `remove_columns`, these columns will be kept.
+ keep_in_memory (`bool`, defaults to `False`): Keep the dataset in memory instead of writing it to a cache file.
+ cache_file_name (`str`, optional, defaults to `None`): Provide the name of a path for the cache file. It is used to store the
+ results of the computation instead of the automatically generated cache file name.
+ writer_batch_size (`int`, default `1000`): Number of rows per write operation for the cache file writer.
+ This value is a good trade-off between memory usage during the processing, and processing speed.
+ Higher value makes the processing do fewer lookups, lower value consume less temporary memory while running `.map()`.
+ features (`Optional[datasets.Features]`, defaults to `None`): Use a specific Features to store the cache file
+ instead of the automatically generated one.
+ disable_nullable (`bool`, defaults to `False`): Disallow null values in the table.
+ fn_kwargs (`Dict`, optional, defaults to `None`): Keyword arguments to be passed to `function`
+ new_fingerprint (`str`, optional, defaults to `None`): the new fingerprint of the dataset after transform.
+ If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments
+ rank: (`int`, optional, defaults to `None`): If specified, this is the process rank when doing multiprocessing
+ offset: (`int`, defaults to 0): If specified, this is an offset applied to the indices passed to `function` if `with_indices=True`.
+ try_original_type: (`Optional[bool]`, defaults to `True`):
+ Try to keep the types of the original columns (e.g. int32 -> int32).
+ Set to False if you want to always infer new types.
+ """
+ if fn_kwargs is None:
+ fn_kwargs = {}
+
+ # If we do batch computation but no batch size is provided, default to the full dataset
+ if batched and (batch_size is None or batch_size <= 0):
+ batch_size = shard.num_rows
+
+ # We set this variable to True after processing the first example/batch in
+ # `apply_function_on_filtered_inputs` if the map function returns a dict.
+ # If set to False, no new arrow table will be created
+
+ update_data = None
+
+ format_kwargs = shard._format_kwargs.copy()
+ # Lazy formatting is only available for the default format (None/python)
+ if not input_columns and shard._format_type is None:
+ format_kwargs["lazy"] = True
+ input_formatter = get_formatter(
+ shard._format_type,
+ features=shard.features,
+ **format_kwargs,
+ )
+
+ check_same_num_examples = batched and len(shard.list_indexes()) > 0
+
+ def validate_function_output(processed_inputs):
+ """Validate output of the map function."""
+ allowed_processed_inputs_types = (Mapping, pa.Table, pd.DataFrame)
+ if config.POLARS_AVAILABLE and "polars" in sys.modules:
+ import polars as pl
+
+ allowed_processed_inputs_types += (pl.DataFrame,)
+ if processed_inputs is not None and not isinstance(processed_inputs, allowed_processed_inputs_types):
+ raise TypeError(
+ f"Provided `function` which is applied to all elements of table returns a variable of type {type(processed_inputs)}. Make sure provided `function` returns a variable of type `dict` (or a pyarrow table) to update the dataset or `None` if you are only interested in side effects."
+ )
+ if batched and isinstance(processed_inputs, Mapping):
+ allowed_batch_return_types = (list, np.ndarray, pd.Series)
+ if config.POLARS_AVAILABLE and "polars" in sys.modules:
+ import polars as pl
+
+ allowed_batch_return_types += (pl.Series, pl.DataFrame)
+ if config.TF_AVAILABLE and "tensorflow" in sys.modules:
+ import tensorflow as tf
+
+ allowed_batch_return_types += (tf.Tensor,)
+ if config.TORCH_AVAILABLE and "torch" in sys.modules:
+ import torch
+
+ allowed_batch_return_types += (torch.Tensor,)
+ if config.JAX_AVAILABLE and "jax" in sys.modules:
+ import jax.numpy as jnp
+
+ allowed_batch_return_types += (jnp.ndarray,)
+ all_dict_values_are_lists = all(
+ isinstance(value, allowed_batch_return_types) for value in processed_inputs.values()
+ )
+ if all_dict_values_are_lists is False:
+ raise TypeError(
+ f"Provided `function` which is applied to all elements of table returns a `dict` of types {[type(x) for x in processed_inputs.values()]}. When using `batched=True`, make sure provided `function` returns a `dict` of types like `{allowed_batch_return_types}`."
+ )
+
+ def prepare_inputs(pa_inputs, indices, offset=0):
+ """Utility to apply the function on a selection of columns."""
+ inputs = format_table(
+ pa_inputs,
+ 0 if not batched else range(pa_inputs.num_rows),
+ format_columns=input_columns,
+ formatter=input_formatter,
+ )
+ fn_args = [inputs] if input_columns is None else [inputs[col] for col in input_columns]
+ if offset == 0:
+ effective_indices = indices
+ else:
+ effective_indices = [i + offset for i in indices] if isinstance(indices, list) else indices + offset
+ additional_args = ()
+ if with_indices:
+ additional_args += (effective_indices,)
+ if with_rank:
+ additional_args += (rank,)
+ return inputs, fn_args, additional_args, fn_kwargs
+
+ def prepare_outputs(pa_inputs, inputs, processed_inputs):
+ nonlocal update_data
+ if not (update_data := (processed_inputs is not None)):
+ return None
+ if isinstance(processed_inputs, LazyDict):
+ processed_inputs = {
+ k: v for k, v in processed_inputs.data.items() if k not in processed_inputs.keys_to_format
+ }
+ returned_lazy_dict = True
+ else:
+ returned_lazy_dict = False
+ validate_function_output(processed_inputs)
+ if shard._format_type or input_columns:
+ # TODO(QL, MS): ideally the behavior should be the same even if the dataset is formatted (may require major release)
+ inputs_to_merge = dict(zip(pa_inputs.column_names, pa_inputs.itercolumns()))
+ elif isinstance(inputs, LazyDict):
+ inputs_to_merge = {
+ k: (v if k not in inputs.keys_to_format else pa_inputs[k]) for k, v in inputs.data.items()
+ }
+ else:
+ inputs_to_merge = inputs
+ if remove_columns is not None:
+ for column in remove_columns:
+ # `function` can modify input in-place causing column to be already removed.
+ if column in inputs_to_merge:
+ inputs_to_merge.pop(column)
+ if returned_lazy_dict and column in processed_inputs:
+ processed_inputs.pop(column)
+ if check_same_num_examples:
+ input_num_examples = len(pa_inputs)
+ processed_inputs_num_examples = len(processed_inputs[next(iter(processed_inputs.keys()))])
+ if input_num_examples != processed_inputs_num_examples:
+ raise DatasetTransformationNotAllowedError(
+ "Using `.map` in batched mode on a dataset with attached indexes is allowed only if it doesn't create or remove existing examples. You can first run `.drop_index() to remove your index and then re-add it."
+ ) from None
+ if isinstance(inputs, Mapping) and isinstance(processed_inputs, Mapping):
+ # The .map() transform *updates* the dataset:
+ # the output dictionary contains both the the input data and the output data.
+ # The output dictionary may contain Arrow values from `inputs_to_merge` so that we can re-write them efficiently.
+ return {**inputs_to_merge, **processed_inputs}
+ else:
+ return processed_inputs
+
+ def apply_function(pa_inputs, indices, offset=0):
+ """Utility to apply the function on a selection of columns."""
+ inputs, fn_args, additional_args, fn_kwargs = prepare_inputs(pa_inputs, indices, offset=offset)
+ processed_inputs = function(*fn_args, *additional_args, **fn_kwargs)
+ return prepare_outputs(pa_inputs, inputs, processed_inputs)
+
+ async def async_apply_function(pa_inputs, indices, offset=0):
+ """Utility to apply the function on a selection of columns. Same code but async"""
+ inputs, fn_args, additional_args, fn_kwargs = prepare_inputs(pa_inputs, indices, offset=offset)
+ processed_inputs = await function(*fn_args, *additional_args, **fn_kwargs)
+ return prepare_outputs(pa_inputs, inputs, processed_inputs)
+
+ def init_buffer_and_writer():
+ # Prepare output buffer and batched writer in memory or on file if we update the table
+ writer_features = features
+ if writer_features is None:
+ writer_features = shard.features
+ update_features = True
+ else:
+ update_features = False
+ if keep_in_memory or cache_file_name is None:
+ buf_writer = pa.BufferOutputStream()
+ tmp_file = None
+ writer = ArrowWriter(
+ features=writer_features,
+ stream=buf_writer,
+ writer_batch_size=writer_batch_size,
+ update_features=update_features,
+ fingerprint=new_fingerprint,
+ disable_nullable=disable_nullable,
+ )
+ else:
+ buf_writer = None
+ logger.info(f"Caching processed dataset at {cache_file_name}")
+ cache_dir = os.path.dirname(cache_file_name)
+ os.makedirs(cache_dir, exist_ok=True)
+ tmp_file = tempfile.NamedTemporaryFile("wb", dir=cache_dir, delete=False)
+ writer = ArrowWriter(
+ features=writer_features,
+ path=tmp_file.name,
+ writer_batch_size=writer_batch_size,
+ update_features=update_features,
+ fingerprint=new_fingerprint,
+ disable_nullable=disable_nullable,
+ )
+ return buf_writer, writer, tmp_file
+
+ tasks: list[asyncio.Task] = []
+ if inspect.iscoroutinefunction(function):
+ try:
+ loop = asyncio.get_running_loop()
+ except RuntimeError:
+ loop = asyncio.new_event_loop()
+ else:
+ loop = None
+
+ def iter_outputs(shard_iterable):
+ nonlocal tasks, loop
+ if inspect.iscoroutinefunction(function):
+ indices: Union[list[int], list[list[int]]] = []
+ for i, example in shard_iterable:
+ indices.append(i)
+ tasks.append(loop.create_task(async_apply_function(example, i, offset=offset)))
+ # keep the total active tasks under a certain number
+ if len(tasks) >= config.MAX_NUM_RUNNING_ASYNC_MAP_FUNCTIONS_IN_PARALLEL:
+ done, pending = loop.run_until_complete(
+ asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)
+ )
+ while tasks and len(pending) >= config.MAX_NUM_RUNNING_ASYNC_MAP_FUNCTIONS_IN_PARALLEL:
+ done, pending = loop.run_until_complete(
+ asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)
+ )
+ # yield finished tasks
+ while tasks and tasks[0].done():
+ yield indices.pop(0), tasks.pop(0).result()
+ while tasks:
+ yield indices[0], loop.run_until_complete(tasks[0])
+ indices.pop(0), tasks.pop(0)
+ else:
+ for i, example in shard_iterable:
+ yield i, apply_function(example, i, offset=offset)
+
+ num_examples_progress_update = 0
+ # If `update_data` is True after processing the first example/batch, initialize these resources with `init_buffer_and_writer`
+ buf_writer, writer, tmp_file = None, None, None
+
+ # Check if Polars is available and import it if so
+ if config.POLARS_AVAILABLE and "polars" in sys.modules:
+ import polars as pl
+
+ # Optionally initialize the writer as a context manager
+ with contextlib.ExitStack() as stack:
+ try:
+ arrow_formatted_shard = shard.with_format("arrow")
+
+ # Loop over single examples or batches and write to buffer/file if examples are to be updated
+ if not batched:
+ shard_iterable = enumerate(arrow_formatted_shard)
+ else:
+ num_rows = len(shard) if not drop_last_batch else len(shard) // batch_size * batch_size
+ shard_iterable = zip(
+ (list(range(i, min(i + batch_size, num_rows))) for i in range(0, num_rows, batch_size)),
+ arrow_formatted_shard.iter(batch_size, drop_last_batch=drop_last_batch),
+ )
+ if not batched:
+ _time = time.time()
+ for i, example in iter_outputs(shard_iterable):
+ if update_data:
+ if i == 0:
+ buf_writer, writer, tmp_file = init_buffer_and_writer()
+ stack.enter_context(writer)
+ if isinstance(example, pa.Table):
+ writer.write_row(example)
+ elif isinstance(example, pd.DataFrame):
+ writer.write_row(pa.Table.from_pandas(example))
+ elif (
+ config.POLARS_AVAILABLE
+ and "polars" in sys.modules
+ and isinstance(example, pl.DataFrame)
+ ):
+ writer.write_row(example.to_arrow())
+ else:
+ writer.write(example)
+ num_examples_progress_update += 1
+ if time.time() > _time + config.PBAR_REFRESH_TIME_INTERVAL:
+ _time = time.time()
+ yield rank, False, num_examples_progress_update
+ num_examples_progress_update = 0
+ else:
+ _time = time.time()
+ for i, batch in iter_outputs(shard_iterable):
+ num_examples_in_batch = len(i)
+ if update_data:
+ if i and i[0] == 0:
+ buf_writer, writer, tmp_file = init_buffer_and_writer()
+ stack.enter_context(writer)
+ if isinstance(batch, pa.Table):
+ writer.write_table(batch)
+ elif isinstance(batch, pd.DataFrame):
+ writer.write_table(pa.Table.from_pandas(batch))
+ elif (
+ config.POLARS_AVAILABLE and "polars" in sys.modules and isinstance(batch, pl.DataFrame)
+ ):
+ writer.write_table(batch.to_arrow())
+ else:
+ writer.write_batch(batch, try_original_type=try_original_type)
+ num_examples_progress_update += num_examples_in_batch
+ if time.time() > _time + config.PBAR_REFRESH_TIME_INTERVAL:
+ _time = time.time()
+ yield rank, False, num_examples_progress_update
+ num_examples_progress_update = 0
+ if update_data and writer is not None:
+ writer.finalize() # close_stream=bool(buf_writer is None)) # We only close if we are writing in a file
+ except (Exception, KeyboardInterrupt):
+ yield rank, False, num_examples_progress_update
+ if update_data:
+ if writer is not None:
+ writer.finalize()
+ if tmp_file is not None:
+ tmp_file.close()
+ if os.path.exists(tmp_file.name):
+ os.remove(tmp_file.name)
+ if loop:
+ logger.debug(f"Canceling {len(tasks)} async tasks.")
+ for task in tasks:
+ task.cancel(msg="KeyboardInterrupt")
+ try:
+ loop.run_until_complete(asyncio.gather(*tasks))
+ except (asyncio.CancelledError, ValueError):
+ logger.debug("Tasks canceled.")
+ raise
+
+ yield rank, False, num_examples_progress_update
+ if update_data and tmp_file is not None:
+ tmp_file.close()
+ shutil.move(tmp_file.name, cache_file_name)
+ umask = os.umask(0o666)
+ os.umask(umask)
+ os.chmod(cache_file_name, 0o666 & ~umask)
+
+ if update_data:
+ # Create new Dataset from buffer or file
+ info = shard.info.copy()
+ info.features = writer._features
+ if buf_writer is None:
+ yield rank, True, Dataset.from_file(cache_file_name, info=info, split=shard.split)
+ else:
+ yield rank, True, Dataset.from_buffer(buf_writer.getvalue(), info=info, split=shard.split)
+ else:
+ yield rank, True, shard
+
+ @transmit_format
+ @fingerprint_transform(inplace=False)
+ def batch(
+ self,
+ batch_size: int,
+ drop_last_batch: bool = False,
+ num_proc: Optional[int] = None,
+ new_fingerprint: Optional[str] = None,
+ ) -> "Dataset":
+ """
+ Group samples from the dataset into batches.
+
+ Args:
+ batch_size (`int`):
+ The number of samples in each batch.
+ drop_last_batch (`bool`, defaults to `False`):
+ Whether to drop the last incomplete batch.
+ num_proc (`int`, *optional*, defaults to `None`):
+ Max number of processes when generating cache. Already cached shards are loaded sequentially.
+ new_fingerprint (`str`, *optional*, defaults to `None`):
+ The new fingerprint of the dataset after transform.
+ If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments.
+
+ Returns:
+ [`Dataset`]: A new Dataset where each item is a batch of multiple samples from the original dataset.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train")
+ >>> batched_ds = ds.batch(batch_size=4)
+ >>> batched_ds[0]
+ {'text': ['compassionately explores the seemingly irreconcilable situation...', ...], # 4 items
+ 'label': [1, 1, 1, 1]}
+ ```
+ """
+
+ def batch_fn(example):
+ return {k: [v] for k, v in example.items()}
+
+ return self.map(
+ batch_fn,
+ batched=True,
+ batch_size=batch_size,
+ drop_last_batch=drop_last_batch,
+ num_proc=num_proc,
+ new_fingerprint=new_fingerprint,
+ desc="Batching examples",
+ )
+
+ @transmit_format
+ @fingerprint_transform(
+ inplace=False, ignore_kwargs=["load_from_cache_file", "cache_file_name", "desc"], version="2.0.1"
+ )
+ def filter(
+ self,
+ function: Optional[Callable] = None,
+ with_indices: bool = False,
+ with_rank: bool = False,
+ input_columns: Optional[Union[str, list[str]]] = None,
+ batched: bool = False,
+ batch_size: Optional[int] = 1000,
+ keep_in_memory: bool = False,
+ load_from_cache_file: Optional[bool] = None,
+ cache_file_name: Optional[str] = None,
+ writer_batch_size: Optional[int] = 1000,
+ fn_kwargs: Optional[dict] = None,
+ num_proc: Optional[int] = None,
+ suffix_template: str = "_{rank:05d}_of_{num_proc:05d}",
+ new_fingerprint: Optional[str] = None,
+ desc: Optional[str] = None,
+ ) -> "Dataset":
+ """Apply a filter function to all the elements in the table in batches
+ and update the table so that the dataset only includes examples according to the filter function.
+
+ If the function is asynchronous, then `filter` will run your function in parallel, with up to one thousand simultaneous calls (configurable).
+ It is recommended to use a `asyncio.Semaphore` in your function if you want to set a maximum number of operations that can run at the same time.
+
+ Args:
+ function (`Callable`): Callable with one of the following signatures:
+
+ - `function(example: Dict[str, Any]) -> bool` if `batched=False` and `with_indices=False` and `with_rank=False`
+ - `function(example: Dict[str, Any], *extra_args) -> bool` if `batched=False` and `with_indices=True` and/or `with_rank=True` (one extra arg for each)
+ - `function(batch: Dict[str, List]) -> List[bool]` if `batched=True` and `with_indices=False` and `with_rank=False`
+ - `function(batch: Dict[str, List], *extra_args) -> List[bool]` if `batched=True` and `with_indices=True` and/or `with_rank=True` (one extra arg for each)
+
+ If the function is asynchronous, then `filter` will run your function in parallel.
+ If no function is provided, defaults to an always `True` function: `lambda x: True`.
+ with_indices (`bool`, defaults to `False`):
+ Provide example indices to `function`. Note that in this case the
+ signature of `function` should be `def function(example, idx[, rank]): ...`.
+ with_rank (`bool`, defaults to `False`):
+ Provide process rank to `function`. Note that in this case the
+ signature of `function` should be `def function(example[, idx], rank): ...`.
+ input_columns (`str` or `List[str]`, *optional*):
+ The columns to be passed into `function` as
+ positional arguments. If `None`, a `dict` mapping to all formatted columns is passed as one argument.
+ batched (`bool`, defaults to `False`):
+ Provide batch of examples to `function`.
+ batch_size (`int`, *optional*, defaults to `1000`):
+ Number of examples per batch provided to `function` if
+ `batched = True`. If `batched = False`, one example per batch is passed to `function`.
+ If `batch_size <= 0` or `batch_size == None`, provide the full dataset as a single batch to `function`.
+ keep_in_memory (`bool`, defaults to `False`):
+ Keep the dataset in memory instead of writing it to a cache file.
+ load_from_cache_file (`Optional[bool]`, defaults to `True` if caching is enabled):
+ If a cache file storing the current computation from `function`
+ can be identified, use it instead of recomputing.
+ cache_file_name (`str`, *optional*):
+ Provide the name of a path for the cache file. It is used to store the
+ results of the computation instead of the automatically generated cache file name.
+ writer_batch_size (`int`, defaults to `1000`):
+ Number of rows per write operation for the cache file writer.
+ This value is a good trade-off between memory usage during the processing, and processing speed.
+ Higher value makes the processing do fewer lookups, lower value consume less temporary memory while running `map`.
+ fn_kwargs (`dict`, *optional*):
+ Keyword arguments to be passed to `function`.
+ num_proc (`int`, *optional*, defaults to `None`):
+ The number of processes to use for multiprocessing.
+ - If `None` or `0`, no multiprocessing is used and the operation runs in the main process.
+ - If greater than `1`, one or multiple worker processes are used to process data in parallel.
+ Note: The function passed to `map()` must be picklable for multiprocessing to work correctly
+ (i.e., prefer functions defined at the top level of a module, not inside another function or class).
+ suffix_template (`str`):
+ If `cache_file_name` is specified, then this suffix will be added at the end of the base name of each.
+ For example, if `cache_file_name` is `"processed.arrow"`, then for `rank = 1` and `num_proc = 4`,
+ the resulting file would be `"processed_00001_of_00004.arrow"` for the default suffix (default
+ `_{rank:05d}_of_{num_proc:05d}`).
+ new_fingerprint (`str`, *optional*):
+ The new fingerprint of the dataset after transform.
+ If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments.
+ desc (`str`, *optional*, defaults to `None`):
+ Meaningful description to be displayed alongside with the progress bar while filtering examples.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> ds = ds.filter(lambda x: x["label"] == 1)
+ >>> ds
+ Dataset({
+ features: ['text', 'label'],
+ num_rows: 533
+ })
+ ```
+
+ """
+ if len(self.list_indexes()) > 0:
+ raise DatasetTransformationNotAllowedError(
+ "Using `.filter` on a dataset with attached indexes is not allowed. You can first run `.drop_index() to remove your index and then re-add it.`"
+ )
+
+ if function is None:
+ function = lambda x: True # noqa: E731
+
+ if len(self) == 0:
+ return self
+
+ # We generally batch the underlying map() to get faster throughput,
+ # but in case of async we force batch_size=1 to enable parallelism
+ if inspect.iscoroutinefunction(function) and not batched:
+ batch_size = 1
+
+ indices = self.map(
+ function=partial(
+ async_get_indices_from_mask_function
+ if inspect.iscoroutinefunction(function)
+ else get_indices_from_mask_function,
+ function,
+ batched,
+ with_indices,
+ with_rank,
+ input_columns,
+ self._indices,
+ ),
+ with_indices=True,
+ with_rank=True,
+ features=Features({"indices": Value("uint64")}),
+ batched=True,
+ batch_size=batch_size,
+ remove_columns=self.column_names,
+ keep_in_memory=keep_in_memory,
+ load_from_cache_file=load_from_cache_file,
+ cache_file_name=cache_file_name,
+ writer_batch_size=writer_batch_size,
+ fn_kwargs=fn_kwargs,
+ num_proc=num_proc,
+ suffix_template=suffix_template,
+ new_fingerprint=new_fingerprint,
+ input_columns=input_columns,
+ desc=desc or "Filter",
+ )
+ new_dataset = copy.deepcopy(self)
+ new_dataset._indices = indices.data
+ new_dataset._fingerprint = new_fingerprint
+ return new_dataset
+
+ @transmit_format
+ @fingerprint_transform(inplace=False, ignore_kwargs=["cache_file_name"])
+ def flatten_indices(
+ self,
+ keep_in_memory: bool = False,
+ cache_file_name: Optional[str] = None,
+ writer_batch_size: Optional[int] = 1000,
+ features: Optional[Features] = None,
+ disable_nullable: bool = False,
+ num_proc: Optional[int] = None,
+ new_fingerprint: Optional[str] = None,
+ ) -> "Dataset":
+ """Create and cache a new Dataset by flattening the indices mapping.
+
+ Args:
+ keep_in_memory (`bool`, defaults to `False`):
+ Keep the dataset in memory instead of writing it to a cache file.
+ cache_file_name (`str`, *optional*, default `None`):
+ Provide the name of a path for the cache file. It is used to store the
+ results of the computation instead of the automatically generated cache file name.
+ writer_batch_size (`int`, defaults to `1000`):
+ Number of rows per write operation for the cache file writer.
+ This value is a good trade-off between memory usage during the processing, and processing speed.
+ Higher value makes the processing do fewer lookups, lower value consume less temporary memory while running `map`.
+ features (`Optional[datasets.Features]`, defaults to `None`):
+ Use a specific [`Features`] to store the cache file
+ instead of the automatically generated one.
+ disable_nullable (`bool`, defaults to `False`):
+ Allow null values in the table.
+ num_proc (`int`, optional, default `None`):
+ Max number of processes when generating cache. Already cached shards are loaded sequentially
+ new_fingerprint (`str`, *optional*, defaults to `None`):
+ The new fingerprint of the dataset after transform.
+ If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments
+ """
+
+ return self.map(
+ batched=True, # for speed
+ keep_in_memory=keep_in_memory,
+ cache_file_name=cache_file_name,
+ writer_batch_size=writer_batch_size,
+ features=features,
+ disable_nullable=disable_nullable,
+ new_fingerprint=new_fingerprint,
+ desc="Flattening the indices",
+ num_proc=num_proc,
+ )
+
+ def _new_dataset_with_indices(
+ self,
+ indices_cache_file_name: Optional[str] = None,
+ indices_buffer: Optional[pa.Buffer] = None,
+ fingerprint: Optional[str] = None,
+ ) -> "Dataset":
+ """Return a new Dataset obtained by adding indices (provided in indices_cache_file_name or in a buffer) to the
+ current Dataset.
+ """
+
+ if indices_cache_file_name is None and indices_buffer is None:
+ raise ValueError("At least one of indices_cache_file_name or indices_buffer must be provided.")
+
+ if fingerprint is None:
+ raise ValueError("please specify a fingerprint for the dataset with indices")
+
+ if indices_cache_file_name is not None:
+ indices_table = MemoryMappedTable.from_file(indices_cache_file_name)
+ else:
+ indices_table = InMemoryTable.from_buffer(indices_buffer)
+
+ # Return new Dataset object
+ # don't forget to copy the objects
+ return Dataset(
+ self._data,
+ info=self.info.copy(),
+ split=self.split,
+ indices_table=indices_table,
+ fingerprint=fingerprint,
+ )
+
+ @transmit_format
+ @fingerprint_transform(inplace=False, ignore_kwargs=["indices_cache_file_name"])
+ def select(
+ self,
+ indices: Iterable,
+ keep_in_memory: bool = False,
+ indices_cache_file_name: Optional[str] = None,
+ writer_batch_size: Optional[int] = 1000,
+ new_fingerprint: Optional[str] = None,
+ ) -> "Dataset":
+ """Create a new dataset with rows selected following the list/array of indices.
+
+ Args:
+ indices (`range`, `list`, `iterable`, `ndarray` or `Series`):
+ Range, list or 1D-array of integer indices for indexing.
+ If the indices correspond to a contiguous range, the Arrow table is simply sliced.
+ However passing a list of indices that are not contiguous creates indices mapping, which is much less efficient,
+ but still faster than recreating an Arrow table made of the requested rows.
+ keep_in_memory (`bool`, defaults to `False`):
+ Keep the indices mapping in memory instead of writing it to a cache file.
+ indices_cache_file_name (`str`, *optional*, defaults to `None`):
+ Provide the name of a path for the cache file. It is used to store the
+ indices mapping instead of the automatically generated cache file name.
+ writer_batch_size (`int`, defaults to `1000`):
+ Number of rows per write operation for the cache file writer.
+ This value is a good trade-off between memory usage during the processing, and processing speed.
+ Higher value makes the processing do fewer lookups, lower value consume less temporary memory while running `map`.
+ new_fingerprint (`str`, *optional*, defaults to `None`):
+ The new fingerprint of the dataset after transform.
+ If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> ds = ds.select(range(4))
+ >>> ds
+ Dataset({
+ features: ['text', 'label'],
+ num_rows: 4
+ })
+ ```
+ """
+ if keep_in_memory and indices_cache_file_name is not None:
+ raise ValueError("Please use either `keep_in_memory` or `indices_cache_file_name` but not both.")
+
+ if len(self.list_indexes()) > 0:
+ raise DatasetTransformationNotAllowedError(
+ "Using `.select` on a dataset with attached indexes is not allowed. You can first run `.drop_index() to remove your index and then re-add it."
+ )
+
+ # If the array is empty we do nothing
+ if len(self) == 0:
+ return self
+
+ # If indices is a PyArrow array, we convert to NumPy
+ if isinstance(indices, (pa.Array, pa.ChunkedArray)):
+ indices = indices.to_numpy().astype(np.int64)
+
+ # Convert generator objects to lists
+ if isinstance(indices, Iterator):
+ indices = list(indices)
+
+ # If the indices are contiguous, simply slice the arrow table
+ if isinstance(indices, range):
+ if _is_range_contiguous(indices) and indices.start >= 0:
+ start, length = indices.start, indices.stop - indices.start
+ return self._select_contiguous(start, length, new_fingerprint=new_fingerprint)
+ else:
+ try:
+ start = next(iter(indices))
+ except StopIteration:
+ # if `indices` is an empty iterable, we return an empty dataset
+ return self._select_contiguous(0, 0, new_fingerprint=new_fingerprint)
+ if start >= 0:
+ counter_from_start = itertools.count(start=start)
+ if all(i == j for i, j in zip(indices, counter_from_start)):
+ length = next(counter_from_start) - start
+ return self._select_contiguous(start, length, new_fingerprint=new_fingerprint)
+
+ # If not contiguous, we need to create a new indices mapping
+ return self._select_with_indices_mapping(
+ indices,
+ keep_in_memory=keep_in_memory,
+ indices_cache_file_name=indices_cache_file_name,
+ writer_batch_size=writer_batch_size,
+ new_fingerprint=new_fingerprint,
+ )
+
+ @transmit_format
+ @fingerprint_transform(inplace=False)
+ def _select_contiguous(
+ self,
+ start: int,
+ length: int,
+ new_fingerprint: Optional[str] = None,
+ ) -> "Dataset":
+ """Create a new dataset with rows from a contiguous slice of data.
+ The slice is defined by that start index and its length.
+
+ Args:
+ start (`int`): start index.
+ length (`int`): length of the slice to select.
+ new_fingerprint (`str`, optional, default `None`): the new fingerprint of the dataset after transform.
+ If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> ds._select_contiguous(0, 4)
+ Dataset({
+ features: ['text', 'label'],
+ num_rows: 4
+ })
+ ```
+ """
+ if len(self.list_indexes()) > 0:
+ raise DatasetTransformationNotAllowedError(
+ "Using `.select` on a dataset with attached indexes is not allowed. You can first run `.drop_index() to remove your index and then re-add it."
+ )
+
+ # If the array is empty we do nothing
+ if len(self) == 0:
+ return self
+
+ _check_valid_indices_value(start, len(self))
+ _check_valid_indices_value(start + length - 1, len(self))
+ if self._indices is None or length == 0:
+ return Dataset(
+ self.data.slice(start, length),
+ info=self.info.copy(),
+ split=self.split,
+ fingerprint=new_fingerprint,
+ )
+ else:
+ return Dataset(
+ self.data,
+ info=self.info.copy(),
+ split=self.split,
+ indices_table=self._indices.slice(start, length),
+ fingerprint=new_fingerprint,
+ )
+
+ @transmit_format
+ @fingerprint_transform(inplace=False, ignore_kwargs=["indices_cache_file_name"])
+ def _select_with_indices_mapping(
+ self,
+ indices: Iterable,
+ keep_in_memory: bool = False,
+ indices_cache_file_name: Optional[str] = None,
+ writer_batch_size: Optional[int] = 1000,
+ new_fingerprint: Optional[str] = None,
+ ) -> "Dataset":
+ """Create a new dataset with rows selected following the list/array of indices.
+ The new dataset is made by creating a new indices mapping on top of the main arrow table.
+
+ Args:
+ indices (sequence, iterable, range, ndarray or Series): List or 1D-array of integer indices for indexing.
+ keep_in_memory (`bool`, default `False`): Keep the indices mapping in memory instead of writing it to a cache file.
+ indices_cache_file_name (`str`, optional, default `None`): Provide the name of a path for the cache file. It is used to store the
+ indices mapping instead of the automatically generated cache file name.
+ writer_batch_size (`int`, default `1000`): Number of rows per write operation for the cache file writer.
+ This value is a good trade-off between memory usage during the processing, and processing speed.
+ Higher value makes the processing do fewer lookups, lower value consume less temporary memory while running `.map()`.
+ new_fingerprint (`str`, optional, default `None`): the new fingerprint of the dataset after transform.
+ If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> ds._select_with_indices_mapping(range(4))
+ Dataset({
+ features: ['text', 'label'],
+ num_rows: 4
+ })
+ ```
+ """
+ if keep_in_memory and indices_cache_file_name is not None:
+ raise ValueError("Please use either `keep_in_memory` or `indices_cache_file_name` but not both.")
+
+ if len(self.list_indexes()) > 0:
+ raise DatasetTransformationNotAllowedError(
+ "Using `.select` on a dataset with attached indexes is not allowed. You can first run `.drop_index() to remove your index and then re-add it."
+ )
+
+ # If the array is empty we do nothing
+ if len(self) == 0:
+ return self
+
+ # Prepare the writer for our indices arrow table
+ if keep_in_memory or indices_cache_file_name is None:
+ buf_writer = pa.BufferOutputStream()
+ tmp_file = None
+ writer = ArrowWriter(
+ stream=buf_writer, writer_batch_size=writer_batch_size, fingerprint=new_fingerprint, unit="indices"
+ )
+ else:
+ buf_writer = None
+ logger.info(f"Caching indices mapping at {indices_cache_file_name}")
+ cache_dir = os.path.dirname(indices_cache_file_name)
+ os.makedirs(cache_dir, exist_ok=True)
+ tmp_file = tempfile.NamedTemporaryFile("wb", dir=cache_dir, delete=False)
+ writer = ArrowWriter(
+ path=tmp_file.name, writer_batch_size=writer_batch_size, fingerprint=new_fingerprint, unit="indices"
+ )
+
+ indices = indices if isinstance(indices, list) else list(indices)
+
+ size = len(self)
+ if indices:
+ _check_valid_indices_value(int(max(indices)), size=size)
+ _check_valid_indices_value(int(min(indices)), size=size)
+ else:
+ return self._select_contiguous(0, 0, new_fingerprint=new_fingerprint)
+
+ indices_array = pa.array(indices, type=pa.uint64())
+ # Check if we need to convert indices
+ if self._indices is not None:
+ indices_array = self._indices.column(0).take(indices_array)
+
+ indices_table = pa.Table.from_arrays([indices_array], names=["indices"])
+
+ with writer:
+ try:
+ writer.write_table(indices_table)
+ writer.finalize() # close_stream=bool(buf_writer is None)) We only close if we are writing in a file
+ except (Exception, KeyboardInterrupt):
+ if tmp_file is not None:
+ tmp_file.close()
+ if os.path.exists(tmp_file.name):
+ os.remove(tmp_file.name)
+ raise
+
+ if tmp_file is not None:
+ tmp_file.close()
+ shutil.move(tmp_file.name, indices_cache_file_name)
+ umask = os.umask(0o666)
+ os.umask(umask)
+ os.chmod(indices_cache_file_name, 0o666 & ~umask)
+
+ # Return new Dataset object
+ if buf_writer is None:
+ return self._new_dataset_with_indices(
+ indices_cache_file_name=indices_cache_file_name, fingerprint=new_fingerprint
+ )
+ else:
+ return self._new_dataset_with_indices(indices_buffer=buf_writer.getvalue(), fingerprint=new_fingerprint)
+
+ def skip(self, n: int) -> "Dataset":
+ """
+ Create a new [`Dataset`] that skips the first `n` elements.
+
+ Args:
+ n (`int`):
+ Number of elements to skip.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train")
+ >>> list(ds.take(3))
+ [{'label': 1,
+ 'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'},
+ {'label': 1,
+ 'text': 'the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson\'s expanded vision of j . r . r . tolkien\'s middle-earth .'},
+ {'label': 1, 'text': 'effective but too-tepid biopic'}]
+ >>> ds = ds.skip(1)
+ >>> list(ds.take(3))
+ [{'label': 1,
+ 'text': 'the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson\'s expanded vision of j . r . r . tolkien\'s middle-earth .'},
+ {'label': 1, 'text': 'effective but too-tepid biopic'},
+ {'label': 1,
+ 'text': 'if you sometimes like to go to the movies to have fun , wasabi is a good place to start .'}]
+ ```
+ """
+ return self.select(range(n, len(self)))
+
+ def repeat(self, num_times: int) -> "Dataset":
+ """
+ Create a new [`Dataset`] that repeats the underlying dataset `num_times` times.
+
+ Like itertools.repeat, repeating once just returns the full dataset.
+
+ Args:
+ num_times (`int`):
+ Number of times to repeat the dataset.
+
+ Example:
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train")
+ >>> ds = ds.take(2).repeat(2)
+ >>> list(ds)
+ [{'label': 1,
+ 'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'},
+ {'label': 1,
+ 'text': 'the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson\'s expanded vision of j . r . r . tolkien\'s middle-earth .'},
+ {'label': 1, 'text': 'effective but too-tepid biopic'},
+ {'label': 1,
+ 'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'},
+ {'label': 1,
+ 'text': 'the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson\'s expanded vision of j . r . r . tolkien\'s middle-earth .'},
+ {'label': 1, 'text': 'effective but too-tepid biopic'}]
+ ```
+ """
+ if num_times is None:
+ raise ValueError("Map style datasets do not support indefinite repetition.")
+ return _concatenate_map_style_datasets([self] * num_times) if num_times > 0 else self.select([])
+
+ def take(self, n: int) -> "Dataset":
+ """
+ Create a new [`Dataset`] with only the first `n` elements.
+
+ Args:
+ n (`int`):
+ Number of elements to take.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train")
+ >>> small_ds = ds.take(2)
+ >>> list(small_ds)
+ [{'label': 1,
+ 'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'},
+ {'label': 1,
+ 'text': 'the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson\'s expanded vision of j . r . r . tolkien\'s middle-earth .'}]
+ ```
+ """
+ return self.select(range(n))
+
+ @transmit_format
+ @fingerprint_transform(inplace=False, ignore_kwargs=["load_from_cache_file", "indices_cache_file_name"])
+ def sort(
+ self,
+ column_names: Union[str, Sequence_[str]],
+ reverse: Union[bool, Sequence_[bool]] = False,
+ null_placement: str = "at_end",
+ keep_in_memory: bool = False,
+ load_from_cache_file: Optional[bool] = None,
+ indices_cache_file_name: Optional[str] = None,
+ writer_batch_size: Optional[int] = 1000,
+ new_fingerprint: Optional[str] = None,
+ ) -> "Dataset":
+ """Create a new dataset sorted according to a single or multiple columns.
+
+ Args:
+ column_names (`Union[str, Sequence[str]]`):
+ Column name(s) to sort by.
+ reverse (`Union[bool, Sequence[bool]]`, defaults to `False`):
+ If `True`, sort by descending order rather than ascending. If a single bool is provided,
+ the value is applied to the sorting of all column names. Otherwise a list of bools with the
+ same length and order as column_names must be provided.
+ null_placement (`str`, defaults to `at_end`):
+ Put `None` values at the beginning if `at_start` or `first` or at the end if `at_end` or `last`
+
+
+ keep_in_memory (`bool`, defaults to `False`):
+ Keep the sorted indices in memory instead of writing it to a cache file.
+ load_from_cache_file (`Optional[bool]`, defaults to `True` if caching is enabled):
+ If a cache file storing the sorted indices
+ can be identified, use it instead of recomputing.
+ indices_cache_file_name (`str`, *optional*, defaults to `None`):
+ Provide the name of a path for the cache file. It is used to store the
+ sorted indices instead of the automatically generated cache file name.
+ writer_batch_size (`int`, defaults to `1000`):
+ Number of rows per write operation for the cache file writer.
+ Higher value gives smaller cache files, lower value consume less temporary memory.
+ new_fingerprint (`str`, *optional*, defaults to `None`):
+ The new fingerprint of the dataset after transform.
+ If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset('cornell-movie-review-data/rotten_tomatoes', split='validation')
+ >>> ds['label'][:10]
+ [1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
+ >>> sorted_ds = ds.sort('label')
+ >>> sorted_ds['label'][:10]
+ [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
+ >>> another_sorted_ds = ds.sort(['label', 'text'], reverse=[True, False])
+ >>> another_sorted_ds['label'][:10]
+ [1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
+ ```
+ """
+ if len(self.list_indexes()) > 0:
+ raise DatasetTransformationNotAllowedError(
+ "Using `.sort` on a dataset with attached indexes is not allowed. You can first run `.drop_index() to remove your index and then re-add it."
+ )
+ # If the array is empty we do nothing
+ if len(self) == 0:
+ return self
+
+ # Check proper format of and for duplicates in column_names
+ if isinstance(column_names, str):
+ column_names = [column_names]
+
+ # Check proper format and length of reverse
+ if not isinstance(reverse, bool):
+ if len(reverse) != len(column_names):
+ raise ValueError(
+ "Parameter 'reverse' should be either a boolean or a list of booleans with the same length as 'column_names'."
+ )
+ else:
+ reverse = [reverse] * len(column_names)
+
+ # Check whether column name(s) exist in dataset
+ for column in column_names:
+ if not isinstance(column, str) or column not in self._data.column_names:
+ raise ValueError(
+ f"Column '{column}' not found in the dataset. Please provide a column selected in: {self._data.column_names}"
+ )
+
+ # Change null_placement to conform to pyarrow's sort_indices() while ensuring backwards compatibility
+ if null_placement not in ["at_start", "at_end"]:
+ if null_placement == "first":
+ null_placement = "at_start"
+ elif null_placement == "last":
+ null_placement = "at_end"
+ else:
+ raise ValueError(
+ f"null_placement '{null_placement}' is an invalid parameter value. Must be either 'last', 'at_end', 'first' or 'at_start'."
+ )
+
+ load_from_cache_file = load_from_cache_file if load_from_cache_file is not None else is_caching_enabled()
+
+ # Check if we've already cached this computation (indexed by a hash)
+ if self.cache_files:
+ if indices_cache_file_name is None:
+ # we create a unique hash from the function, current dataset file and the mapping args
+ indices_cache_file_name = self._get_cache_file_path(new_fingerprint)
+ if os.path.exists(indices_cache_file_name) and load_from_cache_file:
+ logger.info(f"Loading cached sorted indices for dataset at {indices_cache_file_name}")
+ return self._new_dataset_with_indices(
+ fingerprint=new_fingerprint, indices_cache_file_name=indices_cache_file_name
+ )
+
+ sort_table = query_table(
+ table=self._data,
+ key=slice(0, len(self)),
+ indices=self._indices,
+ )
+
+ sort_keys = [
+ (col, "ascending" if not col_reverse else "descending") for col, col_reverse in zip(column_names, reverse)
+ ]
+
+ indices = pc.sort_indices(sort_table, sort_keys=sort_keys, null_placement=null_placement)
+
+ return self.select(
+ indices=indices,
+ keep_in_memory=keep_in_memory,
+ indices_cache_file_name=indices_cache_file_name,
+ writer_batch_size=writer_batch_size,
+ new_fingerprint=new_fingerprint,
+ )
+
+ @transmit_format
+ @fingerprint_transform(
+ inplace=False, randomized_function=True, ignore_kwargs=["load_from_cache_file", "indices_cache_file_name"]
+ )
+ def shuffle(
+ self,
+ seed: Optional[int] = None,
+ generator: Optional[np.random.Generator] = None,
+ keep_in_memory: bool = False,
+ load_from_cache_file: Optional[bool] = None,
+ indices_cache_file_name: Optional[str] = None,
+ writer_batch_size: Optional[int] = 1000,
+ new_fingerprint: Optional[str] = None,
+ ) -> "Dataset":
+ """Create a new Dataset where the rows are shuffled.
+
+ Currently shuffling uses numpy random generators.
+ You can either supply a NumPy BitGenerator to use, or a seed to initiate NumPy's default random generator (PCG64).
+
+ Shuffling takes the list of indices `[0:len(my_dataset)]` and shuffles it to create an indices mapping.
+ However as soon as your [`Dataset`] has an indices mapping, the speed can become 10x slower.
+ This is because there is an extra step to get the row index to read using the indices mapping, and most importantly, you aren't reading contiguous chunks of data anymore.
+ To restore the speed, you'd need to rewrite the entire dataset on your disk again using [`Dataset.flatten_indices`], which removes the indices mapping.
+ This may take a lot of time depending of the size of your dataset though:
+
+ ```python
+ my_dataset[0] # fast
+ my_dataset = my_dataset.shuffle(seed=42)
+ my_dataset[0] # up to 10x slower
+ my_dataset = my_dataset.flatten_indices() # rewrite the shuffled dataset on disk as contiguous chunks of data
+ my_dataset[0] # fast again
+ ```
+
+ In this case, we recommend switching to an [`IterableDataset`] and leveraging its fast approximate shuffling method [`IterableDataset.shuffle`].
+ It only shuffles the shards order and adds a shuffle buffer to your dataset, which keeps the speed of your dataset optimal:
+
+ ```python
+ my_iterable_dataset = my_dataset.to_iterable_dataset(num_shards=128)
+ for example in enumerate(my_iterable_dataset): # fast
+ pass
+
+ shuffled_iterable_dataset = my_iterable_dataset.shuffle(seed=42, buffer_size=100)
+
+ for example in enumerate(shuffled_iterable_dataset): # as fast as before
+ pass
+ ```
+
+ Args:
+ seed (`int`, *optional*):
+ A seed to initialize the default BitGenerator if `generator=None`.
+ If `None`, then fresh, unpredictable entropy will be pulled from the OS.
+ If an `int` or `array_like[ints]` is passed, then it will be passed to SeedSequence to derive the initial BitGenerator state.
+ generator (`numpy.random.Generator`, *optional*):
+ Numpy random Generator to use to compute the permutation of the dataset rows.
+ If `generator=None` (default), uses `np.random.default_rng` (the default BitGenerator (PCG64) of NumPy).
+ keep_in_memory (`bool`, default `False`):
+ Keep the shuffled indices in memory instead of writing it to a cache file.
+ load_from_cache_file (`Optional[bool]`, defaults to `True` if caching is enabled):
+ If a cache file storing the shuffled indices
+ can be identified, use it instead of recomputing.
+ indices_cache_file_name (`str`, *optional*):
+ Provide the name of a path for the cache file. It is used to store the
+ shuffled indices instead of the automatically generated cache file name.
+ writer_batch_size (`int`, defaults to `1000`):
+ Number of rows per write operation for the cache file writer.
+ This value is a good trade-off between memory usage during the processing, and processing speed.
+ Higher value makes the processing do fewer lookups, lower value consume less temporary memory while running `map`.
+ new_fingerprint (`str`, *optional*, defaults to `None`):
+ The new fingerprint of the dataset after transform.
+ If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> ds['label'][:10]
+ [1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
+
+ # set a seed
+ >>> shuffled_ds = ds.shuffle(seed=42)
+ >>> shuffled_ds['label'][:10]
+ [1, 0, 1, 1, 0, 0, 0, 0, 0, 0]
+ ```
+ """
+ if len(self.list_indexes()) > 0:
+ raise DatasetTransformationNotAllowedError(
+ "Using `.shuffle` on a dataset with attached indexes is not allowed. You can first run `.drop_index() to remove your index and then re-add it."
+ )
+ # If the array is empty we do nothing
+ if len(self) == 0:
+ return self
+
+ if keep_in_memory and indices_cache_file_name is not None:
+ raise ValueError("Please use either `keep_in_memory` or `indices_cache_file_name` but not both.")
+
+ if seed is not None and generator is not None:
+ raise ValueError("Both `seed` and `generator` were provided. Please specify just one of them.")
+
+ if generator is not None and not isinstance(generator, np.random.Generator):
+ raise ValueError("The provided generator must be an instance of numpy.random.Generator")
+
+ load_from_cache_file = load_from_cache_file if load_from_cache_file is not None else is_caching_enabled()
+
+ if generator is None:
+ if seed is None:
+ _, seed, pos, *_ = np.random.get_state()
+ seed = seed[pos] if pos < 624 else seed[0]
+ _ = np.random.random() # do 1 step of rng
+ generator = np.random.default_rng(seed)
+
+ # Check if we've already cached this computation (indexed by a hash)
+ if self.cache_files:
+ if indices_cache_file_name is None:
+ # we create a unique hash from the function, current dataset file and the mapping args
+ indices_cache_file_name = self._get_cache_file_path(new_fingerprint)
+ if os.path.exists(indices_cache_file_name) and load_from_cache_file:
+ logger.info(f"Loading cached shuffled indices for dataset at {indices_cache_file_name}")
+ return self._new_dataset_with_indices(
+ fingerprint=new_fingerprint, indices_cache_file_name=indices_cache_file_name
+ )
+
+ permutation = generator.permutation(len(self))
+
+ return self.select(
+ indices=permutation,
+ keep_in_memory=keep_in_memory,
+ indices_cache_file_name=indices_cache_file_name if not keep_in_memory else None,
+ writer_batch_size=writer_batch_size,
+ new_fingerprint=new_fingerprint,
+ )
+
+ @transmit_format
+ @fingerprint_transform(
+ inplace=False,
+ randomized_function=True,
+ fingerprint_names=["train_new_fingerprint", "test_new_fingerprint"],
+ ignore_kwargs=["load_from_cache_file", "train_indices_cache_file_name", "test_indices_cache_file_name"],
+ )
+ def train_test_split(
+ self,
+ test_size: Union[float, int, None] = None,
+ train_size: Union[float, int, None] = None,
+ shuffle: bool = True,
+ stratify_by_column: Optional[str] = None,
+ seed: Optional[int] = None,
+ generator: Optional[np.random.Generator] = None,
+ keep_in_memory: bool = False,
+ load_from_cache_file: Optional[bool] = None,
+ train_indices_cache_file_name: Optional[str] = None,
+ test_indices_cache_file_name: Optional[str] = None,
+ writer_batch_size: Optional[int] = 1000,
+ train_new_fingerprint: Optional[str] = None,
+ test_new_fingerprint: Optional[str] = None,
+ ) -> "DatasetDict":
+ """Return a dictionary ([`datasets.DatasetDict`]) with two random train and test subsets (`train` and `test` `Dataset` splits).
+ Splits are created from the dataset according to `test_size`, `train_size` and `shuffle`.
+
+ This method is similar to scikit-learn `train_test_split`.
+
+ Args:
+ test_size (`Union[float, int, None]`, *optional*):
+ Size of the test split
+ If `float`, should be between `0.0` and `1.0` and represent the proportion of the dataset to include in the test split.
+ If `int`, represents the absolute number of test samples.
+ If `None`, the value is set to the complement of the train size.
+ If `train_size` is also `None`, it will be set to `0.25`.
+ train_size (`Union[float, int, None]`, *optional*):
+ Size of the train split
+ If `float`, should be between `0.0` and `1.0` and represent the proportion of the dataset to include in the train split.
+ If `int`, represents the absolute number of train samples.
+ If `None`, the value is automatically set to the complement of the test size.
+ shuffle (`bool`, *optional*, defaults to `True`):
+ Whether or not to shuffle the data before splitting.
+ stratify_by_column (`str`, *optional*, defaults to `None`):
+ The column name of labels to be used to perform stratified split of data.
+ seed (`int`, *optional*):
+ A seed to initialize the default BitGenerator if `generator=None`.
+ If `None`, then fresh, unpredictable entropy will be pulled from the OS.
+ If an `int` or `array_like[ints]` is passed, then it will be passed to SeedSequence to derive the initial BitGenerator state.
+ generator (`numpy.random.Generator`, *optional*):
+ Numpy random Generator to use to compute the permutation of the dataset rows.
+ If `generator=None` (default), uses `np.random.default_rng` (the default BitGenerator (PCG64) of NumPy).
+ keep_in_memory (`bool`, defaults to `False`):
+ Keep the splits indices in memory instead of writing it to a cache file.
+ load_from_cache_file (`Optional[bool]`, defaults to `True` if caching is enabled):
+ If a cache file storing the splits indices
+ can be identified, use it instead of recomputing.
+ train_cache_file_name (`str`, *optional*):
+ Provide the name of a path for the cache file. It is used to store the
+ train split indices instead of the automatically generated cache file name.
+ test_cache_file_name (`str`, *optional*):
+ Provide the name of a path for the cache file. It is used to store the
+ test split indices instead of the automatically generated cache file name.
+ writer_batch_size (`int`, defaults to `1000`):
+ Number of rows per write operation for the cache file writer.
+ This value is a good trade-off between memory usage during the processing, and processing speed.
+ Higher value makes the processing do fewer lookups, lower value consume less temporary memory while running `map`.
+ train_new_fingerprint (`str`, *optional*, defaults to `None`):
+ The new fingerprint of the train set after transform.
+ If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments
+ test_new_fingerprint (`str`, *optional*, defaults to `None`):
+ The new fingerprint of the test set after transform.
+ If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> ds = ds.train_test_split(test_size=0.2, shuffle=True)
+ DatasetDict({
+ train: Dataset({
+ features: ['text', 'label'],
+ num_rows: 852
+ })
+ test: Dataset({
+ features: ['text', 'label'],
+ num_rows: 214
+ })
+ })
+
+ # set a seed
+ >>> ds = ds.train_test_split(test_size=0.2, seed=42)
+
+ # stratified split
+ >>> ds = load_dataset("imdb",split="train")
+ Dataset({
+ features: ['text', 'label'],
+ num_rows: 25000
+ })
+ >>> ds = ds.train_test_split(test_size=0.2, stratify_by_column="label")
+ DatasetDict({
+ train: Dataset({
+ features: ['text', 'label'],
+ num_rows: 20000
+ })
+ test: Dataset({
+ features: ['text', 'label'],
+ num_rows: 5000
+ })
+ })
+ ```
+ """
+ from .dataset_dict import DatasetDict # import here because of circular dependency
+
+ if len(self.list_indexes()) > 0:
+ raise DatasetTransformationNotAllowedError(
+ "Using `.train_test_split` on a dataset with attached indexes is not allowed. You can first run `.drop_index() to remove your index and then re-add it."
+ )
+ # If the array is empty we do nothing
+ if len(self) == 0:
+ return DatasetDict({"train": self, "test": self})
+
+ if test_size is None and train_size is None:
+ test_size = 0.25
+
+ # Safety checks similar to scikit-learn's ones.
+ # (adapted from https://github.com/scikit-learn/scikit-learn/blob/fd237278e895b42abe8d8d09105cbb82dc2cbba7/sklearn/model_selection/_split.py#L1750)
+ n_samples = len(self)
+ if (
+ isinstance(test_size, int)
+ and (test_size >= n_samples or test_size <= 0)
+ or isinstance(test_size, float)
+ and (test_size <= 0 or test_size >= 1)
+ ):
+ raise ValueError(
+ f"test_size={test_size} should be either positive and smaller "
+ f"than the number of samples {n_samples} or a float in the (0, 1) range"
+ )
+
+ if (
+ isinstance(train_size, int)
+ and (train_size >= n_samples or train_size <= 0)
+ or isinstance(train_size, float)
+ and (train_size <= 0 or train_size >= 1)
+ ):
+ raise ValueError(
+ f"train_size={train_size} should be either positive and smaller "
+ f"than the number of samples {n_samples} or a float in the (0, 1) range"
+ )
+
+ if train_size is not None and not isinstance(train_size, (int, float)):
+ raise ValueError(f"Invalid value for train_size: {train_size} of type {type(train_size)}")
+ if test_size is not None and not isinstance(test_size, (int, float)):
+ raise ValueError(f"Invalid value for test_size: {test_size} of type {type(test_size)}")
+
+ if isinstance(train_size, float) and isinstance(test_size, float) and train_size + test_size > 1:
+ raise ValueError(
+ f"The sum of test_size and train_size = {train_size + test_size}, should be in the (0, 1)"
+ " range. Reduce test_size and/or train_size."
+ )
+
+ if isinstance(test_size, float):
+ n_test = ceil(test_size * n_samples)
+ elif isinstance(test_size, int):
+ n_test = float(test_size)
+
+ if isinstance(train_size, float):
+ n_train = floor(train_size * n_samples)
+ elif isinstance(train_size, int):
+ n_train = float(train_size)
+
+ if train_size is None:
+ n_train = n_samples - n_test
+ elif test_size is None:
+ n_test = n_samples - n_train
+
+ if n_train + n_test > n_samples:
+ raise ValueError(
+ f"The sum of train_size and test_size = {n_train + n_test}, "
+ "should be smaller than the number of "
+ f"samples {n_samples}. Reduce test_size and/or "
+ "train_size."
+ )
+
+ n_train, n_test = int(n_train), int(n_test)
+
+ if n_train == 0:
+ raise ValueError(
+ f"With n_samples={n_samples}, test_size={test_size} and train_size={train_size}, the "
+ "resulting train set will be empty. Adjust any of the "
+ "aforementioned parameters."
+ )
+
+ load_from_cache_file = load_from_cache_file if load_from_cache_file is not None else is_caching_enabled()
+
+ if generator is None and shuffle is True:
+ if seed is None:
+ _, seed, pos, *_ = np.random.get_state()
+ seed = seed[pos] if pos < 624 else seed[0]
+ _ = np.random.random() # do 1 step of rng
+ generator = np.random.default_rng(seed)
+
+ # Check if we've already cached this computation (indexed by a hash)
+ if self.cache_files:
+ if train_indices_cache_file_name is None or test_indices_cache_file_name is None:
+ # we create a unique hash from the function, current dataset file and the mapping args
+
+ if train_indices_cache_file_name is None:
+ train_indices_cache_file_name = self._get_cache_file_path(train_new_fingerprint)
+ if test_indices_cache_file_name is None:
+ test_indices_cache_file_name = self._get_cache_file_path(test_new_fingerprint)
+ if (
+ os.path.exists(train_indices_cache_file_name)
+ and os.path.exists(test_indices_cache_file_name)
+ and load_from_cache_file
+ ):
+ logger.info(
+ f"Loading cached split indices for dataset at {train_indices_cache_file_name} and {test_indices_cache_file_name}"
+ )
+ return DatasetDict(
+ {
+ "train": self._new_dataset_with_indices(
+ fingerprint=train_new_fingerprint, indices_cache_file_name=train_indices_cache_file_name
+ ),
+ "test": self._new_dataset_with_indices(
+ fingerprint=test_new_fingerprint, indices_cache_file_name=test_indices_cache_file_name
+ ),
+ }
+ )
+ if not shuffle:
+ if stratify_by_column is not None:
+ raise ValueError("Stratified train/test split is not implemented for `shuffle=False`")
+ train_indices = np.arange(n_train)
+ test_indices = np.arange(n_train, n_train + n_test)
+ else:
+ # stratified partition
+ if stratify_by_column is not None:
+ if stratify_by_column not in self._info.features.keys():
+ raise ValueError(f"Key {stratify_by_column} not found in {self._info.features.keys()}")
+ if not isinstance(self._info.features[stratify_by_column], ClassLabel):
+ raise ValueError(
+ f"Stratifying by column is only supported for {ClassLabel.__name__} column, and column {stratify_by_column} is {type(self._info.features[stratify_by_column]).__name__}."
+ )
+ try:
+ train_indices, test_indices = next(
+ stratified_shuffle_split_generate_indices(
+ np.asarray(self.with_format("numpy")[stratify_by_column]), n_train, n_test, rng=generator
+ )
+ )
+ except Exception as error:
+ if str(error) == "Minimum class count error":
+ raise ValueError(
+ f"The least populated class in {stratify_by_column} column has only 1"
+ " member, which is too few. The minimum"
+ " number of groups for any class cannot"
+ " be less than 2."
+ )
+ else:
+ raise error
+
+ # random partition
+ else:
+ permutation = generator.permutation(len(self))
+ test_indices = permutation[:n_test]
+ train_indices = permutation[n_test : (n_test + n_train)]
+
+ train_split = self.select(
+ indices=train_indices,
+ keep_in_memory=keep_in_memory,
+ indices_cache_file_name=train_indices_cache_file_name,
+ writer_batch_size=writer_batch_size,
+ new_fingerprint=train_new_fingerprint,
+ )
+ test_split = self.select(
+ indices=test_indices,
+ keep_in_memory=keep_in_memory,
+ indices_cache_file_name=test_indices_cache_file_name,
+ writer_batch_size=writer_batch_size,
+ new_fingerprint=test_new_fingerprint,
+ )
+
+ return DatasetDict({"train": train_split, "test": test_split})
+
+ def shard(
+ self,
+ num_shards: int,
+ index: int,
+ contiguous: bool = True,
+ keep_in_memory: bool = False,
+ indices_cache_file_name: Optional[str] = None,
+ writer_batch_size: Optional[int] = 1000,
+ ) -> "Dataset":
+ """Return the `index`-nth shard from dataset split into `num_shards` pieces.
+
+ This shards deterministically. `dataset.shard(n, i)` splits the dataset into contiguous chunks,
+ so it can be easily concatenated back together after processing. If `len(dataset) % n == l`, then the
+ first `l` dataset each have length `(len(dataset) // n) + 1`, and the remaining dataset have length `(len(dataset) // n)`.
+ `datasets.concatenate_datasets([dset.shard(n, i) for i in range(n)])` returns a dataset with the same order as the original.
+
+ Note: n should be less or equal to the number of elements in the dataset `len(dataset)`.
+
+ On the other hand, `dataset.shard(n, i, contiguous=False)` contains all elements of the dataset whose index mod `n = i`.
+
+ Be sure to shard before using any randomizing operator (such as `shuffle`).
+ It is best if the shard operator is used early in the dataset pipeline.
+
+ Args:
+ num_shards (`int`):
+ How many shards to split the dataset into.
+ index (`int`):
+ Which shard to select and return.
+ contiguous: (`bool`, defaults to `True`):
+ Whether to select contiguous blocks of indices for shards.
+ keep_in_memory (`bool`, defaults to `False`):
+ Keep the dataset in memory instead of writing it to a cache file.
+ indices_cache_file_name (`str`, *optional*):
+ Provide the name of a path for the cache file. It is used to store the
+ indices of each shard instead of the automatically generated cache file name.
+ writer_batch_size (`int`, defaults to `1000`):
+ This only concerns the indices mapping.
+ Number of indices per write operation for the cache file writer.
+ This value is a good trade-off between memory usage during the processing, and processing speed.
+ Higher value makes the processing do fewer lookups, lower value consume less temporary memory while running `map`.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> ds
+ Dataset({
+ features: ['text', 'label'],
+ num_rows: 1066
+ })
+ >>> ds = ds.shard(num_shards=2, index=0)
+ >>> ds
+ Dataset({
+ features: ['text', 'label'],
+ num_rows: 533
+ })
+ ```
+ """
+ if not 0 <= index < num_shards:
+ raise ValueError("index should be in [0, num_shards-1]")
+ if contiguous:
+ div = len(self) // num_shards
+ mod = len(self) % num_shards
+ start = div * index + min(index, mod)
+ end = start + div + (1 if index < mod else 0)
+ indices = range(start, end)
+ else:
+ indices = np.arange(index, len(self), num_shards)
+
+ return self.select(
+ indices=indices,
+ keep_in_memory=keep_in_memory,
+ indices_cache_file_name=indices_cache_file_name,
+ writer_batch_size=writer_batch_size,
+ )
+
+ def to_csv(
+ self,
+ path_or_buf: Union[PathLike, BinaryIO],
+ batch_size: Optional[int] = None,
+ num_proc: Optional[int] = None,
+ storage_options: Optional[dict] = None,
+ **to_csv_kwargs,
+ ) -> int:
+ """Exports the dataset to csv
+
+ Args:
+ path_or_buf (`PathLike` or `FileOrBuffer`):
+ Either a path to a file (e.g. `file.csv`), a remote URI (e.g. `hf://datasets/username/my_dataset_name/data.csv`),
+ or a BinaryIO, where the dataset will be saved to in the specified format.
+ batch_size (`int`, *optional*):
+ Size of the batch to load in memory and write at once.
+ Defaults to `datasets.config.DEFAULT_MAX_BATCH_SIZE`.
+ num_proc (`int`, *optional*):
+ Number of processes for multiprocessing. By default it doesn't
+ use multiprocessing. `batch_size` in this case defaults to
+ `datasets.config.DEFAULT_MAX_BATCH_SIZE` but feel free to make it 5x or 10x of the default
+ value if you have sufficient compute power.
+ storage_options (`dict`, *optional*):
+ Key/value pairs to be passed on to the file-system backend, if any.
+
+
+ **to_csv_kwargs (additional keyword arguments):
+ Parameters to pass to pandas's [`pandas.DataFrame.to_csv`](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.to_csv.html).
+
+
+
+ Now, `index` defaults to `False` if not specified.
+
+ If you would like to write the index, pass `index=True` and also set a name for the index column by
+ passing `index_label`.
+
+
+
+ Returns:
+ `int`: The number of characters or bytes written.
+
+ Example:
+
+ ```py
+ >>> ds.to_csv("path/to/dataset/directory")
+ ```
+ """
+ # Dynamic import to avoid circular dependency
+ from .io.csv import CsvDatasetWriter
+
+ return CsvDatasetWriter(
+ self,
+ path_or_buf,
+ batch_size=batch_size,
+ num_proc=num_proc,
+ storage_options=storage_options,
+ **to_csv_kwargs,
+ ).write()
+
+ def to_dict(self, batch_size: Optional[int] = None, batched: bool = False) -> Union[dict, Iterator[dict]]:
+ """Returns the dataset as a Python dict. Can also return a generator for large datasets.
+
+ Args:
+ batch_size (`int`, *optional*): The size (number of rows) of the batches if `batched` is `True`.
+ Defaults to `datasets.config.DEFAULT_MAX_BATCH_SIZE`.
+ batched (`bool`):
+ Set to `True` to return a generator that yields the dataset as batches
+ of `batch_size` rows. Defaults to `False` (returns the whole datasets once).
+
+ Returns:
+ `dict` or `Iterator[dict]`
+
+ Example:
+
+ ```py
+ >>> ds.to_dict()
+ ```
+ """
+ return query_table(
+ table=self._data,
+ key=slice(0, len(self)),
+ indices=self._indices,
+ ).to_pydict()
+
+ def to_list(self) -> list:
+ """Returns the dataset as a Python list.
+
+ Returns:
+ `list`
+
+ Example:
+
+ ```py
+ >>> ds.to_list()
+ ```
+ """
+ return query_table(
+ table=self._data,
+ key=slice(0, len(self)),
+ indices=self._indices,
+ ).to_pylist()
+
+ def to_json(
+ self,
+ path_or_buf: Union[PathLike, BinaryIO],
+ batch_size: Optional[int] = None,
+ num_proc: Optional[int] = None,
+ storage_options: Optional[dict] = None,
+ **to_json_kwargs,
+ ) -> int:
+ """Export the dataset to JSON Lines or JSON.
+
+ The default output format is [JSON Lines](https://jsonlines.org/).
+ To export to [JSON](https://www.json.org), pass `lines=False` argument and the desired `orient`.
+
+ Args:
+ path_or_buf (`PathLike` or `FileOrBuffer`):
+ Either a path to a file (e.g. `file.json`), a remote URI (e.g. `hf://datasets/username/my_dataset_name/data.json`),
+ or a BinaryIO, where the dataset will be saved to in the specified format.
+ batch_size (`int`, *optional*):
+ Size of the batch to load in memory and write at once.
+ Defaults to `datasets.config.DEFAULT_MAX_BATCH_SIZE`.
+ num_proc (`int`, *optional*):
+ Number of processes for multiprocessing. By default, it doesn't
+ use multiprocessing. `batch_size` in this case defaults to
+ `datasets.config.DEFAULT_MAX_BATCH_SIZE` but feel free to make it 5x or 10x of the default
+ value if you have sufficient compute power.
+ storage_options (`dict`, *optional*):
+ Key/value pairs to be passed on to the file-system backend, if any.
+
+
+ **to_json_kwargs (additional keyword arguments):
+ Parameters to pass to pandas's [`pandas.DataFrame.to_json`](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.to_json.html).
+ Default arguments are `lines=True` and `orient="records".
+
+
+
+ The parameter `index` defaults to `False` if `orient` is `"split"` or `"table"`.
+
+ If you would like to write the index, pass `index=True`.
+
+
+
+ Returns:
+ `int`: The number of characters or bytes written.
+
+ Example:
+
+ ```py
+ >>> ds.to_json("path/to/dataset/directory/filename.jsonl")
+ ```
+ """
+ # Dynamic import to avoid circular dependency
+ from .io.json import JsonDatasetWriter
+
+ return JsonDatasetWriter(
+ self,
+ path_or_buf,
+ batch_size=batch_size,
+ num_proc=num_proc,
+ storage_options=storage_options,
+ **to_json_kwargs,
+ ).write()
+
+ def to_pandas(
+ self, batch_size: Optional[int] = None, batched: bool = False
+ ) -> Union[pd.DataFrame, Iterator[pd.DataFrame]]:
+ """Returns the dataset as a `pandas.DataFrame`. Can also return a generator for large datasets.
+
+ Args:
+ batch_size (`int`, *optional*):
+ The size (number of rows) of the batches if `batched` is `True`.
+ Defaults to `datasets.config.DEFAULT_MAX_BATCH_SIZE`.
+ batched (`bool`):
+ Set to `True` to return a generator that yields the dataset as batches
+ of `batch_size` rows. Defaults to `False` (returns the whole datasets once).
+
+ Returns:
+ `pandas.DataFrame` or `Iterator[pandas.DataFrame]`
+
+ Example:
+
+ ```py
+ >>> ds.to_pandas()
+ ```
+ """
+ if not batched:
+ return query_table(
+ table=self._data,
+ key=slice(0, len(self)),
+ indices=self._indices,
+ ).to_pandas(types_mapper=pandas_types_mapper)
+ else:
+ batch_size = batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
+ return (
+ query_table(
+ table=self._data,
+ key=slice(offset, offset + batch_size),
+ indices=self._indices,
+ ).to_pandas(types_mapper=pandas_types_mapper)
+ for offset in range(0, len(self), batch_size)
+ )
+
+ def to_polars(
+ self,
+ batch_size: Optional[int] = None,
+ batched: bool = False,
+ schema_overrides: Optional[dict] = None,
+ rechunk: bool = True,
+ ) -> Union["pl.DataFrame", Iterator["pl.DataFrame"]]:
+ """Returns the dataset as a `polars.DataFrame`. Can also return a generator for large datasets.
+
+ Args:
+ batch_size (`int`, *optional*):
+ The size (number of rows) of the batches if `batched` is `True`.
+ Defaults to `datasets.config.DEFAULT_MAX_BATCH_SIZE`.
+ batched (`bool`):
+ Set to `True` to return a generator that yields the dataset as batches
+ of `batch_size` rows. Defaults to `False` (returns the whole datasets once).
+ schema_overrides (`dict`, *optional*):
+ Support type specification or override of one or more columns; note that
+ any dtypes inferred from the schema param will be overridden.
+ rechunk (`bool`):
+ Make sure that all data is in contiguous memory. Defaults to `True`.
+ Returns:
+ `polars.DataFrame` or `Iterator[polars.DataFrame]`
+
+ Example:
+
+ ```py
+ >>> ds.to_polars()
+ ```
+ """
+ if config.POLARS_AVAILABLE:
+ import polars as pl
+
+ if not batched:
+ return pl.from_arrow(
+ query_table(
+ table=self._data,
+ key=slice(0, len(self)),
+ indices=self._indices if self._indices is not None else None,
+ ),
+ schema_overrides=schema_overrides,
+ rechunk=rechunk,
+ )
+ else:
+ batch_size = batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
+ return (
+ pl.from_arrow(
+ query_table(
+ table=self._data,
+ key=slice(offset, offset + batch_size),
+ indices=self._indices if self._indices is not None else None,
+ ),
+ schema_overrides=schema_overrides,
+ rechunk=rechunk,
+ )
+ for offset in range(0, len(self), batch_size)
+ )
+ else:
+ raise ValueError("Polars needs to be installed to be able to return Polars dataframes.")
+
+ def to_parquet(
+ self,
+ path_or_buf: Union[PathLike, BinaryIO],
+ batch_size: Optional[int] = None,
+ storage_options: Optional[dict] = None,
+ **parquet_writer_kwargs,
+ ) -> int:
+ """Exports the dataset to parquet
+
+ Args:
+ path_or_buf (`PathLike` or `FileOrBuffer`):
+ Either a path to a file (e.g. `file.parquet`), a remote URI (e.g. `hf://datasets/username/my_dataset_name/data.parquet`),
+ or a BinaryIO, where the dataset will be saved to in the specified format.
+ batch_size (`int`, *optional*):
+ Size of the batch to load in memory and write at once.
+ By default it aims for row groups with maximum uncompressed byte size of "100MB",
+ defined by `datasets.config.MAX_ROW_GROUP_SIZE`.
+ storage_options (`dict`, *optional*):
+ Key/value pairs to be passed on to the file-system backend, if any.
+
+
+ **parquet_writer_kwargs (additional keyword arguments):
+ Parameters to pass to PyArrow's `pyarrow.parquet.ParquetWriter`.
+
+ Returns:
+ `int`: The number of characters or bytes written.
+
+ Example:
+
+ ```py
+ >>> ds.to_parquet("path/to/dataset/directory")
+ ```
+ """
+ # Dynamic import to avoid circular dependency
+ from .io.parquet import ParquetDatasetWriter
+
+ return ParquetDatasetWriter(
+ self, path_or_buf, batch_size=batch_size, storage_options=storage_options, **parquet_writer_kwargs
+ ).write()
+
+ def to_sql(
+ self,
+ name: str,
+ con: Union[str, "sqlalchemy.engine.Connection", "sqlalchemy.engine.Engine", "sqlite3.Connection"],
+ batch_size: Optional[int] = None,
+ **sql_writer_kwargs,
+ ) -> int:
+ """Exports the dataset to a SQL database.
+
+ Args:
+ name (`str`):
+ Name of SQL table.
+ con (`str` or `sqlite3.Connection` or `sqlalchemy.engine.Connection` or `sqlalchemy.engine.Connection`):
+ A [URI string](https://docs.sqlalchemy.org/en/13/core/engines.html#database-urls) or a SQLite3/SQLAlchemy connection object used to write to a database.
+ batch_size (`int`, *optional*):
+ Size of the batch to load in memory and write at once.
+ Defaults to `datasets.config.DEFAULT_MAX_BATCH_SIZE`.
+ **sql_writer_kwargs (additional keyword arguments):
+ Parameters to pass to pandas's [`pandas.DataFrame.to_sql`](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.to_sql.html).
+
+
+
+ Now, `index` defaults to `False` if not specified.
+
+ If you would like to write the index, pass `index=True` and also set a name for the index column by
+ passing `index_label`.
+
+
+
+ Returns:
+ `int`: The number of records written.
+
+ Example:
+
+ ```py
+ >>> # con provided as a connection URI string
+ >>> ds.to_sql("data", "sqlite:///my_own_db.sql")
+ >>> # con provided as a sqlite3 connection object
+ >>> import sqlite3
+ >>> con = sqlite3.connect("my_own_db.sql")
+ >>> with con:
+ ... ds.to_sql("data", con)
+ ```
+ """
+ # Dynamic import to avoid circular dependency
+ from .io.sql import SqlDatasetWriter
+
+ return SqlDatasetWriter(self, name, con, batch_size=batch_size, **sql_writer_kwargs).write()
+
+ def _estimate_nbytes(self) -> int:
+ dataset_nbytes = self.data.nbytes
+
+ # Find decodable columns, because if there are any, we need to
+ # adjust the dataset size computation (needed for sharding) to account for possible external files
+ decodable_columns = [
+ k for k, v in self._info.features.items() if require_decoding(v, ignore_decode_attribute=True)
+ ]
+
+ if decodable_columns:
+ # Approximate the space needed to store the bytes from the external files by analyzing the first 1000 examples
+ extra_nbytes = 0
+
+ def extra_nbytes_visitor(array, feature):
+ nonlocal extra_nbytes
+ if isinstance(feature, (Audio, Image, Video)):
+ for x in array.to_pylist():
+ if x is not None and x["bytes"] is None and x["path"] is not None:
+ size = xgetsize(x["path"])
+ extra_nbytes += size
+ extra_nbytes -= array.field("path").nbytes
+
+ table = self.with_format("arrow")[:1000]
+ table_visitor(table, extra_nbytes_visitor)
+
+ extra_nbytes = extra_nbytes * len(self.data) // len(table)
+ dataset_nbytes = dataset_nbytes + extra_nbytes
+
+ if self._indices is not None:
+ dataset_nbytes = dataset_nbytes * len(self._indices) // len(self.data)
+ return dataset_nbytes
+
+ @staticmethod
+ def _generate_tables_from_shards(shards: list["Dataset"], batch_size: int):
+ for shard_idx, shard in enumerate(shards):
+ for pa_table in shard.with_format("arrow").iter(batch_size):
+ yield shard_idx, pa_table
+
+ @staticmethod
+ def _generate_tables_from_cache_file(filename: str):
+ for batch_idx, batch in enumerate(_memory_mapped_record_batch_reader_from_file(filename)):
+ yield batch_idx, pa.Table.from_batches([batch])
+
+ def to_iterable_dataset(self, num_shards: Optional[int] = 1) -> "IterableDataset":
+ """Get an [`datasets.IterableDataset`] from a map-style [`datasets.Dataset`].
+ This is equivalent to loading a dataset in streaming mode with [`datasets.load_dataset`], but much faster since the data is streamed from local files.
+
+ Contrary to map-style datasets, iterable datasets are lazy and can only be iterated over (e.g. using a for loop).
+ Since they are read sequentially in training loops, iterable datasets are much faster than map-style datasets.
+ All the transformations applied to iterable datasets like filtering or processing are done on-the-fly when you start iterating over the dataset.
+
+ Still, it is possible to shuffle an iterable dataset using [`datasets.IterableDataset.shuffle`].
+ This is a fast approximate shuffling that works best if you have multiple shards and if you specify a buffer size that is big enough.
+
+ To get the best speed performance, make sure your dataset doesn't have an indices mapping.
+ If this is the case, the data are not read contiguously, which can be slow sometimes.
+ You can use `ds = ds.flatten_indices()` to write your dataset in contiguous chunks of data and have optimal speed before switching to an iterable dataset.
+
+ Args:
+ num_shards (`int`, default to `1`):
+ Number of shards to define when instantiating the iterable dataset. This is especially useful for big datasets to be able to shuffle properly,
+ and also to enable fast parallel loading using a PyTorch DataLoader or in distributed setups for example.
+ Shards are defined using [`datasets.Dataset.shard`]: it simply slices the data without writing anything on disk.
+
+ Returns:
+ [`datasets.IterableDataset`]
+
+ Example:
+
+ Basic usage:
+ ```python
+ >>> ids = ds.to_iterable_dataset()
+ >>> for example in ids:
+ ... pass
+ ```
+
+ With lazy filtering and processing:
+ ```python
+ >>> ids = ds.to_iterable_dataset()
+ >>> ids = ids.filter(filter_fn).map(process_fn) # will filter and process on-the-fly when you start iterating over the iterable dataset
+ >>> for example in ids:
+ ... pass
+ ```
+
+ With sharding to enable efficient shuffling:
+ ```python
+ >>> ids = ds.to_iterable_dataset(num_shards=64) # the dataset is split into 64 shards to be iterated over
+ >>> ids = ids.shuffle(buffer_size=10_000) # will shuffle the shards order and use a shuffle buffer for fast approximate shuffling when you start iterating
+ >>> for example in ids:
+ ... pass
+ ```
+
+ With a PyTorch DataLoader:
+ ```python
+ >>> import torch
+ >>> ids = ds.to_iterable_dataset(num_shards=64)
+ >>> ids = ids.filter(filter_fn).map(process_fn)
+ >>> dataloader = torch.utils.data.DataLoader(ids, num_workers=4) # will assign 64 / 4 = 16 shards to each worker to load, filter and process when you start iterating
+ >>> for example in ids:
+ ... pass
+ ```
+
+ With a PyTorch DataLoader and shuffling:
+ ```python
+ >>> import torch
+ >>> ids = ds.to_iterable_dataset(num_shards=64)
+ >>> ids = ids.shuffle(buffer_size=10_000) # will shuffle the shards order and use a shuffle buffer when you start iterating
+ >>> dataloader = torch.utils.data.DataLoader(ids, num_workers=4) # will assign 64 / 4 = 16 shards from the shuffled list of shards to each worker when you start iterating
+ >>> for example in ids:
+ ... pass
+ ```
+
+ In a distributed setup like PyTorch DDP with a PyTorch DataLoader and shuffling
+ ```python
+ >>> from datasets.distributed import split_dataset_by_node
+ >>> ids = ds.to_iterable_dataset(num_shards=512)
+ >>> ids = ids.shuffle(buffer_size=10_000, seed=42) # will shuffle the shards order and use a shuffle buffer when you start iterating
+ >>> ids = split_dataset_by_node(ds, world_size=8, rank=0) # will keep only 512 / 8 = 64 shards from the shuffled lists of shards when you start iterating
+ >>> dataloader = torch.utils.data.DataLoader(ids, num_workers=4) # will assign 64 / 4 = 16 shards from this node's list of shards to each worker when you start iterating
+ >>> for example in ids:
+ ... pass
+ ```
+
+ With shuffling and multiple epochs:
+ ```python
+ >>> ids = ds.to_iterable_dataset(num_shards=64)
+ >>> ids = ids.shuffle(buffer_size=10_000, seed=42) # will shuffle the shards order and use a shuffle buffer when you start iterating
+ >>> for epoch in range(n_epochs):
+ ... ids.set_epoch(epoch) # will use effective_seed = seed + epoch to shuffle the shards and for the shuffle buffer when you start iterating
+ ... for example in ids:
+ ... pass
+ ```
+ Feel free to also use [`IterableDataset.set_epoch`] when using a PyTorch DataLoader or in distributed setups.
+ """
+ from .iterable_dataset import ArrowExamplesIterable, IterableDataset
+
+ if self._format_type is not None:
+ if self._format_kwargs or (
+ self._format_columns is not None and set(self._format_columns) != set(self.column_names)
+ ):
+ raise NotImplementedError(
+ "Converting a formatted dataset with kwargs or selected columns to a formatted iterable dataset is not implemented yet. Please run `my_dataset = my_dataset.with_format(None)` before calling to_iterable_dataset"
+ )
+ if num_shards > len(self):
+ raise ValueError(
+ f"Unable to shard a dataset of size {len(self)} into {num_shards} shards (the number of shards exceeds the number of samples)."
+ )
+ if self._indices is not None:
+ logger.info(
+ "Converting an Arrow dataset to iterable but it has an indices mapping that can make it slower. "
+ "You can use `ds = ds.flatten_indices()` to write your dataset in contiguous chunks of data and have optimal speed."
+ )
+ shards = (
+ [copy.deepcopy(self)]
+ if num_shards == 1
+ else [
+ self.shard(num_shards=num_shards, index=shard_idx, contiguous=True) for shard_idx in range(num_shards)
+ ]
+ )
+ ex_iterable = ArrowExamplesIterable(
+ Dataset._generate_tables_from_shards,
+ kwargs={"shards": shards, "batch_size": config.DEFAULT_MAX_BATCH_SIZE},
+ )
+ ds = IterableDataset(ex_iterable, info=DatasetInfo(features=self.features))
+ if self._format_type:
+ ds = ds.with_format(self._format_type)
+ return ds
+
+ def _push_parquet_shards_to_hub_single(
+ self,
+ job_id: int,
+ num_jobs: int,
+ repo_id: str,
+ data_dir: str,
+ split: str,
+ token: Optional[str],
+ revision: Optional[str],
+ create_pr: Optional[bool],
+ num_shards: int,
+ embed_external_files: bool,
+ writer_batch_size: int,
+ ):
+ div = num_shards // num_jobs
+ mod = num_shards % num_jobs
+ start = div * job_id + min(job_id, mod)
+ end = start + div + (1 if job_id < mod else 0)
+
+ index_shards = (
+ (start + i, self.shard(num_shards=end - start, index=i, contiguous=True)) for i in range(end - start)
+ )
+
+ api = HfApi(endpoint=config.HF_ENDPOINT, token=token)
+
+ uploaded_size = 0
+ additions: list[CommitOperationAdd] = []
+ for index, shard in index_shards:
+ if embed_external_files:
+ format = shard.format
+ shard = shard.with_format("arrow")
+ shard = shard.map(
+ embed_table_storage,
+ batched=True,
+ batch_size=writer_batch_size,
+ keep_in_memory=True,
+ )
+ shard = shard.with_format(**format)
+ shard_path_in_repo = f"{data_dir}/{split}-{index:05d}-of-{num_shards:05d}.parquet"
+ buffer = BytesIO()
+ shard.to_parquet(buffer, batch_size=writer_batch_size)
+ parquet_content = buffer.getvalue()
+ uploaded_size += len(parquet_content)
+ del buffer
+ shard_addition = CommitOperationAdd(path_in_repo=shard_path_in_repo, path_or_fileobj=parquet_content)
+ api.preupload_lfs_files(
+ repo_id=repo_id,
+ additions=[shard_addition],
+ repo_type="dataset",
+ revision=revision,
+ create_pr=create_pr,
+ )
+ additions.append(shard_addition)
+ yield job_id, False, 1
+
+ yield job_id, True, additions
+
+ def _push_parquet_shards_to_hub(
+ self,
+ repo_id: str,
+ data_dir: str,
+ split: str,
+ token: Optional[str],
+ revision: Optional[str],
+ create_pr: Optional[bool],
+ max_shard_size: Optional[Union[int, str]],
+ num_shards: Optional[int],
+ embed_external_files: bool,
+ num_proc: Optional[int],
+ ) -> tuple[list[CommitOperationAdd], int, int]:
+ """Pushes the dataset shards as Parquet files to the hub.
+
+ Returns:
+ additions (`List[CommitOperation]`): list of the `CommitOperationAdd` of the uploaded shards
+ uploaded_size (`int`): number of uploaded bytes to the repository
+ dataset_nbytes (`int`): approximate size in bytes of the uploaded dataset after uncompression
+ """
+ from .arrow_writer import get_writer_batch_size_from_data_size, get_writer_batch_size_from_features
+
+ dataset_nbytes = self._estimate_nbytes()
+ writer_batch_size = get_writer_batch_size_from_features(self.features) or get_writer_batch_size_from_data_size(
+ len(self), dataset_nbytes
+ )
+
+ # Find decodable columns, because if there are any, we need to:
+ # embed the bytes from the files in the shards
+ decodable_columns = (
+ [k for k, v in self._info.features.items() if require_decoding(v, ignore_decode_attribute=True)]
+ if embed_external_files
+ else []
+ )
+ embed_external_files = embed_external_files and bool(decodable_columns)
+
+ if num_shards is None:
+ max_shard_size = convert_file_size_to_int(max_shard_size or config.MAX_SHARD_SIZE)
+ num_shards = int(dataset_nbytes / max_shard_size) + 1
+ num_shards = max(num_shards, num_proc or 1)
+
+ additions: list[CommitOperationAdd] = []
+
+ num_jobs = num_proc or 1
+ kwargs_iterable = [
+ {
+ "self": self.shard(num_shards=num_jobs, index=job_id, contiguous=True),
+ "job_id": job_id,
+ "num_jobs": num_jobs,
+ "repo_id": repo_id,
+ "data_dir": data_dir,
+ "split": split,
+ "token": token,
+ "revision": revision,
+ "create_pr": create_pr,
+ "num_shards": num_shards,
+ "embed_external_files": embed_external_files,
+ "writer_batch_size": writer_batch_size,
+ }
+ for job_id in range(num_jobs)
+ ]
+ desc = "Uploading the dataset shards"
+ desc += f" (num_proc={num_proc})" if num_proc is not None and num_proc >= 1 else ""
+ pbar = hf_tqdm(
+ unit=" shards",
+ total=num_shards,
+ desc=desc,
+ )
+ with contextlib.nullcontext() if num_proc is None or num_proc < 1 else Pool(num_proc) as pool:
+ update_stream = (
+ Dataset._push_parquet_shards_to_hub_single(**kwargs_iterable[0])
+ if pool is None
+ else iflatmap_unordered(
+ pool,
+ Dataset._push_parquet_shards_to_hub_single,
+ kwargs_iterable=kwargs_iterable,
+ )
+ )
+ for job_id, done, content in update_stream:
+ if not done:
+ pbar.update(content)
+ else:
+ additions += content
+
+ uploaded_size = sum(addition.upload_info.size for addition in additions)
+ return additions, uploaded_size, dataset_nbytes
+
+ def push_to_hub(
+ self,
+ repo_id: str,
+ config_name: str = "default",
+ set_default: Optional[bool] = None,
+ split: Optional[str] = None,
+ data_dir: Optional[str] = None,
+ commit_message: Optional[str] = None,
+ commit_description: Optional[str] = None,
+ private: Optional[bool] = None,
+ token: Optional[str] = None,
+ revision: Optional[str] = None,
+ create_pr: Optional[bool] = False,
+ max_shard_size: Optional[Union[int, str]] = None,
+ num_shards: Optional[int] = None,
+ embed_external_files: bool = True,
+ num_proc: Optional[int] = None,
+ ) -> CommitInfo:
+ """Pushes the dataset to the hub as a Parquet dataset.
+ The dataset is pushed using HTTP requests and does not need to have neither git or git-lfs installed.
+
+ The resulting Parquet files are self-contained by default. If your dataset contains [`Image`], [`Audio`] or [`Video`]
+ data, the Parquet files will store the bytes of your images or audio files.
+ You can disable this by setting `embed_external_files` to `False`.
+
+ Args:
+ repo_id (`str`):
+ The ID of the repository to push to in the following format: `/` or
+ `/`. Also accepts ``, which will default to the namespace
+ of the logged-in user.
+ config_name (`str`, defaults to "default"):
+ The configuration name (or subset) of a dataset. Defaults to "default".
+ set_default (`bool`, *optional*):
+ Whether to set this configuration as the default one. Otherwise, the default configuration is the one
+ named "default".
+ split (`str`, *optional*):
+ The name of the split that will be given to that dataset. Defaults to `self.split`.
+ data_dir (`str`, *optional*):
+ Directory name that will contain the uploaded data files. Defaults to the `config_name` if different
+ from "default", else "data".
+
+
+ commit_message (`str`, *optional*):
+ Message to commit while pushing. Will default to `"Upload dataset"`.
+ commit_description (`str`, *optional*):
+ Description of the commit that will be created.
+ Additionally, description of the PR if a PR is created (`create_pr` is True).
+
+
+ private (`bool`, *optional*):
+ Whether to make the repo private. If `None` (default), the repo will be public unless the
+ organization's default is private. This value is ignored if the repo already exists.
+ token (`str`, *optional*):
+ An optional authentication token for the Hugging Face Hub. If no token is passed, will default
+ to the token saved locally when logging in with `huggingface-cli login`. Will raise an error
+ if no token is passed and the user is not logged-in.
+ revision (`str`, *optional*):
+ Branch to push the uploaded files to. Defaults to the `"main"` branch.
+
+
+ create_pr (`bool`, *optional*, defaults to `False`):
+ Whether to create a PR with the uploaded files or directly commit.
+
+
+ max_shard_size (`int` or `str`, *optional*, defaults to `"500MB"`):
+ The maximum size of the dataset shards to be uploaded to the hub. If expressed as a string, needs to be digits followed by
+ a unit (like `"5MB"`).
+ num_shards (`int`, *optional*):
+ Number of shards to write. By default, the number of shards depends on `max_shard_size`.
+
+
+ embed_external_files (`bool`, defaults to `True`):
+ Whether to embed file bytes in the shards.
+ In particular, this will do the following before the push for the fields of type:
+
+ - [`Audio`] and [`Image`]: remove local path information and embed file content in the Parquet files.
+ num_proc (`int`, *optional*, defaults to `None`):
+ Number of processes when preparing and uploading the dataset.
+ This is helpful if the dataset is made of many samples or media files to embed.
+ Multiprocessing is disabled by default.
+
+
+
+ Return:
+ huggingface_hub.CommitInfo
+
+ Example:
+
+ ```python
+ >>> dataset.push_to_hub("/")
+ >>> dataset_dict.push_to_hub("/", private=True)
+ >>> dataset.push_to_hub("/", max_shard_size="1GB")
+ >>> dataset.push_to_hub("/", num_shards=1024)
+ ```
+
+ If your dataset has multiple splits (e.g. train/validation/test):
+
+ ```python
+ >>> train_dataset.push_to_hub("/", split="train")
+ >>> val_dataset.push_to_hub("/", split="validation")
+ >>> # later
+ >>> dataset = load_dataset("/")
+ >>> train_dataset = dataset["train"]
+ >>> val_dataset = dataset["validation"]
+ ```
+
+ If you want to add a new configuration (or subset) to a dataset (e.g. if the dataset has multiple tasks/versions/languages):
+
+ ```python
+ >>> english_dataset.push_to_hub("/", "en")
+ >>> french_dataset.push_to_hub("/", "fr")
+ >>> # later
+ >>> english_dataset = load_dataset("/", "en")
+ >>> french_dataset = load_dataset("/", "fr")
+ ```
+ """
+ if "Video(" in str(self.features):
+ raise NotImplementedError(
+ "push_to_hub is not implemented for video datasets, instead you should upload the video files "
+ "using e.g. the huggingface_hub library and optionally upload a metadata.csv or metadata.jsonl "
+ "file containing other information like video captions, features or labels. More information "
+ "at https://huggingface.co/docs/datasets/main/en/video_load#videofolder"
+ )
+ if config_name == "data":
+ raise ValueError("`config_name` cannot be 'data'. Please, choose another name for configuration.")
+
+ if max_shard_size is not None and num_shards is not None:
+ raise ValueError(
+ "Failed to push_to_hub: please specify either max_shard_size or num_shards, but not both."
+ )
+
+ if split is None:
+ split = str(self.split) if self.split is not None else "train"
+
+ if not re.match(_split_re, split):
+ raise ValueError(f"Split name should match '{_split_re}' but got '{split}'.")
+
+ api = HfApi(endpoint=config.HF_ENDPOINT, token=token)
+
+ try:
+ repo_id = api.repo_info(repo_id, repo_type="dataset").id
+ except RepositoryNotFoundError:
+ repo_url = api.create_repo(
+ repo_id,
+ repo_type="dataset",
+ private=private,
+ exist_ok=True,
+ )
+ repo_id = repo_url.repo_id
+
+ if revision is not None and not revision.startswith("refs/pr/"):
+ # We do not call create_branch for a PR reference: 400 Bad Request
+ api.create_branch(repo_id, branch=revision, repo_type="dataset", exist_ok=True)
+
+ if not data_dir:
+ data_dir = config_name if config_name != "default" else "data" # for backward compatibility
+
+ additions, uploaded_size, dataset_nbytes = self._push_parquet_shards_to_hub(
+ repo_id=repo_id,
+ data_dir=data_dir,
+ split=split,
+ token=token,
+ revision=revision,
+ max_shard_size=max_shard_size,
+ num_shards=num_shards,
+ create_pr=create_pr,
+ embed_external_files=embed_external_files,
+ num_proc=num_proc,
+ )
+
+ def get_deletions_and_dataset_card() -> tuple[str, list[CommitOperationDelete], str, Optional[str]]:
+ parent_commit = api.repo_info(repo_id, repo_type="dataset", revision=revision).sha
+
+ # Check if the repo already has a README.md and/or a dataset_infos.json to update them with the new split info (size and pattern)
+ # and delete old split shards (if they exist)
+ repo_with_dataset_card, repo_with_dataset_infos = False, False
+ deletions: list[CommitOperationDelete] = []
+ deleted_size = 0
+ repo_splits: list[str] = [] # use a list to keep the order of the splits
+ repo_files_to_add = [addition.path_in_repo for addition in additions]
+ for repo_file in api.list_repo_tree(
+ repo_id=repo_id, revision=parent_commit, repo_type="dataset", token=token, recursive=True
+ ):
+ if not isinstance(repo_file, RepoFile):
+ continue
+ if repo_file.rfilename == config.REPOCARD_FILENAME:
+ repo_with_dataset_card = True
+ elif repo_file.rfilename == config.DATASETDICT_INFOS_FILENAME:
+ repo_with_dataset_infos = True
+ elif (
+ repo_file.rfilename.startswith(f"{data_dir}/{split}-")
+ and repo_file.rfilename not in repo_files_to_add
+ ):
+ deletions.append(CommitOperationDelete(path_in_repo=repo_file.rfilename))
+ deleted_size += repo_file.size
+ elif fnmatch.fnmatch(
+ repo_file.rfilename,
+ PUSH_TO_HUB_WITHOUT_METADATA_CONFIGS_SPLIT_PATTERN_SHARDED.replace("{split}", "*"),
+ ):
+ pattern = glob_pattern_to_regex(PUSH_TO_HUB_WITHOUT_METADATA_CONFIGS_SPLIT_PATTERN_SHARDED)
+ split_pattern_fields = string_to_dict(repo_file.rfilename, pattern)
+ assert split_pattern_fields is not None
+ repo_split = split_pattern_fields["split"]
+ if repo_split not in repo_splits:
+ repo_splits.append(repo_split)
+
+ organization, dataset_name = repo_id.split("/") if "/" in repo_id else (None, repo_id)
+ info_to_dump = self.info.copy()
+ info_to_dump.download_checksums = None
+ info_to_dump.download_size = uploaded_size
+ info_to_dump.dataset_size = dataset_nbytes
+ info_to_dump.size_in_bytes = uploaded_size + dataset_nbytes
+ info_to_dump.config_name = config_name
+ info_to_dump.splits = SplitDict(
+ {split: SplitInfo(split, num_bytes=dataset_nbytes, num_examples=len(self), dataset_name=dataset_name)}
+ )
+ # get the info from the README to update them
+ if repo_with_dataset_card:
+ dataset_card_path = api.hf_hub_download(
+ repo_id, config.REPOCARD_FILENAME, repo_type="dataset", revision=parent_commit
+ )
+ dataset_card = DatasetCard.load(Path(dataset_card_path))
+ dataset_card_data = dataset_card.data
+ metadata_configs = MetadataConfigs.from_dataset_card_data(dataset_card_data)
+ dataset_infos: DatasetInfosDict = DatasetInfosDict.from_dataset_card_data(dataset_card_data)
+ if dataset_infos and config_name in dataset_infos:
+ repo_info = dataset_infos[config_name]
+ else:
+ repo_info = None
+ # get the deprecated dataset_infos.json to update them
+ elif repo_with_dataset_infos:
+ dataset_card = None
+ dataset_card_data = DatasetCardData()
+ metadata_configs = MetadataConfigs()
+ dataset_infos_path = api.hf_hub_download(
+ repo_id, config.DATASETDICT_INFOS_FILENAME, repo_type="dataset", revision=parent_commit
+ )
+ with open(dataset_infos_path, encoding="utf-8") as f:
+ dataset_infos: dict = json.load(f)
+ dataset_info = dataset_infos.get(config_name, None) if dataset_infos else None
+ repo_info = DatasetInfo.from_dict(dataset_info) if dataset_info else None
+ else:
+ dataset_card = None
+ dataset_card_data = DatasetCardData()
+ metadata_configs = MetadataConfigs()
+ repo_info = None
+ # update the total info to dump from existing info
+ if repo_info is not None:
+ logger.info("Updating downloaded metadata with the new split.")
+ if repo_info.splits and list(repo_info.splits) != [split]:
+ if self._info.features != repo_info.features:
+ raise ValueError(
+ f"Features of the new split don't match the features of the existing splits on the hub: {self._info.features} != {repo_info.features}"
+ )
+
+ if split in repo_info.splits:
+ repo_info.download_size -= deleted_size
+ repo_info.dataset_size -= repo_info.splits.get(split, SplitInfo()).num_bytes or 0
+
+ repo_info.download_checksums = None
+ repo_info.download_size = (repo_info.download_size or 0) + uploaded_size
+ repo_info.dataset_size = (repo_info.dataset_size or 0) + dataset_nbytes
+ repo_info.size_in_bytes = repo_info.download_size + repo_info.dataset_size
+ repo_info.splits.pop(split, None)
+ repo_info.splits[split] = SplitInfo(
+ split, num_bytes=dataset_nbytes, num_examples=len(self), dataset_name=dataset_name
+ )
+ info_to_dump = repo_info
+ # create the metadata configs if it was uploaded with push_to_hub before metadata configs existed
+ if not metadata_configs and repo_splits:
+ default_metadata_configs_to_dump = {
+ "data_files": [{"split": split, "path": f"data/{split}-*"} for split in repo_splits]
+ }
+ MetadataConfigs({"default": default_metadata_configs_to_dump}).to_dataset_card_data(dataset_card_data)
+ # update the metadata configs
+ if config_name in metadata_configs:
+ metadata_config = metadata_configs[config_name]
+ if "data_files" in metadata_config:
+ data_files_to_dump = sanitize_patterns(metadata_config["data_files"])
+ else:
+ data_files_to_dump = {}
+ # add the new split
+ data_files_to_dump[split] = [f"{data_dir}/{split}-*"]
+ metadata_config_to_dump = {
+ "data_files": [
+ {
+ "split": _split,
+ "path": _pattern[0] if len(_pattern) == 1 else _pattern,
+ }
+ for _split, _pattern in data_files_to_dump.items()
+ ]
+ }
+ else:
+ metadata_config_to_dump = {"data_files": [{"split": split, "path": f"{data_dir}/{split}-*"}]}
+ configs_to_dump = {config_name: metadata_config_to_dump}
+ if set_default and config_name != "default":
+ if metadata_configs:
+ current_default_config_name = metadata_configs.get_default_config_name()
+ if current_default_config_name == "default":
+ raise ValueError(
+ "There exists a configuration named 'default'. To set a different configuration as default, "
+ "rename the 'default' one first."
+ )
+ if current_default_config_name:
+ _ = metadata_configs[current_default_config_name].pop("default")
+ configs_to_dump[current_default_config_name] = metadata_configs[current_default_config_name]
+ metadata_config_to_dump["default"] = True
+ # push to the deprecated dataset_infos.json
+ if repo_with_dataset_infos:
+ dataset_infos_path = api.hf_hub_download(
+ repo_id, config.DATASETDICT_INFOS_FILENAME, repo_type="dataset", revision=parent_commit
+ )
+ with open(dataset_infos_path, encoding="utf-8") as f:
+ dataset_infos: dict = json.load(f)
+ dataset_infos[config_name] = asdict(info_to_dump)
+ new_dataset_infos = json.dumps(dataset_infos, indent=4)
+ else:
+ new_dataset_infos = None
+ # push to README
+ DatasetInfosDict({config_name: info_to_dump}).to_dataset_card_data(dataset_card_data)
+ MetadataConfigs(configs_to_dump).to_dataset_card_data(dataset_card_data)
+ new_dataset_card = (
+ DatasetCard(f"---\n{dataset_card_data}\n---\n") if dataset_card is None else dataset_card
+ )
+ return parent_commit, deletions, new_dataset_card, new_dataset_infos
+
+ commit_message = commit_message if commit_message is not None else "Upload dataset"
+ if len(additions) > config.UPLOADS_MAX_NUMBER_PER_COMMIT:
+ logger.info(
+ f"Number of files to upload is larger than {config.UPLOADS_MAX_NUMBER_PER_COMMIT}. Splitting the push into multiple commits."
+ )
+ num_commits = math.ceil(len(additions) / config.UPLOADS_MAX_NUMBER_PER_COMMIT)
+ for i in range(0, num_commits):
+ operations = additions[
+ i * config.UPLOADS_MAX_NUMBER_PER_COMMIT : (i + 1) * config.UPLOADS_MAX_NUMBER_PER_COMMIT
+ ]
+ for retry, sleep_time in enumerate(itertools.chain(range(10), itertools.repeat(30)), start=1):
+ # We need to retry if another commit happens at the same time
+ sleep_time *= 1 + random.random()
+ try:
+ commit_info = api.create_commit(
+ repo_id,
+ operations=operations,
+ commit_message=commit_message + f" (part {i:05d}-of-{num_commits:05d})",
+ commit_description=commit_description,
+ repo_type="dataset",
+ revision=revision,
+ create_pr=create_pr,
+ )
+ except HfHubHTTPError as err:
+ if (
+ err.__context__
+ and isinstance(err.__context__, HfHubHTTPError)
+ and err.__context__.response.status_code == 409
+ ):
+ # 409 is Conflict (another commit is in progress)
+ time.sleep(sleep_time)
+ logger.info(
+ f"Retrying intermediate commit for {repo_id}, {config_name} ({retry}/n with status_code {err.__context__.response.status_code})"
+ )
+ continue
+ else:
+ raise
+ break
+ logger.info(
+ f"Commit #{i + 1} completed"
+ + (f" (still {num_commits - i - 1} to go)" if num_commits - i - 1 else "")
+ + "."
+ )
+ last_commit_additions = []
+ else:
+ last_commit_additions = additions
+
+ for retry, sleep_time in enumerate(itertools.chain(range(10), itertools.repeat(30)), start=1):
+ # We need to retry if there was a commit in between in case it touched the dataset card data
+ sleep_time *= 1 + random.random()
+ parent_commit, deletions, dataset_card, dataset_infos = get_deletions_and_dataset_card()
+ dataset_card_additions = []
+ if dataset_infos:
+ dataset_card_additions.append(
+ CommitOperationAdd(
+ path_in_repo=config.DATASETDICT_INFOS_FILENAME,
+ path_or_fileobj=dataset_infos.encode("utf-8"),
+ )
+ )
+ dataset_card_additions.append(
+ CommitOperationAdd(path_in_repo=config.REPOCARD_FILENAME, path_or_fileobj=str(dataset_card).encode())
+ )
+ try:
+ commit_info = api.create_commit(
+ repo_id,
+ operations=last_commit_additions + dataset_card_additions + deletions,
+ commit_message=commit_message,
+ commit_description=commit_description,
+ repo_type="dataset",
+ revision=revision,
+ create_pr=create_pr,
+ parent_commit=parent_commit,
+ )
+ except HfHubHTTPError as err:
+ if (
+ err.__context__
+ and isinstance(err.__context__, HfHubHTTPError)
+ and err.__context__.response.status_code in (412, 409)
+ ):
+ # 412 is Precondition failed (parent_commit isn't satisfied)
+ # 409 is Conflict (another commit is in progress)
+ time.sleep(sleep_time)
+ logger.info(
+ f"Retrying commit for {repo_id}, {config_name} ({retry}/n with status_code {err.__context__.response.status_code})"
+ )
+ continue
+ else:
+ raise
+ break
+
+ return commit_info
+
+ @transmit_format
+ @fingerprint_transform(inplace=False)
+ def add_column(
+ self, name: str, column: Union[list, np.ndarray], new_fingerprint: str, feature: Optional[FeatureType] = None
+ ):
+ """Add column to Dataset.
+
+
+
+ Args:
+ name (`str`):
+ Column name.
+ column (`list` or `np.array`):
+ Column data to be added.
+ feature (`FeatureType` or `None`, defaults to `None`):
+ Column datatype.
+
+ Returns:
+ [`Dataset`]
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> more_text = ds["text"]
+ >>> ds = ds.add_column(name="text_2", column=more_text)
+ >>> ds
+ Dataset({
+ features: ['text', 'label', 'text_2'],
+ num_rows: 1066
+ })
+ ```
+ """
+
+ if feature:
+ pyarrow_schema = Features({name: feature}).arrow_schema
+ else:
+ pyarrow_schema = None
+
+ column_table = InMemoryTable.from_pydict({name: column}, schema=pyarrow_schema)
+ _check_column_names(self._data.column_names + column_table.column_names)
+ dataset = self.flatten_indices() if self._indices is not None else self
+ # Concatenate tables horizontally
+ table = concat_tables([dataset._data, column_table], axis=1)
+ # Update features
+ info = dataset.info.copy()
+ info.features.update(Features.from_arrow_schema(column_table.schema))
+ table = update_metadata_with_features(table, info.features)
+ return Dataset(table, info=info, split=self.split, indices_table=None, fingerprint=new_fingerprint)
+
+ def add_faiss_index(
+ self,
+ column: str,
+ index_name: Optional[str] = None,
+ device: Optional[int] = None,
+ string_factory: Optional[str] = None,
+ metric_type: Optional[int] = None,
+ custom_index: Optional["faiss.Index"] = None, # noqa: F821
+ batch_size: int = 1000,
+ train_size: Optional[int] = None,
+ faiss_verbose: bool = False,
+ dtype=np.float32,
+ ):
+ """Add a dense index using Faiss for fast retrieval.
+ By default the index is done over the vectors of the specified column.
+ You can specify `device` if you want to run it on GPU (`device` must be the GPU index).
+ You can find more information about Faiss here:
+
+ - For [string factory](https://github.com/facebookresearch/faiss/wiki/The-index-factory)
+
+ Args:
+ column (`str`):
+ The column of the vectors to add to the index.
+ index_name (`str`, *optional*):
+ The `index_name`/identifier of the index.
+ This is the `index_name` that is used to call [`~datasets.Dataset.get_nearest_examples`] or [`~datasets.Dataset.search`].
+ By default it corresponds to `column`.
+ device (`Union[int, List[int]]`, *optional*):
+ If positive integer, this is the index of the GPU to use. If negative integer, use all GPUs.
+ If a list of positive integers is passed in, run only on those GPUs. By default it uses the CPU.
+ string_factory (`str`, *optional*):
+ This is passed to the index factory of Faiss to create the index.
+ Default index class is `IndexFlat`.
+ metric_type (`int`, *optional*):
+ Type of metric. Ex: `faiss.METRIC_INNER_PRODUCT` or `faiss.METRIC_L2`.
+ custom_index (`faiss.Index`, *optional*):
+ Custom Faiss index that you already have instantiated and configured for your needs.
+ batch_size (`int`):
+ Size of the batch to use while adding vectors to the `FaissIndex`. Default value is `1000`.
+
+ train_size (`int`, *optional*):
+ If the index needs a training step, specifies how many vectors will be used to train the index.
+ faiss_verbose (`bool`, defaults to `False`):
+ Enable the verbosity of the Faiss index.
+ dtype (`data-type`):
+ The dtype of the numpy arrays that are indexed.
+ Default is `np.float32`.
+
+ Example:
+
+ ```python
+ >>> ds = datasets.load_dataset('crime_and_punish', split='train')
+ >>> ds_with_embeddings = ds.map(lambda example: {'embeddings': embed(example['line']}))
+ >>> ds_with_embeddings.add_faiss_index(column='embeddings')
+ >>> # query
+ >>> scores, retrieved_examples = ds_with_embeddings.get_nearest_examples('embeddings', embed('my new query'), k=10)
+ >>> # save index
+ >>> ds_with_embeddings.save_faiss_index('embeddings', 'my_index.faiss')
+
+ >>> ds = datasets.load_dataset('crime_and_punish', split='train')
+ >>> # load index
+ >>> ds.load_faiss_index('embeddings', 'my_index.faiss')
+ >>> # query
+ >>> scores, retrieved_examples = ds.get_nearest_examples('embeddings', embed('my new query'), k=10)
+ ```
+ """
+ with self.formatted_as(type="numpy", columns=[column], dtype=dtype):
+ super().add_faiss_index(
+ column=column,
+ index_name=index_name,
+ device=device,
+ string_factory=string_factory,
+ metric_type=metric_type,
+ custom_index=custom_index,
+ batch_size=batch_size,
+ train_size=train_size,
+ faiss_verbose=faiss_verbose,
+ )
+ return self
+
+ def add_faiss_index_from_external_arrays(
+ self,
+ external_arrays: np.array,
+ index_name: str,
+ device: Optional[int] = None,
+ string_factory: Optional[str] = None,
+ metric_type: Optional[int] = None,
+ custom_index: Optional["faiss.Index"] = None, # noqa: F821
+ batch_size: int = 1000,
+ train_size: Optional[int] = None,
+ faiss_verbose: bool = False,
+ dtype=np.float32,
+ ):
+ """Add a dense index using Faiss for fast retrieval.
+ The index is created using the vectors of `external_arrays`.
+ You can specify `device` if you want to run it on GPU (`device` must be the GPU index).
+ You can find more information about Faiss here:
+
+ - For [string factory](https://github.com/facebookresearch/faiss/wiki/The-index-factory)
+
+ Args:
+ external_arrays (`np.array`):
+ If you want to use arrays from outside the lib for the index, you can set `external_arrays`.
+ It will use `external_arrays` to create the Faiss index instead of the arrays in the given `column`.
+ index_name (`str`):
+ The `index_name`/identifier of the index.
+ This is the `index_name` that is used to call [`~datasets.Dataset.get_nearest_examples`] or [`~datasets.Dataset.search`].
+ device (Optional `Union[int, List[int]]`, *optional*):
+ If positive integer, this is the index of the GPU to use. If negative integer, use all GPUs.
+ If a list of positive integers is passed in, run only on those GPUs. By default it uses the CPU.
+ string_factory (`str`, *optional*):
+ This is passed to the index factory of Faiss to create the index.
+ Default index class is `IndexFlat`.
+ metric_type (`int`, *optional*):
+ Type of metric. Ex: `faiss.faiss.METRIC_INNER_PRODUCT` or `faiss.METRIC_L2`.
+ custom_index (`faiss.Index`, *optional*):
+ Custom Faiss index that you already have instantiated and configured for your needs.
+ batch_size (`int`, *optional*):
+ Size of the batch to use while adding vectors to the FaissIndex. Default value is 1000.
+
+ train_size (`int`, *optional*):
+ If the index needs a training step, specifies how many vectors will be used to train the index.
+ faiss_verbose (`bool`, defaults to False):
+ Enable the verbosity of the Faiss index.
+ dtype (`numpy.dtype`):
+ The dtype of the numpy arrays that are indexed. Default is np.float32.
+ """
+ super().add_faiss_index_from_external_arrays(
+ external_arrays=external_arrays.astype(dtype),
+ index_name=index_name,
+ device=device,
+ string_factory=string_factory,
+ metric_type=metric_type,
+ custom_index=custom_index,
+ batch_size=batch_size,
+ train_size=train_size,
+ faiss_verbose=faiss_verbose,
+ )
+
+ def add_elasticsearch_index(
+ self,
+ column: str,
+ index_name: Optional[str] = None,
+ host: Optional[str] = None,
+ port: Optional[int] = None,
+ es_client: Optional["elasticsearch.Elasticsearch"] = None, # noqa: F821
+ es_index_name: Optional[str] = None,
+ es_index_config: Optional[dict] = None,
+ ):
+ """Add a text index using ElasticSearch for fast retrieval. This is done in-place.
+
+ Args:
+ column (`str`):
+ The column of the documents to add to the index.
+ index_name (`str`, *optional*):
+ The `index_name`/identifier of the index.
+ This is the index name that is used to call [`~Dataset.get_nearest_examples`] or [`~Dataset.search`].
+ By default it corresponds to `column`.
+ host (`str`, *optional*, defaults to `localhost`):
+ Host of where ElasticSearch is running.
+ port (`str`, *optional*, defaults to `9200`):
+ Port of where ElasticSearch is running.
+ es_client (`elasticsearch.Elasticsearch`, *optional*):
+ The elasticsearch client used to create the index if host and port are `None`.
+ es_index_name (`str`, *optional*):
+ The elasticsearch index name used to create the index.
+ es_index_config (`dict`, *optional*):
+ The configuration of the elasticsearch index.
+ Default config is:
+ ```
+ {
+ "settings": {
+ "number_of_shards": 1,
+ "analysis": {"analyzer": {"stop_standard": {"type": "standard", " stopwords": "_english_"}}},
+ },
+ "mappings": {
+ "properties": {
+ "text": {
+ "type": "text",
+ "analyzer": "standard",
+ "similarity": "BM25"
+ },
+ }
+ },
+ }
+ ```
+ Example:
+
+ ```python
+ >>> es_client = elasticsearch.Elasticsearch()
+ >>> ds = datasets.load_dataset('crime_and_punish', split='train')
+ >>> ds.add_elasticsearch_index(column='line', es_client=es_client, es_index_name="my_es_index")
+ >>> scores, retrieved_examples = ds.get_nearest_examples('line', 'my new query', k=10)
+ ```
+ """
+ with self.formatted_as(type=None, columns=[column]):
+ super().add_elasticsearch_index(
+ column=column,
+ index_name=index_name,
+ host=host,
+ port=port,
+ es_client=es_client,
+ es_index_name=es_index_name,
+ es_index_config=es_index_config,
+ )
+ return self
+
+ @transmit_format
+ @fingerprint_transform(inplace=False)
+ def add_item(self, item: dict, new_fingerprint: str):
+ """Add item to Dataset.
+
+
+
+ Args:
+ item (`dict`):
+ Item data to be added.
+
+ Returns:
+ [`Dataset`]
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> new_review = {'label': 0, 'text': 'this movie is the absolute worst thing I have ever seen'}
+ >>> ds = ds.add_item(new_review)
+ >>> ds[-1]
+ {'label': 0, 'text': 'this movie is the absolute worst thing I have ever seen'}
+ ```
+ """
+ item_table = InMemoryTable.from_pydict({k: [v] for k, v in item.items()})
+ # We don't call _check_if_features_can_be_aligned here so this cast is "unsafe"
+ dset_features, item_features = _align_features(
+ [self._info.features, Features.from_arrow_schema(item_table.schema)]
+ )
+ # Cast to align the schemas of the tables and concatenate the tables
+ table = concat_tables(
+ [
+ self._data.cast(dset_features.arrow_schema) if self._info.features != dset_features else self._data,
+ item_table.cast(item_features.arrow_schema),
+ ]
+ )
+ if self._indices is None:
+ indices_table = None
+ else:
+ item_indices_array = pa.array([len(self._data)], type=pa.uint64())
+ item_indices_table = InMemoryTable.from_arrays([item_indices_array], names=["indices"])
+ indices_table = concat_tables([self._indices, item_indices_table])
+ info = self.info.copy()
+ info.features.update(item_features)
+ table = update_metadata_with_features(table, info.features)
+ return Dataset(
+ table,
+ info=info,
+ split=self.split,
+ indices_table=indices_table,
+ fingerprint=new_fingerprint,
+ )
+
+ def align_labels_with_mapping(self, label2id: dict, label_column: str) -> "Dataset":
+ """Align the dataset's label ID and label name mapping to match an input `label2id` mapping.
+ This is useful when you want to ensure that a model's predicted labels are aligned with the dataset.
+ The alignment in done using the lowercase label names.
+
+ Args:
+ label2id (`dict`):
+ The label name to ID mapping to align the dataset with.
+ label_column (`str`):
+ The column name of labels to align on.
+
+ Example:
+
+ ```python
+ >>> # dataset with mapping {'entailment': 0, 'neutral': 1, 'contradiction': 2}
+ >>> ds = load_dataset("nyu-mll/glue", "mnli", split="train")
+ >>> # mapping to align with
+ >>> label2id = {'CONTRADICTION': 0, 'NEUTRAL': 1, 'ENTAILMENT': 2}
+ >>> ds_aligned = ds.align_labels_with_mapping(label2id, "label")
+ ```
+
+ """
+ # Sanity checks
+ if label_column not in self._data.column_names:
+ raise ValueError(f"Column ({label_column}) not in table columns ({self._data.column_names}).")
+
+ label_feature = self._info.features[label_column]
+ if not (
+ isinstance(label_feature, ClassLabel)
+ or (isinstance(label_feature, Sequence) and isinstance(label_feature.feature, ClassLabel))
+ ):
+ raise ValueError(
+ f"Aligning labels with a mapping is only supported for {ClassLabel.__name__} column or {Sequence.__name__} column with the inner type {ClassLabel.__name__}, and column {label_feature} is of type {type(label_feature).__name__}."
+ )
+
+ # Sort input mapping by ID value to ensure the label names are aligned
+ label2id = dict(sorted(label2id.items(), key=lambda item: item[1]))
+ label_names = list(label2id.keys())
+ # Some label mappings use uppercase label names so we lowercase them during alignment
+ label2id = {k.lower(): v for k, v in label2id.items()}
+ int2str_function = (
+ label_feature.int2str if isinstance(label_feature, ClassLabel) else label_feature.feature.int2str
+ )
+
+ if isinstance(label_feature, ClassLabel):
+
+ def process_label_ids(batch):
+ dset_label_names = [
+ int2str_function(label_id).lower() if label_id is not None else None
+ for label_id in batch[label_column]
+ ]
+ batch[label_column] = [
+ label2id[label_name] if label_name is not None else None for label_name in dset_label_names
+ ]
+ return batch
+
+ else:
+
+ def process_label_ids(batch):
+ dset_label_names = [
+ [int2str_function(label_id).lower() if label_id is not None else None for label_id in seq]
+ for seq in batch[label_column]
+ ]
+ batch[label_column] = [
+ [label2id[label_name] if label_name is not None else None for label_name in seq]
+ for seq in dset_label_names
+ ]
+ return batch
+
+ features = self.features
+ features[label_column] = (
+ ClassLabel(num_classes=len(label_names), names=label_names)
+ if isinstance(label_feature, ClassLabel)
+ else List(ClassLabel(num_classes=len(label_names), names=label_names))
+ )
+ return self.map(process_label_ids, features=features, batched=True, desc="Aligning the labels")
+
+
+def _concatenate_map_style_datasets(
+ dsets: list[Dataset],
+ info: Optional[DatasetInfo] = None,
+ split: Optional[NamedSplit] = None,
+ axis: int = 0,
+):
+ """
+ Converts a list of :class:`Dataset` with the same schema into a single :class:`Dataset`.
+ When you concatenate on axis 0, missing data are filled with None values.
+
+ Args:
+ dsets (`List[datasets.Dataset]`): List of Datasets to concatenate.
+ info (:class:`DatasetInfo`, optional): Dataset information, like description, citation, etc.
+ split (:class:`NamedSplit`, optional): Name of the dataset split.
+ axis (``{0, 1}``, default ``0``, meaning over rows):
+ Axis to concatenate over, where ``0`` means over rows (vertically) and ``1`` means over columns
+ (horizontally).
+
+ *New in version 1.6.0*
+
+ Example:
+
+ ```py
+ >>> ds3 = _concatenate_map_style_datasets([ds1, ds2])
+ ```
+ """
+ # Ignore datasets with no rows
+ if any(dset.num_rows > 0 for dset in dsets):
+ dsets = [dset for dset in dsets if dset.num_rows > 0]
+ else:
+ # Return first dataset if all datasets are empty
+ return dsets[0]
+
+ # Perform checks (and a potential cast if axis=0)
+ if axis == 0:
+ _check_if_features_can_be_aligned([dset.features for dset in dsets])
+ else:
+ if not all(dset.num_rows == dsets[0].num_rows for dset in dsets):
+ raise ValueError("Number of rows must match for all datasets")
+ _check_column_names([col_name for dset in dsets for col_name in dset._data.column_names])
+
+ # Find common format or reset format
+ format = dsets[0].format
+ if any(dset.format != format for dset in dsets):
+ format = {}
+ logger.info("Some of the datasets have disparate format. Resetting the format of the concatenated dataset.")
+
+ def apply_offset_to_indices_table(table, offset):
+ if offset == 0:
+ return table
+ else:
+ array = table["indices"]
+ new_array = pc.add(array, pa.scalar(offset, type=pa.uint64()))
+ return InMemoryTable.from_arrays([new_array], names=["indices"])
+
+ # Concatenate indices if they exist
+ if any(dset._indices is not None for dset in dsets):
+ if axis == 0:
+ # Datasets with no indices tables are replaced with a dataset with an indices table in memory.
+ # Applying an offset to an indices table also brings the table in memory.
+ indices_tables = []
+ for i in range(len(dsets)):
+ if dsets[i]._indices is None:
+ dsets[i] = dsets[i]._select_with_indices_mapping(range(len(dsets[i])))
+ indices_tables.append(dsets[i]._indices)
+
+ # An offset needs to be applied to the indices before concatenating
+ offset = 0
+ for i in range(len(dsets)):
+ indices_tables[i] = apply_offset_to_indices_table(indices_tables[i], offset)
+ offset += len(dsets[i]._data)
+
+ # Concatenate indices
+ indices_tables = [t for t in indices_tables if len(t) > 0]
+ if indices_tables:
+ indices_table = concat_tables(indices_tables)
+ else:
+ indices_table = InMemoryTable.from_batches([], schema=pa.schema({"indices": pa.int64()}))
+ else:
+ if len(dsets) == 1:
+ indices_table = dsets[0]._indices
+ else:
+ for i in range(len(dsets)):
+ dsets[i] = dsets[i].flatten_indices()
+ indices_table = None
+ else:
+ indices_table = None
+
+ table = concat_tables([dset._data for dset in dsets], axis=axis)
+ if axis == 0:
+ features_list = _align_features([dset.features for dset in dsets])
+ else:
+ features_list = [dset.features for dset in dsets]
+ table = update_metadata_with_features(table, {k: v for features in features_list for k, v in features.items()})
+
+ # Concatenate infos
+ if info is None:
+ info = DatasetInfo.from_merge([dset.info for dset in dsets])
+ fingerprint = update_fingerprint(
+ "".join(dset._fingerprint for dset in dsets), _concatenate_map_style_datasets, {"info": info, "split": split}
+ )
+
+ # Make final concatenated dataset
+ concatenated_dataset = Dataset(
+ table,
+ info=info,
+ split=split,
+ indices_table=indices_table,
+ fingerprint=fingerprint,
+ )
+ concatenated_dataset.set_format(**format)
+ return concatenated_dataset
+
+
+def _interleave_map_style_datasets(
+ datasets: list["Dataset"],
+ probabilities: Optional[list[float]] = None,
+ seed: Optional[int] = None,
+ info: Optional[DatasetInfo] = None,
+ split: Optional[NamedSplit] = None,
+ stopping_strategy: Literal[
+ "first_exhausted", "all_exhausted", "all_exhausted_without_replacement"
+ ] = "first_exhausted",
+ **kwargs,
+) -> "Dataset":
+ """
+ Interleave several map-style datasets (sources) into a single map-style dataset.
+ The new dataset is constructed by alternating between the sources to get the examples.
+ If `probabilities = None` (default) the new dataset is constructed by cycling between each source to get the examples.
+ If `probabilities` is not `None, the new dataset is constructed by getting examples from a random source at a time according to the provided probabilities.
+
+ Args:
+ datasets (`List[Dataset]`): list of datasets to interleave
+ probabilities (`List[float]`, optional, default None): If specified, the new dataset is constructed by sampling
+ examples from one source at a time according to these probabilities.
+ seed (`int`, optional, default None): The random seed used to choose a source for each example.
+ info (:class:`DatasetInfo`, optional): Dataset information, like description, citation, etc.
+ split (:class:`NamedSplit`, optional): Name of the dataset split.
+ stopping_strategy (`str`, defaults to `first_exhausted`):
+ Two strategies are proposed right now.
+ By default, `first_exhausted` is an undersampling strategy, i.e the dataset construction is stopped as soon as one dataset has ran out of samples.
+ If the strategy is `all_exhausted`, we use an oversampling strategy, i.e the dataset construction is stopped as soon as every samples of every dataset has been added at least once.
+ When strategy is `all_exhausted_without_replacement` we make sure that each sample in each dataset is sampled only once.
+ Note that if the strategy is `all_exhausted`, the interleaved dataset size can get enormous:
+ - with no probabilities, the resulting dataset will have max_length_datasets*nb_dataset samples.
+ - with given probabilities, the resulting dataset will have more samples if some datasets have really low probability of visiting.
+ **kwargs (additional keyword arguments): Keyword arguments to be passed to :meth:`datasets.Datasets.select` when selecting the indices used to interleave the datasets.
+
+ Output:
+ :class:`datasets.Dataset`
+ """
+ if stopping_strategy not in ["first_exhausted", "all_exhausted", "all_exhausted_without_replacement"]:
+ raise ValueError(
+ f"{stopping_strategy} stopping strategy in `interleave_datasets` is not implemented yet with a list of {type(datasets[0])}"
+ )
+
+ # To interleave the datasets, we concatenate them and then we re-order the indices
+ concatenated_datasets = _concatenate_map_style_datasets(datasets, info=info, split=split)
+
+ # Let's now build the indices to pass to .select()
+ lengths = [len(dset) for dset in datasets]
+ offsets = np.cumsum([0] + lengths[:-1])
+
+ # if stopping_strategy is "first_exhausted", it is an undersampling situation whereas it is an oversampling situation if it is "all_exhausted"
+ oversampling = stopping_strategy == "all_exhausted"
+
+ if probabilities is None and not oversampling:
+ # Undersampling situation with cycling between each sources
+ # Example:: If lengths of the datasets are [3, 4, 5]
+ # Then the resulting indices should be [0, 3, 7, 1, 4, 8, 2, 6, 9]
+ # Note that we only have 3 examples per dataset since the first dataset ran out of examples
+
+ # Reasoning behind the following operation: keeping the min_length first indices of each dataset
+ # while offsetting in order to correspond to the right indices of the concatenated dataset
+ # and flattening to effectively interleave the datasets
+ indices = (offsets.reshape(1, -1) + np.arange(min(lengths)).reshape(-1, 1)).flatten().tolist()
+ elif probabilities is None:
+ # Oversampling situation with cycling between each sources
+ # Then the resulting indices should be [0, 3, 7, 1, 4, 8, 2, 5, 9, 0, 6, 10, 1, 3, 11]
+ # Note that we have 5 examples per dataset with a rolling window since the longest dataset has 5 samples
+
+ # Reasoning behind the following operation: for each dataset indices (i.e column) repeat the indices to have max_length indices per dataset
+ # For example, if the max_length is 5 and the i-th dataset has 3 samples, the i-th column will be [0,1,2,0,1]
+ indices = np.mod(np.arange(max(lengths)).reshape(-1, 1), np.array(lengths).reshape(1, -1))
+
+ # We have to keep the indices to their respective dataset offsets and to flatten to effectively interleave the datasets
+ indices = (indices + offsets).flatten().tolist()
+
+ else:
+ # boolean array indicating if at index i if the dataset_i has been fully exhausted
+ is_exhausted = np.full(len(lengths), False)
+
+ # if undersampling ("first_exhausted"), we stop as soon as one dataset is exhausted
+ # if oversampling ("all_exhausted"), we stop as soons as every dataset is exhausted, i.e as soon as every samples of every dataset has been visited at least once
+ bool_strategy_func = (
+ np.all if (oversampling or stopping_strategy == "all_exhausted_without_replacement") else np.any
+ )
+
+ def iter_random_indices():
+ """Get an infinite iterator that randomly samples the index of the source to pick examples from."""
+ rng = np.random.default_rng(seed)
+ while True:
+ yield from (int(i) for i in rng.choice(len(datasets), size=1000, p=probabilities))
+
+ current_index = [0] * len(datasets)
+ indices = []
+ for source_idx in iter_random_indices():
+ # If no oversampling, we stop as soon as a dataset has ran out of examples (np.any)
+ # Otherwise, we stop as soon as every dataset has ran out of examples (np.all)
+ if bool_strategy_func(is_exhausted):
+ # the stopping condition was reached, let's stop
+ break
+
+ # let's add the example at the current index of the `source_idx`-th dataset
+ # For without replacement sampling we additionally need to make sure the current source is not exhausted to not oversample.
+ if stopping_strategy != "all_exhausted_without_replacement" or not is_exhausted[source_idx]:
+ indices.append(current_index[source_idx] + offsets[source_idx])
+ current_index[source_idx] += 1
+
+ # we've ran out of examples for the current dataset, let's update our boolean array and bring the current_index back to 0
+ if current_index[source_idx] >= lengths[source_idx]:
+ is_exhausted[source_idx] = True
+ # We don't want to reset the iterator when stopping strategy is without replacement.
+ if stopping_strategy != "all_exhausted_without_replacement":
+ current_index[source_idx] = 0
+
+ return concatenated_datasets.select(indices, **kwargs)
+
+
+def _split_by_node_map_style_dataset(dataset: Dataset, rank: int, world_size: int) -> Dataset:
+ """
+ Split a dataset for the node at rank `rank` in a pool of nodes of size `world_size`.
+ Each node is assigned a chunk of data, e.g. rank 0 is given the first chunk of the dataset.
+ To maximize data loading throughput, chunks are made of contiguous data on disk if possible.
+
+ Args:
+ dataset ([`Dataset`]):
+ The dataset to split by node.
+ rank (`int`):
+ Rank of the current node.
+ world_size (`int`):
+ Total number of nodes.
+
+ Returns:
+ [`Dataset`]: The dataset to be used on the node at rank `rank`.
+ """
+ return dataset.shard(num_shards=world_size, index=rank, contiguous=True)
+
+
+# This is outside Dataset.filter as it needs to be picklable for multiprocessing
+
+
+def get_indices_from_mask_function(
+ function: Callable,
+ batched: bool,
+ with_indices: bool,
+ with_rank: bool,
+ input_columns: Optional[Union[str, list[str]]],
+ indices_mapping: Optional[Table] = None,
+ *args,
+ **fn_kwargs,
+):
+ if batched:
+ # we extract indices and rank from args
+ *inputs, indices, rank = args
+ additional_args = ()
+ if with_indices:
+ additional_args += (indices,)
+ if with_rank:
+ additional_args += (rank,)
+ mask = function(*inputs, *additional_args, **fn_kwargs)
+ if isinstance(mask, (pa.Array, pa.ChunkedArray)):
+ mask = mask.to_pylist()
+ else:
+ # we get batched data (to return less data than input) but `function` only accepts one example
+ # therefore we need to call `function` on each example of the batch to get the mask
+ *inputs, indices, rank = args
+ mask = []
+ if input_columns is None:
+ # inputs only contains a batch of examples
+ batch: dict = inputs[0]
+ num_examples = len(batch[next(iter(batch.keys()))])
+ for i in range(num_examples):
+ example = {key: batch[key][i] for key in batch}
+ additional_args = ()
+ if with_indices:
+ additional_args += (indices[i],)
+ if with_rank:
+ additional_args += (rank,)
+ mask.append(function(example, *additional_args, **fn_kwargs))
+ else:
+ # inputs is a list of columns
+ columns: list[list] = inputs
+ num_examples = len(columns[0])
+ for i in range(num_examples):
+ input = [column[i] for column in columns]
+ additional_args = ()
+ if with_indices:
+ additional_args += (indices[i],)
+ if with_rank:
+ additional_args += (rank,)
+ mask.append(function(*input, *additional_args, **fn_kwargs))
+ indices_array = [i for i, to_keep in zip(indices, mask) if to_keep]
+ if indices_mapping is not None:
+ indices_array = pa.array(indices_array, type=pa.uint64())
+ indices_array = indices_mapping.column(0).take(indices_array)
+ indices_array = indices_array.to_pylist()
+ return {"indices": indices_array}
+
+
+async def async_get_indices_from_mask_function(
+ function: Callable,
+ batched: bool,
+ with_indices: bool,
+ with_rank: bool,
+ input_columns: Optional[Union[str, list[str]]],
+ indices_mapping: Optional[Table] = None,
+ *args,
+ **fn_kwargs,
+):
+ """same function but async"""
+ if batched:
+ # we extract indices and rank from args
+ *inputs, indices, rank = args
+ additional_args = ()
+ if with_indices:
+ additional_args += (indices,)
+ if with_rank:
+ additional_args += (rank,)
+ mask = await function(*inputs, *additional_args, **fn_kwargs)
+ if isinstance(mask, (pa.Array, pa.ChunkedArray)):
+ mask = mask.to_pylist()
+ else:
+ # we get batched data (to return less data than input) but `function` only accepts one example
+ # therefore we need to call `function` on each example of the batch to get the mask
+ *inputs, indices, rank = args
+ mask = []
+ if input_columns is None:
+ # inputs only contains a batch of examples
+ batch: dict = inputs[0]
+ num_examples = len(batch[next(iter(batch.keys()))])
+ for i in range(num_examples):
+ example = {key: batch[key][i] for key in batch}
+ additional_args = ()
+ if with_indices:
+ additional_args += (indices[i],)
+ if with_rank:
+ additional_args += (rank,)
+ mask.append(await function(example, *additional_args, **fn_kwargs))
+ else:
+ # inputs is a list of columns
+ columns: list[list] = inputs
+ num_examples = len(columns[0])
+ for i in range(num_examples):
+ input = [column[i] for column in columns]
+ additional_args = ()
+ if with_indices:
+ additional_args += (indices[i],)
+ if with_rank:
+ additional_args += (rank,)
+ mask.append(await function(*input, *additional_args, **fn_kwargs))
+ indices_array = [i for i, to_keep in zip(indices, mask) if to_keep]
+ if indices_mapping is not None:
+ indices_array = pa.array(indices_array, type=pa.uint64())
+ indices_array = indices_mapping.column(0).take(indices_array)
+ indices_array = indices_array.to_pylist()
+ return {"indices": indices_array}
diff --git a/venv/lib/python3.10/site-packages/datasets/arrow_reader.py b/venv/lib/python3.10/site-packages/datasets/arrow_reader.py
new file mode 100644
index 0000000000000000000000000000000000000000..3bbb58a59c3966d2f039ebb5d9ef86997d4c3cbd
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/arrow_reader.py
@@ -0,0 +1,620 @@
+# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""Arrow ArrowReader."""
+
+import copy
+import math
+import os
+import re
+from dataclasses import dataclass
+from functools import partial
+from typing import TYPE_CHECKING, Optional, Union
+
+import pyarrow as pa
+import pyarrow.parquet as pq
+from tqdm.contrib.concurrent import thread_map
+
+from .download.download_config import DownloadConfig # noqa: F401
+from .naming import _split_re, filenames_for_dataset_split
+from .table import InMemoryTable, MemoryMappedTable, Table, concat_tables
+from .utils import logging
+from .utils import tqdm as hf_tqdm
+
+
+if TYPE_CHECKING:
+ from .info import DatasetInfo # noqa: F401
+ from .splits import Split, SplitInfo # noqa: F401
+
+
+logger = logging.get_logger(__name__)
+
+HF_GCP_BASE_URL = "https://storage.googleapis.com/huggingface-nlp/cache/datasets"
+
+_SUB_SPEC_RE = re.compile(
+ rf"""
+^
+ (?P{_split_re[1:-1]})
+ (\[
+ ((?P-?[\d_]+)
+ (?P%)?)?
+ :
+ ((?P-?[\d_]+)
+ (?P%)?)?
+ \])?(\((?P[^\)]*)\))?
+$
+""", # remove ^ and $
+ re.X,
+)
+
+_ADDITION_SEP_RE = re.compile(r"\s*\+\s*")
+
+
+class DatasetNotOnHfGcsError(ConnectionError):
+ """When you can't get the dataset from the Hf google cloud storage"""
+
+ pass
+
+
+class MissingFilesOnHfGcsError(ConnectionError):
+ """When some files are missing on the Hf oogle cloud storage"""
+
+ pass
+
+
+@dataclass(frozen=True)
+class FileInstructions:
+ """The file instructions associated with a split ReadInstruction.
+
+ Attributes:
+ num_examples: `int`, The total number of examples
+ file_instructions: List[dict(filename, skip, take)], the files information.
+ The filenames contains the relative path, not absolute.
+ skip/take indicates which example read in the file: `ds.slice(skip, take)`
+ """
+
+ num_examples: int
+ file_instructions: list[dict]
+
+
+def make_file_instructions(
+ name: str,
+ split_infos: list["SplitInfo"],
+ instruction: Union[str, "ReadInstruction"],
+ filetype_suffix: Optional[str] = None,
+ prefix_path: Optional[str] = None,
+) -> FileInstructions:
+ """Returns instructions of the split dict.
+
+ Args:
+ name (`str`): Name of the dataset.
+ split_infos (`list` of `[SplitInfo]`): Dataset splits information.
+ instruction ([`ReadInstruction`] or `str`): Reading instruction for a dataset.
+ filetype_suffix (`str`, *optional*): Suffix of dataset files, e.g. 'arrow' or 'parquet'.
+ prefix_path (`str`, *optional*): Prefix of dataset files, e.g. directory name.
+
+ Returns:
+ [`FileInstructions`]
+ """
+ if not isinstance(name, str):
+ raise TypeError(f"Expected str 'name', but got: {type(name).__name__}")
+ elif not name:
+ raise ValueError("Expected non-empty str 'name'")
+ name2len = {info.name: info.num_examples for info in split_infos}
+ name2shard_lengths = {info.name: info.shard_lengths for info in split_infos}
+ name2filenames = {
+ info.name: filenames_for_dataset_split(
+ path=prefix_path,
+ dataset_name=name,
+ split=info.name,
+ filetype_suffix=filetype_suffix,
+ shard_lengths=name2shard_lengths[info.name],
+ )
+ for info in split_infos
+ }
+ if not isinstance(instruction, ReadInstruction):
+ instruction = ReadInstruction.from_spec(instruction)
+ # Create the absolute instruction (per split)
+ absolute_instructions = instruction.to_absolute(name2len)
+
+ # For each split, return the files instruction (skip/take)
+ file_instructions = []
+ num_examples = 0
+ for abs_instr in absolute_instructions:
+ split_length = name2len[abs_instr.splitname]
+ filenames = name2filenames[abs_instr.splitname]
+ shard_lengths = name2shard_lengths[abs_instr.splitname]
+ from_ = 0 if abs_instr.from_ is None else abs_instr.from_
+ to = split_length if abs_instr.to is None else abs_instr.to
+ if shard_lengths is None: # not sharded
+ for filename in filenames:
+ take = to - from_
+ if take == 0:
+ continue
+ num_examples += take
+ file_instructions.append({"filename": filename, "skip": from_, "take": take})
+ else: # sharded
+ index_start = 0 # Beginning (included) of moving window.
+ index_end = 0 # End (excluded) of moving window.
+ for filename, shard_length in zip(filenames, shard_lengths):
+ index_end += shard_length
+ if from_ < index_end and to > index_start: # There is something to take.
+ skip = from_ - index_start if from_ > index_start else 0
+ take = to - index_start - skip if to < index_end else -1
+ if take == 0:
+ continue
+ file_instructions.append({"filename": filename, "skip": skip, "take": take})
+ num_examples += shard_length - skip if take == -1 else take
+ index_start += shard_length
+ return FileInstructions(
+ num_examples=num_examples,
+ file_instructions=file_instructions,
+ )
+
+
+class BaseReader:
+ """
+ Build a Dataset object out of Instruction instance(s).
+ """
+
+ def __init__(self, path: str, info: Optional["DatasetInfo"]):
+ """Initializes ArrowReader.
+
+ Args:
+ path (str): path where tfrecords are stored.
+ info (DatasetInfo): info about the dataset.
+ """
+ self._path: str = path
+ self._info: Optional["DatasetInfo"] = info
+ self._filetype_suffix: Optional[str] = None
+
+ def _get_table_from_filename(self, filename_skip_take, in_memory=False) -> Table:
+ """Returns a Dataset instance from given (filename, skip, take)."""
+ raise NotImplementedError
+
+ def _read_files(self, files, in_memory=False) -> Table:
+ """Returns Dataset for given file instructions.
+
+ Args:
+ files: List[dict(filename, skip, take)], the files information.
+ The filenames contain the absolute path, not relative.
+ skip/take indicates which example read in the file: `ds.slice(skip, take)`
+ in_memory (bool, default False): Whether to copy the data in-memory.
+ """
+ if len(files) == 0 or not all(isinstance(f, dict) for f in files):
+ raise ValueError("please provide valid file informations")
+ files = copy.deepcopy(files)
+ for f in files:
+ f["filename"] = os.path.join(self._path, f["filename"])
+
+ pa_tables = thread_map(
+ partial(self._get_table_from_filename, in_memory=in_memory),
+ files,
+ tqdm_class=hf_tqdm,
+ desc="Loading dataset shards",
+ # set `disable=None` rather than `disable=False` by default to disable progress bar when no TTY attached
+ disable=len(files) <= 16 or None,
+ )
+ pa_tables = [t for t in pa_tables if len(t) > 0]
+ if not pa_tables and (self._info is None or self._info.features is None):
+ raise ValueError(
+ "Tried to read an empty table. Please specify at least info.features to create an empty table with the right type."
+ )
+ pa_tables = pa_tables or [InMemoryTable.from_batches([], schema=pa.schema(self._info.features.type))]
+ pa_table = concat_tables(pa_tables) if len(pa_tables) != 1 else pa_tables[0]
+ return pa_table
+
+ def get_file_instructions(self, name, instruction, split_infos):
+ """Return list of dict {'filename': str, 'skip': int, 'take': int}"""
+ file_instructions = make_file_instructions(
+ name, split_infos, instruction, filetype_suffix=self._filetype_suffix, prefix_path=self._path
+ )
+ files = file_instructions.file_instructions
+ return files
+
+ def read(
+ self,
+ name,
+ instructions,
+ split_infos,
+ in_memory=False,
+ ):
+ """Returns Dataset instance(s).
+
+ Args:
+ name (str): name of the dataset.
+ instructions (ReadInstruction): instructions to read.
+ Instruction can be string and will then be passed to the Instruction
+ constructor as it.
+ split_infos (list of SplitInfo proto): the available splits for dataset.
+ in_memory (bool, default False): Whether to copy the data in-memory.
+
+ Returns:
+ kwargs to build a single Dataset instance.
+ """
+
+ files = self.get_file_instructions(name, instructions, split_infos)
+ if not files:
+ msg = f'Instruction "{instructions}" corresponds to no data!'
+ raise ValueError(msg)
+ return self.read_files(files=files, original_instructions=instructions, in_memory=in_memory)
+
+ def read_files(
+ self,
+ files: list[dict],
+ original_instructions: Union[None, "ReadInstruction", "Split"] = None,
+ in_memory=False,
+ ):
+ """Returns single Dataset instance for the set of file instructions.
+
+ Args:
+ files: List[dict(filename, skip, take)], the files information.
+ The filenames contains the relative path, not absolute.
+ skip/take indicates which example read in the file: `ds.skip().take()`
+ original_instructions: store the original instructions used to build the dataset split in the dataset.
+ in_memory (bool, default False): Whether to copy the data in-memory.
+
+ Returns:
+ kwargs to build a Dataset instance.
+ """
+ # Prepend path to filename
+ pa_table = self._read_files(files, in_memory=in_memory)
+ # If original_instructions is not None, convert it to a human-readable NamedSplit
+ if original_instructions is not None:
+ from .splits import Split # noqa
+
+ split = Split(str(original_instructions))
+ else:
+ split = None
+ dataset_kwargs = {"arrow_table": pa_table, "info": self._info, "split": split}
+ return dataset_kwargs
+
+
+class ArrowReader(BaseReader):
+ """
+ Build a Dataset object out of Instruction instance(s).
+ This Reader uses either memory mapping or file descriptors (in-memory) on arrow files.
+ """
+
+ def __init__(self, path: str, info: Optional["DatasetInfo"]):
+ """Initializes ArrowReader.
+
+ Args:
+ path (str): path where Arrow files are stored.
+ info (DatasetInfo): info about the dataset.
+ """
+ super().__init__(path, info)
+ self._filetype_suffix = "arrow"
+
+ def _get_table_from_filename(self, filename_skip_take, in_memory=False) -> Table:
+ """Returns a Dataset instance from given (filename, skip, take)."""
+ filename, skip, take = (
+ filename_skip_take["filename"],
+ filename_skip_take["skip"] if "skip" in filename_skip_take else None,
+ filename_skip_take["take"] if "take" in filename_skip_take else None,
+ )
+ table = ArrowReader.read_table(filename, in_memory=in_memory)
+ if take == -1:
+ take = len(table) - skip
+ # here we don't want to slice an empty table, or it may segfault
+ if skip is not None and take is not None and not (skip == 0 and take == len(table)):
+ table = table.slice(skip, take)
+ return table
+
+ @staticmethod
+ def read_table(filename, in_memory=False) -> Table:
+ """
+ Read table from file.
+
+ Args:
+ filename (str): File name of the table.
+ in_memory (bool, default=False): Whether to copy the data in-memory.
+
+ Returns:
+ pyarrow.Table
+ """
+ table_cls = InMemoryTable if in_memory else MemoryMappedTable
+ return table_cls.from_file(filename)
+
+
+class ParquetReader(BaseReader):
+ """
+ Build a Dataset object out of Instruction instance(s).
+ This Reader uses memory mapping on parquet files.
+ """
+
+ def __init__(self, path: str, info: Optional["DatasetInfo"]):
+ """Initializes ParquetReader.
+
+ Args:
+ path (str): path where tfrecords are stored.
+ info (DatasetInfo): info about the dataset.
+ """
+ super().__init__(path, info)
+ self._filetype_suffix = "parquet"
+
+ def _get_table_from_filename(self, filename_skip_take, **kwargs):
+ """Returns a Dataset instance from given (filename, skip, take)."""
+ filename, skip, take = (
+ filename_skip_take["filename"],
+ filename_skip_take["skip"] if "skip" in filename_skip_take else None,
+ filename_skip_take["take"] if "take" in filename_skip_take else None,
+ )
+ # Parquet read_table always loads data in memory, independently of memory_map
+ pa_table = pq.read_table(filename, memory_map=True)
+ # here we don't want to slice an empty table, or it may segfault
+ if skip is not None and take is not None and not (skip == 0 and take == len(pa_table)):
+ pa_table = pa_table.slice(skip, take)
+ return pa_table
+
+
+@dataclass(frozen=True)
+class _AbsoluteInstruction:
+ """A machine friendly slice: defined absolute positive boundaries."""
+
+ splitname: str
+ from_: int # uint (starting index).
+ to: int # uint (ending index).
+
+
+@dataclass(frozen=True)
+class _RelativeInstruction:
+ """Represents a single parsed slicing instruction, can use % and negatives."""
+
+ splitname: str
+ from_: Optional[int] = None # int (starting index) or None if no lower boundary.
+ to: Optional[int] = None # int (ending index) or None if no upper boundary.
+ unit: Optional[str] = None
+ rounding: Optional[str] = None
+
+ def __post_init__(self):
+ if self.unit is not None and self.unit not in ["%", "abs"]:
+ raise ValueError("unit must be either % or abs")
+ if self.rounding is not None and self.rounding not in ["closest", "pct1_dropremainder"]:
+ raise ValueError("rounding must be either closest or pct1_dropremainder")
+ if self.unit != "%" and self.rounding is not None:
+ raise ValueError("It is forbidden to specify rounding if not using percent slicing.")
+ if self.unit == "%" and self.from_ is not None and abs(self.from_) > 100:
+ raise ValueError("Percent slice boundaries must be > -100 and < 100.")
+ if self.unit == "%" and self.to is not None and abs(self.to) > 100:
+ raise ValueError("Percent slice boundaries must be > -100 and < 100.")
+ # Update via __dict__ due to instance being "frozen"
+ self.__dict__["rounding"] = "closest" if self.rounding is None and self.unit == "%" else self.rounding
+
+
+def _str_to_read_instruction(spec):
+ """Returns ReadInstruction for given string."""
+ res = _SUB_SPEC_RE.match(spec)
+ if not res:
+ raise ValueError(f"Unrecognized instruction format: {spec}")
+ unit = "%" if res.group("from_pct") or res.group("to_pct") else "abs"
+ return ReadInstruction(
+ split_name=res.group("split"),
+ rounding=res.group("rounding"),
+ from_=int(res.group("from")) if res.group("from") else None,
+ to=int(res.group("to")) if res.group("to") else None,
+ unit=unit,
+ )
+
+
+def _pct_to_abs_pct1(boundary, num_examples):
+ # Using math.trunc here, since -99.5% should give -99%, not -100%.
+ if num_examples < 100:
+ msg = (
+ 'Using "pct1_dropremainder" rounding on a split with less than 100 '
+ "elements is forbidden: it always results in an empty dataset."
+ )
+ raise ValueError(msg)
+ return boundary * math.trunc(num_examples / 100.0)
+
+
+def _pct_to_abs_closest(boundary, num_examples):
+ return int(round(boundary * num_examples / 100.0))
+
+
+def _rel_to_abs_instr(rel_instr, name2len):
+ """Returns _AbsoluteInstruction instance for given RelativeInstruction.
+
+ Args:
+ rel_instr: RelativeInstruction instance.
+ name2len: dict {split_name: num_examples}.
+ """
+ pct_to_abs = _pct_to_abs_closest if rel_instr.rounding == "closest" else _pct_to_abs_pct1
+ split = rel_instr.splitname
+ if split not in name2len:
+ raise ValueError(f'Unknown split "{split}". Should be one of {list(name2len)}.')
+ num_examples = name2len[split]
+ from_ = rel_instr.from_
+ to = rel_instr.to
+ if rel_instr.unit == "%":
+ from_ = 0 if from_ is None else pct_to_abs(from_, num_examples)
+ to = num_examples if to is None else pct_to_abs(to, num_examples)
+ else:
+ from_ = 0 if from_ is None else from_
+ to = num_examples if to is None else to
+ if from_ < 0:
+ from_ = max(num_examples + from_, 0)
+ if to < 0:
+ to = max(num_examples + to, 0)
+ from_ = min(from_, num_examples)
+ to = min(to, num_examples)
+ return _AbsoluteInstruction(split, from_, to)
+
+
+class ReadInstruction:
+ """Reading instruction for a dataset.
+
+ Examples::
+
+ # The following lines are equivalent:
+ ds = datasets.load_dataset('mnist', split='test[:33%]')
+ ds = datasets.load_dataset('mnist', split=datasets.ReadInstruction.from_spec('test[:33%]'))
+ ds = datasets.load_dataset('mnist', split=datasets.ReadInstruction('test', to=33, unit='%'))
+ ds = datasets.load_dataset('mnist', split=datasets.ReadInstruction(
+ 'test', from_=0, to=33, unit='%'))
+
+ # The following lines are equivalent:
+ ds = datasets.load_dataset('mnist', split='test[:33%]+train[1:-1]')
+ ds = datasets.load_dataset('mnist', split=datasets.ReadInstruction.from_spec(
+ 'test[:33%]+train[1:-1]'))
+ ds = datasets.load_dataset('mnist', split=(
+ datasets.ReadInstruction('test', to=33, unit='%') +
+ datasets.ReadInstruction('train', from_=1, to=-1, unit='abs')))
+
+ # The following lines are equivalent:
+ ds = datasets.load_dataset('mnist', split='test[:33%](pct1_dropremainder)')
+ ds = datasets.load_dataset('mnist', split=datasets.ReadInstruction.from_spec(
+ 'test[:33%](pct1_dropremainder)'))
+ ds = datasets.load_dataset('mnist', split=datasets.ReadInstruction(
+ 'test', from_=0, to=33, unit='%', rounding="pct1_dropremainder"))
+
+ # 10-fold validation:
+ tests = datasets.load_dataset(
+ 'mnist',
+ [datasets.ReadInstruction('train', from_=k, to=k+10, unit='%')
+ for k in range(0, 100, 10)])
+ trains = datasets.load_dataset(
+ 'mnist',
+ [datasets.ReadInstruction('train', to=k, unit='%') + datasets.ReadInstruction('train', from_=k+10, unit='%')
+ for k in range(0, 100, 10)])
+
+ """
+
+ def _init(self, relative_instructions):
+ # Private initializer.
+ self._relative_instructions = relative_instructions
+
+ @classmethod
+ def _read_instruction_from_relative_instructions(cls, relative_instructions):
+ """Returns ReadInstruction obj initialized with relative_instructions."""
+ # Use __new__ to bypass __init__ used by public API and not conveniant here.
+ result = cls.__new__(cls)
+ result._init(relative_instructions) # pylint: disable=protected-access
+ return result
+
+ def __init__(self, split_name, rounding=None, from_=None, to=None, unit=None):
+ """Initialize ReadInstruction.
+
+ Args:
+ split_name (str): name of the split to read. Eg: 'train'.
+ rounding (str, optional): The rounding behaviour to use when percent slicing is
+ used. Ignored when slicing with absolute indices.
+ Possible values:
+ - 'closest' (default): The specified percentages are rounded to the
+ closest value. Use this if you want specified percents to be as
+ much exact as possible.
+ - 'pct1_dropremainder': the specified percentages are treated as
+ multiple of 1%. Use this option if you want consistency. Eg:
+ len(5%) == 5 * len(1%).
+ Using this option, one might not be able to use the full set of
+ examples, if the number of those is not a multiple of 100.
+ from_ (int):
+ to (int): alternative way of specifying slicing boundaries. If any of
+ {from_, to, unit} argument is used, slicing cannot be specified as
+ string.
+ unit (str): optional, one of:
+ '%': to set the slicing unit as percents of the split size.
+ 'abs': to set the slicing unit as absolute numbers.
+ """
+ # This constructor is not always called. See factory method
+ # `_read_instruction_from_relative_instructions`. Common init instructions
+ # MUST be placed in the _init method.
+ self._init([_RelativeInstruction(split_name, from_, to, unit, rounding)])
+
+ @classmethod
+ def from_spec(cls, spec):
+ """Creates a `ReadInstruction` instance out of a string spec.
+
+ Args:
+ spec (`str`):
+ Split(s) + optional slice(s) to read + optional rounding
+ if percents are used as the slicing unit. A slice can be specified,
+ using absolute numbers (`int`) or percentages (`int`).
+
+ Examples:
+
+ ```
+ test: test split.
+ test + validation: test split + validation split.
+ test[10:]: test split, minus its first 10 records.
+ test[:10%]: first 10% records of test split.
+ test[:20%](pct1_dropremainder): first 10% records, rounded with the pct1_dropremainder rounding.
+ test[:-5%]+train[40%:60%]: first 95% of test + middle 20% of train.
+ ```
+
+ Returns:
+ ReadInstruction instance.
+ """
+ spec = str(spec) # Need to convert to str in case of NamedSplit instance.
+ subs = _ADDITION_SEP_RE.split(spec)
+ if not subs:
+ raise ValueError(f"No instructions could be built out of {spec}")
+ instruction = _str_to_read_instruction(subs[0])
+ return sum((_str_to_read_instruction(sub) for sub in subs[1:]), instruction)
+
+ def to_spec(self):
+ rel_instr_specs = []
+ for rel_instr in self._relative_instructions:
+ rel_instr_spec = rel_instr.splitname
+ if rel_instr.from_ is not None or rel_instr.to is not None:
+ from_ = rel_instr.from_
+ to = rel_instr.to
+ unit = rel_instr.unit
+ rounding = rel_instr.rounding
+ unit = unit if unit == "%" else ""
+ from_ = str(from_) + unit if from_ is not None else ""
+ to = str(to) + unit if to is not None else ""
+ slice_str = f"[{from_}:{to}]"
+ rounding_str = (
+ f"({rounding})" if unit == "%" and rounding is not None and rounding != "closest" else ""
+ )
+ rel_instr_spec += slice_str + rounding_str
+ rel_instr_specs.append(rel_instr_spec)
+ return "+".join(rel_instr_specs)
+
+ def __add__(self, other):
+ """Returns a new ReadInstruction obj, result of appending other to self."""
+ if not isinstance(other, ReadInstruction):
+ msg = "ReadInstruction can only be added to another ReadInstruction obj."
+ raise TypeError(msg)
+ self_ris = self._relative_instructions
+ other_ris = other._relative_instructions # pylint: disable=protected-access
+ if (
+ self_ris[0].unit != "abs"
+ and other_ris[0].unit != "abs"
+ and self._relative_instructions[0].rounding != other_ris[0].rounding
+ ):
+ raise ValueError("It is forbidden to sum ReadInstruction instances with different rounding values.")
+ return self._read_instruction_from_relative_instructions(self_ris + other_ris)
+
+ def __str__(self):
+ return self.to_spec()
+
+ def __repr__(self):
+ return f"ReadInstruction({self._relative_instructions})"
+
+ def to_absolute(self, name2len):
+ """Translate instruction into a list of absolute instructions.
+
+ Those absolute instructions are then to be added together.
+
+ Args:
+ name2len (`dict`):
+ Associating split names to number of examples.
+
+ Returns:
+ list of _AbsoluteInstruction instances (corresponds to the + in spec).
+ """
+ return [_rel_to_abs_instr(rel_instr, name2len) for rel_instr in self._relative_instructions]
diff --git a/venv/lib/python3.10/site-packages/datasets/arrow_writer.py b/venv/lib/python3.10/site-packages/datasets/arrow_writer.py
new file mode 100644
index 0000000000000000000000000000000000000000..3174f5cf206b9072742b098f0c68d22d98e33c9e
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/arrow_writer.py
@@ -0,0 +1,766 @@
+# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""To write records into Parquet files."""
+
+import json
+import sys
+from collections.abc import Iterable
+from typing import Any, Optional, Union
+
+import fsspec
+import numpy as np
+import pyarrow as pa
+import pyarrow.parquet as pq
+from fsspec.core import url_to_fs
+
+from . import config
+from .features import Audio, Features, Image, Pdf, Value, Video
+from .features.features import (
+ FeatureType,
+ List,
+ _ArrayXDExtensionType,
+ _visit,
+ cast_to_python_objects,
+ generate_from_arrow_type,
+ get_nested_type,
+ list_of_np_array_to_pyarrow_listarray,
+ numpy_to_pyarrow_listarray,
+ to_pyarrow_listarray,
+)
+from .filesystems import is_remote_filesystem
+from .info import DatasetInfo
+from .keyhash import DuplicatedKeysError, KeyHasher
+from .table import array_cast, cast_array_to_feature, embed_table_storage, table_cast
+from .utils import logging
+from .utils.py_utils import asdict, convert_file_size_to_int, first_non_null_non_empty_value
+
+
+logger = logging.get_logger(__name__)
+
+type_ = type # keep python's type function
+
+
+def get_arrow_writer_batch_size_from_features(features: Optional[Features]) -> Optional[int]:
+ """
+ Get the writer_batch_size that defines the maximum record batch size in the arrow files based on configuration values.
+ The default value is 100 for image/audio datasets and 10 for videos.
+ This allows to avoid overflows in arrow buffers.
+
+ Args:
+ features (`datasets.Features` or `None`):
+ Dataset Features from `datasets`.
+ Returns:
+ writer_batch_size (`Optional[int]`):
+ Writer batch size to pass to a dataset builder.
+ If `None`, then it will use the `datasets` default, i.e. `datasets.config.DEFAULT_MAX_BATCH_SIZE`.
+ """
+ if not features:
+ return None
+
+ batch_size = np.inf
+
+ def set_batch_size(feature: FeatureType) -> None:
+ nonlocal batch_size
+ if isinstance(feature, Image) and config.ARROW_RECORD_BATCH_SIZE_FOR_IMAGE_DATASETS is not None:
+ batch_size = min(batch_size, config.ARROW_RECORD_BATCH_SIZE_FOR_IMAGE_DATASETS)
+ elif isinstance(feature, Audio) and config.ARROW_RECORD_BATCH_SIZE_FOR_AUDIO_DATASETS is not None:
+ batch_size = min(batch_size, config.ARROW_RECORD_BATCH_SIZE_FOR_AUDIO_DATASETS)
+ elif isinstance(feature, Video) and config.ARROW_RECORD_BATCH_SIZE_FOR_VIDEO_DATASETS is not None:
+ batch_size = min(batch_size, config.ARROW_RECORD_BATCH_SIZE_FOR_VIDEO_DATASETS)
+ elif (
+ isinstance(feature, Value)
+ and feature.dtype == "binary"
+ and config.ARROW_RECORD_BATCH_SIZE_FOR_BINARY_DATASETS is not None
+ ):
+ batch_size = min(batch_size, config.ARROW_RECORD_BATCH_SIZE_FOR_BINARY_DATASETS)
+
+ _visit(features, set_batch_size)
+
+ return None if batch_size is np.inf else batch_size
+
+
+def get_writer_batch_size_from_features(features: Optional[Features]) -> Optional[int]:
+ """
+ Get the writer_batch_size that defines the maximum row group size in the parquet files based on configuration values.
+ By default these are not set, but it can be helpful to hard set those values in some cases.
+ This allows to optimize random access to parquet file, since accessing 1 row requires
+ to read its entire row group.
+
+ Args:
+ features (`datasets.Features` or `None`):
+ Dataset Features from `datasets`.
+ Returns:
+ writer_batch_size (`Optional[int]`):
+ Writer batch size to pass to a parquet writer.
+ If `None`, then it will use the `datasets` default, i.e. aiming for row groups of 100MB.
+ """
+ if not features:
+ return None
+
+ batch_size = np.inf
+
+ def set_batch_size(feature: FeatureType) -> None:
+ nonlocal batch_size
+ if isinstance(feature, Image) and config.PARQUET_ROW_GROUP_SIZE_FOR_IMAGE_DATASETS is not None:
+ batch_size = min(batch_size, config.PARQUET_ROW_GROUP_SIZE_FOR_IMAGE_DATASETS)
+ elif isinstance(feature, Audio) and config.PARQUET_ROW_GROUP_SIZE_FOR_AUDIO_DATASETS is not None:
+ batch_size = min(batch_size, config.PARQUET_ROW_GROUP_SIZE_FOR_AUDIO_DATASETS)
+ elif isinstance(feature, Video) and config.PARQUET_ROW_GROUP_SIZE_FOR_VIDEO_DATASETS is not None:
+ batch_size = min(batch_size, config.PARQUET_ROW_GROUP_SIZE_FOR_VIDEO_DATASETS)
+ elif (
+ isinstance(feature, Value)
+ and feature.dtype == "binary"
+ and config.PARQUET_ROW_GROUP_SIZE_FOR_BINARY_DATASETS is not None
+ ):
+ batch_size = min(batch_size, config.PARQUET_ROW_GROUP_SIZE_FOR_BINARY_DATASETS)
+
+ _visit(features, set_batch_size)
+
+ return None if batch_size is np.inf else batch_size
+
+
+def get_writer_batch_size_from_data_size(num_rows: int, num_bytes: int) -> int:
+ """
+ Get the writer_batch_size that defines the maximum row group size in the parquet files.
+ The default in `datasets` is aiming for row groups of maximum 100MB uncompressed.
+ This allows to optimize random access to parquet file, since accessing 1 row requires
+ to read its entire row group.
+
+ This can be improved to get optimized size for querying/iterating
+ but at least it matches the dataset viewer expectations on HF.
+
+ Args:
+ num_rows (`int`):
+ Number of rows in the dataset.
+ num_bytes (`int`):
+ Number of bytes in the dataset.
+ For dataset with external files to embed (image, audio, videos), this can also be an
+ estimate from `dataset._estimate_nbytes()`.
+ Returns:
+ writer_batch_size (`Optional[int]`):
+ Writer batch size to pass to a parquet writer.
+ """
+ return max(10, num_rows * convert_file_size_to_int(config.MAX_ROW_GROUP_SIZE) // num_bytes) if num_bytes > 0 else 1
+
+
+class SchemaInferenceError(ValueError):
+ pass
+
+
+class TypedSequence:
+ """
+ This data container generalizes the typing when instantiating pyarrow arrays, tables or batches.
+
+ More specifically it adds several features:
+ - Support extension types like ``datasets.features.Array2DExtensionType``:
+ By default pyarrow arrays don't return extension arrays. One has to call
+ ``pa.ExtensionArray.from_storage(type, pa.array(data, type.storage_type))``
+ in order to get an extension array.
+ - Support for ``try_type`` parameter that can be used instead of ``type``:
+ When an array is transformed, we like to keep the same type as before if possible.
+ For example when calling :func:`datasets.Dataset.map`, we don't want to change the type
+ of each column by default.
+ - Better error message when a pyarrow array overflows.
+
+ Example::
+
+ from datasets.features import Array2D, Array2DExtensionType, Value
+ from datasets.arrow_writer import TypedSequence
+ import pyarrow as pa
+
+ arr = pa.array(TypedSequence([1, 2, 3], type=Value("int32")))
+ assert arr.type == pa.int32()
+
+ arr = pa.array(TypedSequence([1, 2, 3], try_type=Value("int32")))
+ assert arr.type == pa.int32()
+
+ arr = pa.array(TypedSequence(["foo", "bar"], try_type=Value("int32")))
+ assert arr.type == pa.string()
+
+ arr = pa.array(TypedSequence([[[1, 2, 3]]], type=Array2D((1, 3), "int64")))
+ assert arr.type == Array2DExtensionType((1, 3), "int64")
+
+ table = pa.Table.from_pydict({
+ "image": TypedSequence([[[1, 2, 3]]], type=Array2D((1, 3), "int64"))
+ })
+ assert table["image"].type == Array2DExtensionType((1, 3), "int64")
+
+ """
+
+ def __init__(
+ self,
+ data: Iterable,
+ type: Optional[FeatureType] = None,
+ try_type: Optional[FeatureType] = None,
+ optimized_int_type: Optional[FeatureType] = None,
+ ):
+ # assert type is None or try_type is None,
+ if type is not None and try_type is not None:
+ raise ValueError("You cannot specify both type and try_type")
+ # set attributes
+ self.data = data
+ self.type = type
+ self.try_type = try_type # is ignored if it doesn't match the data
+ self.optimized_int_type = optimized_int_type
+ # when trying a type (is ignored if data is not compatible)
+ self.trying_type = self.try_type is not None
+ self.trying_int_optimization = optimized_int_type is not None and type is None and try_type is None
+ # used to get back the inferred type after __arrow_array__() is called once
+ self._inferred_type = None
+
+ def get_inferred_type(self) -> FeatureType:
+ """Return the inferred feature type.
+ This is done by converting the sequence to an Arrow array, and getting the corresponding
+ feature type.
+
+ Since building the Arrow array can be expensive, the value of the inferred type is cached
+ as soon as pa.array is called on the typed sequence.
+
+ Returns:
+ FeatureType: inferred feature type of the sequence.
+ """
+ if self._inferred_type is None:
+ self._inferred_type = generate_from_arrow_type(pa.array(self).type)
+ return self._inferred_type
+
+ @staticmethod
+ def _infer_custom_type_and_encode(data: Iterable) -> tuple[Iterable, Optional[FeatureType]]:
+ """Implement type inference for custom objects like PIL.Image.Image -> Image type.
+
+ This function is only used for custom python objects that can't be directly passed to build
+ an Arrow array. In such cases is infers the feature type to use, and it encodes the data so
+ that they can be passed to an Arrow array.
+
+ Args:
+ data (Iterable): array of data to infer the type, e.g. a list of PIL images.
+
+ Returns:
+ Tuple[Iterable, Optional[FeatureType]]: a tuple with:
+ - the (possibly encoded) array, if the inferred feature type requires encoding
+ - the inferred feature type if the array is made of supported custom objects like
+ PIL images, else None.
+ """
+ if config.PIL_AVAILABLE and "PIL" in sys.modules:
+ import PIL.Image
+
+ non_null_idx, non_null_value = first_non_null_non_empty_value(data)
+ if isinstance(non_null_value, PIL.Image.Image):
+ return [Image().encode_example(value) if value is not None else None for value in data], Image()
+ if isinstance(non_null_value, list) and isinstance(non_null_value[0], PIL.Image.Image):
+ return [
+ [Image().encode_example(x) for x in value] if value is not None else None for value in data
+ ], List(Image())
+ if config.PDFPLUMBER_AVAILABLE and "pdfplumber" in sys.modules:
+ import pdfplumber
+
+ non_null_idx, non_null_value = first_non_null_non_empty_value(data)
+ if isinstance(non_null_value, pdfplumber.pdf.PDF):
+ return [Pdf().encode_example(value) if value is not None else None for value in data], Pdf()
+ if isinstance(non_null_value, list) and isinstance(non_null_value[0], pdfplumber.pdf.PDF):
+ return [
+ [Pdf().encode_example(x) for x in value] if value is not None else None for value in data
+ ], List(Pdf())
+ return data, None
+
+ def __arrow_array__(self, type: Optional[pa.DataType] = None):
+ """This function is called when calling pa.array(typed_sequence)"""
+
+ if type is not None:
+ raise ValueError("TypedSequence is supposed to be used with pa.array(typed_sequence, type=None)")
+ del type # make sure we don't use it
+ data = self.data
+ # automatic type inference for custom objects
+ if self.type is None and self.try_type is None:
+ data, self._inferred_type = self._infer_custom_type_and_encode(data)
+ if self._inferred_type is None:
+ type = self.try_type if self.trying_type else self.type
+ else:
+ type = self._inferred_type
+ pa_type = get_nested_type(type) if type is not None else None
+ optimized_int_pa_type = (
+ get_nested_type(self.optimized_int_type) if self.optimized_int_type is not None else None
+ )
+ trying_cast_to_python_objects = False
+ try:
+ # custom pyarrow types
+ if isinstance(pa_type, _ArrayXDExtensionType):
+ storage = to_pyarrow_listarray(data, pa_type)
+ return pa.ExtensionArray.from_storage(pa_type, storage)
+
+ # efficient np array to pyarrow array
+ if isinstance(data, np.ndarray):
+ out = numpy_to_pyarrow_listarray(data)
+ elif isinstance(data, list) and data and isinstance(first_non_null_non_empty_value(data)[1], np.ndarray):
+ out = list_of_np_array_to_pyarrow_listarray(data)
+ else:
+ trying_cast_to_python_objects = True
+ out = pa.array(cast_to_python_objects(data, only_1d_for_numpy=True))
+ # use smaller integer precisions if possible
+ if self.trying_int_optimization:
+ if pa.types.is_int64(out.type):
+ out = out.cast(optimized_int_pa_type)
+ elif pa.types.is_list(out.type):
+ if pa.types.is_int64(out.type.value_type):
+ out = array_cast(out, pa.list_(optimized_int_pa_type))
+ elif pa.types.is_list(out.type.value_type) and pa.types.is_int64(out.type.value_type.value_type):
+ out = array_cast(out, pa.list_(pa.list_(optimized_int_pa_type)))
+ # otherwise we can finally use the user's type
+ elif type is not None:
+ # We use cast_array_to_feature to support casting to custom types like Audio and Image
+ # Also, when trying type "string", we don't want to convert integers or floats to "string".
+ # We only do it if trying_type is False - since this is what the user asks for.
+ out = cast_array_to_feature(
+ out, type, allow_primitive_to_str=not self.trying_type, allow_decimal_to_str=not self.trying_type
+ )
+ return out
+ except (
+ TypeError,
+ pa.lib.ArrowInvalid,
+ pa.lib.ArrowNotImplementedError,
+ ) as e: # handle type errors and overflows
+ # Ignore ArrowNotImplementedError caused by trying type, otherwise re-raise
+ if not self.trying_type and isinstance(e, pa.lib.ArrowNotImplementedError):
+ raise
+
+ if self.trying_type:
+ try: # second chance
+ if isinstance(data, np.ndarray):
+ return numpy_to_pyarrow_listarray(data)
+ elif isinstance(data, list) and data and any(isinstance(value, np.ndarray) for value in data):
+ return list_of_np_array_to_pyarrow_listarray(data)
+ else:
+ trying_cast_to_python_objects = True
+ return pa.array(cast_to_python_objects(data, only_1d_for_numpy=True))
+ except pa.lib.ArrowInvalid as e:
+ if "overflow" in str(e):
+ raise OverflowError(
+ f"There was an overflow with type {type_(data)}. Try to reduce writer_batch_size to have batches smaller than 2GB.\n({e})"
+ ) from None
+ elif self.trying_int_optimization and "not in range" in str(e):
+ optimized_int_pa_type_str = np.dtype(optimized_int_pa_type.to_pandas_dtype()).name
+ logger.info(
+ f"Failed to cast a sequence to {optimized_int_pa_type_str}. Falling back to int64."
+ )
+ return out
+ elif trying_cast_to_python_objects and "Could not convert" in str(e):
+ out = pa.array(
+ cast_to_python_objects(data, only_1d_for_numpy=True, optimize_list_casting=False)
+ )
+ if type is not None:
+ out = cast_array_to_feature(
+ out, type, allow_primitive_to_str=True, allow_decimal_to_str=True
+ )
+ return out
+ else:
+ raise
+ elif "overflow" in str(e):
+ raise OverflowError(
+ f"There was an overflow with type {type_(data)}. Try to reduce writer_batch_size to have batches smaller than 2GB.\n({e})"
+ ) from None
+ elif self.trying_int_optimization and "not in range" in str(e):
+ optimized_int_pa_type_str = np.dtype(optimized_int_pa_type.to_pandas_dtype()).name
+ logger.info(f"Failed to cast a sequence to {optimized_int_pa_type_str}. Falling back to int64.")
+ return out
+ elif trying_cast_to_python_objects and "Could not convert" in str(e):
+ out = pa.array(cast_to_python_objects(data, only_1d_for_numpy=True, optimize_list_casting=False))
+ if type is not None:
+ out = cast_array_to_feature(out, type, allow_primitive_to_str=True, allow_decimal_to_str=True)
+ return out
+ else:
+ raise
+
+
+class OptimizedTypedSequence(TypedSequence):
+ def __init__(
+ self,
+ data,
+ type: Optional[FeatureType] = None,
+ try_type: Optional[FeatureType] = None,
+ col: Optional[str] = None,
+ optimized_int_type: Optional[FeatureType] = None,
+ ):
+ optimized_int_type_by_col = {
+ "attention_mask": Value("int8"), # binary tensor
+ "special_tokens_mask": Value("int8"),
+ "input_ids": Value("int32"), # typical vocab size: 0-50k (max ~500k, never > 1M)
+ "token_type_ids": Value(
+ "int8"
+ ), # binary mask; some (XLNetModel) use an additional token represented by a 2
+ }
+ if type is None and try_type is None:
+ optimized_int_type = optimized_int_type_by_col.get(col, None)
+ super().__init__(data, type=type, try_type=try_type, optimized_int_type=optimized_int_type)
+
+
+class ArrowWriter:
+ """Shuffles and writes Examples to Arrow files."""
+
+ def __init__(
+ self,
+ schema: Optional[pa.Schema] = None,
+ features: Optional[Features] = None,
+ path: Optional[str] = None,
+ stream: Optional[pa.NativeFile] = None,
+ fingerprint: Optional[str] = None,
+ writer_batch_size: Optional[int] = None,
+ hash_salt: Optional[str] = None,
+ check_duplicates: Optional[bool] = False,
+ disable_nullable: bool = False,
+ update_features: bool = False,
+ with_metadata: bool = True,
+ unit: str = "examples",
+ embed_local_files: bool = False,
+ storage_options: Optional[dict] = None,
+ ):
+ if path is None and stream is None:
+ raise ValueError("At least one of path and stream must be provided.")
+ if features is not None:
+ self._features = features
+ self._schema = None
+ elif schema is not None:
+ self._schema: pa.Schema = schema
+ self._features = Features.from_arrow_schema(self._schema)
+ else:
+ self._features = None
+ self._schema = None
+
+ if hash_salt is not None:
+ # Create KeyHasher instance using split name as hash salt
+ self._hasher = KeyHasher(hash_salt)
+ else:
+ self._hasher = KeyHasher("")
+
+ self._check_duplicates = check_duplicates
+ self._disable_nullable = disable_nullable
+
+ if stream is None:
+ fs, path = url_to_fs(path, **(storage_options or {}))
+ self._fs: fsspec.AbstractFileSystem = fs
+ self._path = path if not is_remote_filesystem(self._fs) else self._fs.unstrip_protocol(path)
+ self.stream = self._fs.open(path, "wb")
+ self._closable_stream = True
+ else:
+ self._fs = None
+ self._path = None
+ self.stream = stream
+ self._closable_stream = False
+
+ self.fingerprint = fingerprint
+ self.disable_nullable = disable_nullable
+ self.writer_batch_size = (
+ writer_batch_size
+ or get_arrow_writer_batch_size_from_features(self._features)
+ or config.DEFAULT_MAX_BATCH_SIZE
+ )
+ self.update_features = update_features
+ self.with_metadata = with_metadata
+ self.unit = unit
+ self.embed_local_files = embed_local_files
+
+ self._num_examples = 0
+ self._num_bytes = 0
+ self.current_examples: list[tuple[dict[str, Any], str]] = []
+ self.current_rows: list[pa.Table] = []
+ self.pa_writer: Optional[pa.RecordBatchStreamWriter] = None
+ self.hkey_record = []
+
+ def __len__(self):
+ """Return the number of writed and staged examples"""
+ return self._num_examples + len(self.current_examples) + len(self.current_rows)
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ self.close()
+
+ def close(self):
+ # Try closing if opened; if closed: pyarrow.lib.ArrowInvalid: Invalid operation on closed file
+ if self.pa_writer: # it might be None
+ try:
+ self.pa_writer.close()
+ except Exception: # pyarrow.lib.ArrowInvalid, OSError
+ pass
+ if self._closable_stream and not self.stream.closed:
+ self.stream.close() # This also closes self.pa_writer if it is opened
+
+ def _build_schema(self, inferred_schema: pa.Schema):
+ schema = self.schema
+ features = self._features
+ inferred_features = Features.from_arrow_schema(inferred_schema)
+ if self._features is not None:
+ if self.update_features: # keep original features it they match, or update them
+ fields = {field.name: field for field in self._features.type}
+ for inferred_field in inferred_features.type:
+ name = inferred_field.name
+ if name in fields:
+ if inferred_field == fields[name]:
+ inferred_features[name] = self._features[name]
+ features = inferred_features
+ schema: pa.Schema = inferred_schema
+ else:
+ features = inferred_features
+ schema: pa.Schema = inferred_features.arrow_schema
+
+ if self.disable_nullable:
+ schema = pa.schema(pa.field(field.name, field.type, nullable=False) for field in schema)
+ if self.with_metadata:
+ schema = schema.with_metadata(self._build_metadata(DatasetInfo(features=features), self.fingerprint))
+ else:
+ schema = schema.with_metadata({})
+
+ return schema, features
+
+ def _build_writer(self, inferred_schema: pa.Schema):
+ self._schema, self._features = self._build_schema(inferred_schema)
+ self.pa_writer = pa.RecordBatchStreamWriter(self.stream, self._schema)
+
+ @property
+ def schema(self):
+ _schema = (
+ self._schema
+ if self._schema is not None
+ else (pa.schema(self._features.type) if self._features is not None else None)
+ )
+ if self._disable_nullable and _schema is not None:
+ _schema = pa.schema(pa.field(field.name, field.type, nullable=False) for field in _schema)
+ return _schema if _schema is not None else []
+
+ @staticmethod
+ def _build_metadata(info: DatasetInfo, fingerprint: Optional[str] = None) -> dict[str, str]:
+ info_keys = ["features"] # we can add support for more DatasetInfo keys in the future
+ info_as_dict = asdict(info)
+ metadata = {}
+ metadata["info"] = {key: info_as_dict[key] for key in info_keys}
+ if fingerprint is not None:
+ metadata["fingerprint"] = fingerprint
+ return {"huggingface": json.dumps(metadata)}
+
+ def write_examples_on_file(self):
+ """Write stored examples from the write-pool of examples. It makes a table out of the examples and write it."""
+ if not self.current_examples:
+ return
+ # preserve the order the columns
+ if self.schema:
+ schema_cols = set(self.schema.names)
+ examples_cols = self.current_examples[0][0].keys() # .keys() preserves the order (unlike set)
+ common_cols = [col for col in self.schema.names if col in examples_cols]
+ extra_cols = [col for col in examples_cols if col not in schema_cols]
+ cols = common_cols + extra_cols
+ else:
+ cols = list(self.current_examples[0][0])
+ batch_examples = {}
+ for col in cols:
+ # We use row[0][col] since current_examples contains (example, key) tuples.
+ # Moreover, examples could be Arrow arrays of 1 element.
+ # This can happen in `.map()` when we want to re-write the same Arrow data
+ if all(isinstance(row[0][col], (pa.Array, pa.ChunkedArray)) for row in self.current_examples):
+ arrays = [row[0][col] for row in self.current_examples]
+ arrays = [
+ chunk
+ for array in arrays
+ for chunk in (array.chunks if isinstance(array, pa.ChunkedArray) else [array])
+ ]
+ batch_examples[col] = pa.concat_arrays(arrays)
+ else:
+ batch_examples[col] = [
+ row[0][col].to_pylist()[0] if isinstance(row[0][col], (pa.Array, pa.ChunkedArray)) else row[0][col]
+ for row in self.current_examples
+ ]
+ self.write_batch(batch_examples=batch_examples)
+ self.current_examples = []
+
+ def write_rows_on_file(self):
+ """Write stored rows from the write-pool of rows. It concatenates the single-row tables and it writes the resulting table."""
+ if not self.current_rows:
+ return
+ table = pa.concat_tables(self.current_rows)
+ self.write_table(table)
+ self.current_rows = []
+
+ def write(
+ self,
+ example: dict[str, Any],
+ key: Optional[Union[str, int, bytes]] = None,
+ writer_batch_size: Optional[int] = None,
+ ):
+ """Add a given (Example,Key) pair to the write-pool of examples which is written to file.
+
+ Args:
+ example: the Example to add.
+ key: Optional, a unique identifier(str, int or bytes) associated with each example
+ """
+ # Utilize the keys and duplicate checking when `self._check_duplicates` is passed True
+ if self._check_duplicates:
+ # Create unique hash from key and store as (key, example) pairs
+ hash = self._hasher.hash(key)
+ self.current_examples.append((example, hash))
+ # Maintain record of keys and their respective hashes for checking duplicates
+ self.hkey_record.append((hash, key))
+ else:
+ # Store example as a tuple so as to keep the structure of `self.current_examples` uniform
+ self.current_examples.append((example, ""))
+
+ if writer_batch_size is None:
+ writer_batch_size = self.writer_batch_size
+ if writer_batch_size is not None and len(self.current_examples) >= writer_batch_size:
+ if self._check_duplicates:
+ self.check_duplicate_keys()
+ # Re-initializing to empty list for next batch
+ self.hkey_record = []
+
+ self.write_examples_on_file()
+
+ def check_duplicate_keys(self):
+ """Raises error if duplicates found in a batch"""
+ tmp_record = set()
+ for hash, key in self.hkey_record:
+ if hash in tmp_record:
+ duplicate_key_indices = [
+ str(self._num_examples + index)
+ for index, (duplicate_hash, _) in enumerate(self.hkey_record)
+ if duplicate_hash == hash
+ ]
+
+ raise DuplicatedKeysError(key, duplicate_key_indices)
+ else:
+ tmp_record.add(hash)
+
+ def write_row(self, row: pa.Table, writer_batch_size: Optional[int] = None):
+ """Add a given single-row Table to the write-pool of rows which is written to file.
+
+ Args:
+ row: the row to add.
+ """
+ if len(row) != 1:
+ raise ValueError(f"Only single-row pyarrow tables are allowed but got table with {len(row)} rows.")
+ self.current_rows.append(row)
+ if writer_batch_size is None:
+ writer_batch_size = self.writer_batch_size
+ if writer_batch_size is not None and len(self.current_rows) >= writer_batch_size:
+ self.write_rows_on_file()
+
+ def write_batch(
+ self,
+ batch_examples: dict[str, list],
+ writer_batch_size: Optional[int] = None,
+ try_original_type: Optional[bool] = True,
+ ):
+ """Write a batch of Example to file.
+ Ignores the batch if it appears to be empty,
+ preventing a potential schema update of unknown types.
+
+ Args:
+ batch_examples: the batch of examples to add.
+ try_original_type: use `try_type` when instantiating OptimizedTypedSequence if `True`, otherwise `try_type = None`.
+ """
+ if batch_examples and len(next(iter(batch_examples.values()))) == 0:
+ return
+ features = None if self.pa_writer is None and self.update_features else self._features
+ try_features = self._features if self.pa_writer is None and self.update_features else None
+ arrays = []
+ inferred_features = Features()
+ # preserve the order the columns
+ if self.schema:
+ schema_cols = set(self.schema.names)
+ batch_cols = batch_examples.keys() # .keys() preserves the order (unlike set)
+ common_cols = [col for col in self.schema.names if col in batch_cols]
+ extra_cols = [col for col in batch_cols if col not in schema_cols]
+ cols = common_cols + extra_cols
+ else:
+ cols = list(batch_examples)
+ for col in cols:
+ col_values = batch_examples[col]
+ col_type = features[col] if features else None
+ if isinstance(col_values, (pa.Array, pa.ChunkedArray)):
+ array = cast_array_to_feature(col_values, col_type) if col_type is not None else col_values
+ arrays.append(array)
+ inferred_features[col] = generate_from_arrow_type(col_values.type)
+ else:
+ col_try_type = (
+ try_features[col]
+ if try_features is not None and col in try_features and try_original_type
+ else None
+ )
+ typed_sequence = OptimizedTypedSequence(col_values, type=col_type, try_type=col_try_type, col=col)
+ arrays.append(pa.array(typed_sequence))
+ inferred_features[col] = typed_sequence.get_inferred_type()
+ schema = inferred_features.arrow_schema if self.pa_writer is None else self.schema
+ pa_table = pa.Table.from_arrays(arrays, schema=schema)
+ self.write_table(pa_table, writer_batch_size)
+
+ def write_table(self, pa_table: pa.Table, writer_batch_size: Optional[int] = None):
+ """Write a Table to file.
+
+ Args:
+ example: the Table to add.
+ """
+ if writer_batch_size is None:
+ writer_batch_size = self.writer_batch_size
+ if self.pa_writer is None:
+ self._build_writer(inferred_schema=pa_table.schema)
+ pa_table = pa_table.combine_chunks()
+ pa_table = table_cast(pa_table, self._schema)
+ if self.embed_local_files:
+ pa_table = embed_table_storage(pa_table)
+ self._num_bytes += pa_table.nbytes
+ self._num_examples += pa_table.num_rows
+ self.pa_writer.write_table(pa_table, writer_batch_size)
+
+ def finalize(self, close_stream=True):
+ self.write_rows_on_file()
+ # In case current_examples < writer_batch_size, but user uses finalize()
+ if self._check_duplicates:
+ self.check_duplicate_keys()
+ # Re-initializing to empty list for next batch
+ self.hkey_record = []
+ self.write_examples_on_file()
+ # If schema is known, infer features even if no examples were written
+ if self.pa_writer is None and self.schema:
+ self._build_writer(self.schema)
+ if self.pa_writer is not None:
+ self.pa_writer.close()
+ self.pa_writer = None
+ if close_stream:
+ self.stream.close()
+ else:
+ if close_stream:
+ self.stream.close()
+ raise SchemaInferenceError("Please pass `features` or at least one example when writing data")
+ logger.debug(
+ f"Done writing {self._num_examples} {self.unit} in {self._num_bytes} bytes {self._path if self._path else ''}."
+ )
+ return self._num_examples, self._num_bytes
+
+
+class ParquetWriter(ArrowWriter):
+ def __init__(self, *args, use_content_defined_chunking=True, write_page_index=True, **kwargs):
+ super().__init__(*args, **kwargs)
+ if use_content_defined_chunking is True:
+ use_content_defined_chunking = config.DEFAULT_CDC_OPTIONS
+ self.use_content_defined_chunking = use_content_defined_chunking
+ self.write_page_index = write_page_index
+
+ def _build_writer(self, inferred_schema: pa.Schema):
+ self._schema, self._features = self._build_schema(inferred_schema)
+ self.pa_writer = pq.ParquetWriter(
+ self.stream,
+ self._schema,
+ use_content_defined_chunking=self.use_content_defined_chunking,
+ write_page_index=self.write_page_index,
+ )
+ if self.use_content_defined_chunking is not False:
+ self.pa_writer.add_key_value_metadata(
+ {"content_defined_chunking": json.dumps(self.use_content_defined_chunking)}
+ )
diff --git a/venv/lib/python3.10/site-packages/datasets/builder.py b/venv/lib/python3.10/site-packages/datasets/builder.py
new file mode 100644
index 0000000000000000000000000000000000000000..b88aa0bf8f99fa0c833b736022f12c18b8c09c63
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/builder.py
@@ -0,0 +1,1866 @@
+# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""DatasetBuilder base class."""
+
+import abc
+import contextlib
+import copy
+import inspect
+import os
+import posixpath
+import shutil
+import textwrap
+import time
+import urllib
+from collections.abc import Iterable, Mapping
+from dataclasses import dataclass
+from functools import partial
+from pathlib import Path
+from typing import TYPE_CHECKING, Optional, Union
+from unittest.mock import patch
+
+import fsspec
+from fsspec.core import url_to_fs
+from multiprocess import Pool
+from tqdm.contrib.concurrent import thread_map
+
+from . import config, utils
+from .arrow_dataset import Dataset
+from .arrow_reader import (
+ ArrowReader,
+ ReadInstruction,
+)
+from .arrow_writer import ArrowWriter, ParquetWriter, SchemaInferenceError
+from .data_files import DataFilesDict, DataFilesPatternsDict, sanitize_patterns
+from .dataset_dict import DatasetDict, IterableDatasetDict
+from .download.download_config import DownloadConfig
+from .download.download_manager import DownloadManager, DownloadMode
+from .download.streaming_download_manager import StreamingDownloadManager, xjoin
+from .exceptions import DatasetGenerationCastError, DatasetGenerationError, FileFormatError, ManualDownloadError
+from .features import Features
+from .filesystems import (
+ is_remote_filesystem,
+ rename,
+)
+from .fingerprint import Hasher
+from .info import DatasetInfo, PostProcessedInfo
+from .iterable_dataset import ArrowExamplesIterable, ExamplesIterable, IterableDataset
+from .keyhash import DuplicatedKeysError
+from .naming import INVALID_WINDOWS_CHARACTERS_IN_PATH, camelcase_to_snakecase
+from .splits import Split, SplitDict, SplitGenerator, SplitInfo
+from .streaming import extend_dataset_builder_for_streaming
+from .table import CastError
+from .utils import logging
+from .utils import tqdm as hf_tqdm
+from .utils._filelock import FileLock
+from .utils.file_utils import is_remote_url
+from .utils.info_utils import VerificationMode, get_size_checksum_dict, verify_checksums, verify_splits
+from .utils.py_utils import (
+ classproperty,
+ convert_file_size_to_int,
+ has_sufficient_disk_space,
+ iflatmap_unordered,
+ map_nested,
+ memoize,
+ size_str,
+ temporary_assignment,
+)
+from .utils.sharding import _number_of_shards_in_gen_kwargs, _split_gen_kwargs
+from .utils.track import tracked_list
+
+
+if TYPE_CHECKING:
+ from .load import DatasetModule
+
+
+logger = logging.get_logger(__name__)
+
+
+class InvalidConfigName(ValueError):
+ pass
+
+
+@dataclass
+class BuilderConfig:
+ """Base class for `DatasetBuilder` data configuration.
+
+ `DatasetBuilder` subclasses with data configuration options should subclass
+ `BuilderConfig` and add their own properties.
+
+ Attributes:
+ name (`str`, defaults to `default`):
+ The name of the configuration.
+ version (`Version` or `str`, defaults to `0.0.0`):
+ The version of the configuration.
+ data_dir (`str`, *optional*):
+ Path to the directory containing the source data.
+ data_files (`str` or `Sequence` or `Mapping`, *optional*):
+ Path(s) to source data file(s).
+ description (`str`, *optional*):
+ A human description of the configuration.
+ """
+
+ name: str = "default"
+ version: Optional[Union[utils.Version, str]] = utils.Version("0.0.0")
+ data_dir: Optional[str] = None
+ data_files: Optional[Union[DataFilesDict, DataFilesPatternsDict]] = None
+ description: Optional[str] = None
+
+ def __post_init__(self):
+ # The config name is used to name the cache directory.
+ for invalid_char in INVALID_WINDOWS_CHARACTERS_IN_PATH:
+ if invalid_char in self.name:
+ raise InvalidConfigName(
+ f"Bad characters from black list '{INVALID_WINDOWS_CHARACTERS_IN_PATH}' found in '{self.name}'. "
+ f"They could create issues when creating a directory for this config on Windows filesystem."
+ )
+ if self.data_files is not None and not isinstance(self.data_files, (DataFilesDict, DataFilesPatternsDict)):
+ raise ValueError(f"Expected a DataFilesDict in data_files but got {self.data_files}")
+
+ def __eq__(self, o):
+ # we need to override the default dataclass __eq__ since it doesn't check for
+ # other attributes that the ones of the signature.
+ if set(self.__dict__.keys()) != set(o.__dict__.keys()):
+ return False
+ return all((k, getattr(self, k)) == (k, getattr(o, k)) for k in self.__dict__.keys())
+
+ def create_config_id(
+ self,
+ config_kwargs: dict,
+ custom_features: Optional[Features] = None,
+ ) -> str:
+ """
+ The config id is used to build the cache directory.
+ By default it is equal to the config name.
+ However the name of a config is not sufficient to have a unique identifier for the dataset being generated
+ since it doesn't take into account:
+ - the config kwargs that can be used to overwrite attributes
+ - the custom features used to write the dataset
+ - the data_files for json/text/csv/pandas datasets
+
+ Therefore the config id is just the config name with an optional suffix based on these.
+ """
+ # Possibly add a suffix to the name to handle custom features/data_files/config_kwargs
+ suffix: Optional[str] = None
+ config_kwargs_to_add_to_suffix = config_kwargs.copy()
+ # name and version are already used to build the cache directory
+ config_kwargs_to_add_to_suffix.pop("name", None)
+ config_kwargs_to_add_to_suffix.pop("version", None)
+ # data dir handling (when specified it points to the manually downloaded data):
+ # it was previously ignored before the introduction of config id because we didn't want
+ # to change the config name. Now it's fine to take it into account for the config id.
+ # config_kwargs_to_add_to_suffix.pop("data_dir", None)
+ if "data_dir" in config_kwargs_to_add_to_suffix:
+ if config_kwargs_to_add_to_suffix["data_dir"] is None:
+ config_kwargs_to_add_to_suffix.pop("data_dir", None)
+ else:
+ # canonicalize the data dir to avoid two paths to the same location having different
+ # hashes
+ data_dir = config_kwargs_to_add_to_suffix["data_dir"]
+ data_dir = os.path.normpath(data_dir)
+ config_kwargs_to_add_to_suffix["data_dir"] = data_dir
+ if config_kwargs_to_add_to_suffix:
+ # we don't care about the order of the kwargs
+ config_kwargs_to_add_to_suffix = {
+ k: config_kwargs_to_add_to_suffix[k] for k in sorted(config_kwargs_to_add_to_suffix)
+ }
+ if all(isinstance(v, (str, bool, int, float)) for v in config_kwargs_to_add_to_suffix.values()):
+ suffix = ",".join(
+ str(k) + "=" + urllib.parse.quote_plus(str(v)) for k, v in config_kwargs_to_add_to_suffix.items()
+ )
+ if len(suffix) > 32: # hash if too long
+ suffix = Hasher.hash(config_kwargs_to_add_to_suffix)
+ else:
+ suffix = Hasher.hash(config_kwargs_to_add_to_suffix)
+
+ if custom_features is not None:
+ m = Hasher()
+ if suffix:
+ m.update(suffix)
+ m.update(custom_features)
+ suffix = m.hexdigest()
+
+ if suffix:
+ config_id = self.name + "-" + suffix
+ if len(config_id) > config.MAX_DATASET_CONFIG_ID_READABLE_LENGTH:
+ config_id = self.name + "-" + Hasher.hash(suffix)
+ return config_id
+ else:
+ return self.name
+
+ def _resolve_data_files(self, base_path: str, download_config: DownloadConfig) -> None:
+ if isinstance(self.data_files, DataFilesPatternsDict):
+ base_path = xjoin(base_path, self.data_dir) if self.data_dir else base_path
+ self.data_files = self.data_files.resolve(base_path, download_config)
+
+
+class DatasetBuilder:
+ """Abstract base class for all datasets.
+
+ `DatasetBuilder` has 3 key methods:
+
+ - [`DatasetBuilder.info`]: Documents the dataset, including feature
+ names, types, shapes, version, splits, citation, etc.
+ - [`DatasetBuilder.download_and_prepare`]: Downloads the source data
+ and writes it to disk.
+ - [`DatasetBuilder.as_dataset`]: Generates a [`Dataset`].
+
+ Some `DatasetBuilder`s expose multiple variants of the
+ dataset by defining a [`BuilderConfig`] subclass and accepting a
+ config object (or name) on construction. Configurable datasets expose a
+ pre-defined set of configurations in [`DatasetBuilder.builder_configs`].
+
+ Args:
+ cache_dir (`str`, *optional*):
+ Directory to cache data. Defaults to `"~/.cache/huggingface/datasets"`.
+ dataset_name (`str`, *optional*):
+ Name of the dataset, if different from the builder name. Useful for packaged builders
+ like csv, imagefolder, audiofolder, etc. to reflect the difference between datasets
+ that use the same packaged builder.
+ config_name (`str`, *optional*):
+ Name of the dataset configuration.
+ It affects the data generated on disk. Different configurations will have their own subdirectories and
+ versions.
+ If not provided, the default configuration is used (if it exists).
+
+
+
+ Parameter `name` was renamed to `config_name`.
+
+
+ hash (`str`, *optional*):
+ Hash specific to the dataset builder code. Used to update the caching directory when the
+ dataset builder code is updated (to avoid reusing old data).
+ The typical caching directory (defined in `self._relative_data_dir`) is `name/version/hash/`.
+ base_path (`str`, *optional*):
+ Base path for relative paths that are used to download files.
+ This can be a remote URL.
+ features ([`Features`], *optional*):
+ Features types to use with this dataset.
+ It can be used to change the [`Features`] types of a dataset, for example.
+ token (`str` or `bool`, *optional*):
+ String or boolean to use as Bearer token for remote files on the
+ Datasets Hub. If `True`, will get token from `"~/.huggingface"`.
+ repo_id (`str`, *optional*):
+ ID of the dataset repository.
+ Used to distinguish builders with the same name but not coming from the same namespace, for example "rajpurkar/squad"
+ and "lhoestq/squad" repo IDs. In the latter, the builder name would be "lhoestq___squad".
+ data_files (`str` or `Sequence` or `Mapping`, *optional*):
+ Path(s) to source data file(s).
+ For builders like "csv" or "json" that need the user to specify data files. They can be either
+ local or remote files. For convenience, you can use a `DataFilesDict`.
+ data_dir (`str`, *optional*):
+ Path to directory containing source data file(s).
+ Use only if `data_files` is not passed, in which case it is equivalent to passing
+ `os.path.join(data_dir, "**")` as `data_files`.
+ For builders that require manual download, it must be the path to the local directory containing the
+ manually downloaded data.
+ storage_options (`dict`, *optional*):
+ Key/value pairs to be passed on to the dataset file-system backend, if any.
+ writer_batch_size (`int`, *optional*):
+ Batch size used by the ArrowWriter.
+ It defines the number of samples that are kept in memory before writing them
+ and also the length of the arrow chunks.
+ None means that the ArrowWriter will use its default value.
+ **config_kwargs (additional keyword arguments): Keyword arguments to be passed to the corresponding builder
+ configuration class, set on the class attribute [`DatasetBuilder.BUILDER_CONFIG_CLASS`]. The builder
+ configuration class is [`BuilderConfig`] or a subclass of it.
+ """
+
+ # Default version
+ VERSION = None # Default version set in BuilderConfig
+
+ # Class for the builder config.
+ BUILDER_CONFIG_CLASS = BuilderConfig
+
+ # Named configurations that modify the data generated by download_and_prepare.
+ BUILDER_CONFIGS = []
+
+ # Optional default config name to be used when name is None
+ DEFAULT_CONFIG_NAME = None
+
+ # Default batch size used by the ArrowWriter
+ # It defines the number of samples that are kept in memory before writing them
+ # and also the length of the arrow chunks
+ # None means that the ArrowWriter will use its default value
+ DEFAULT_WRITER_BATCH_SIZE = None
+
+ def __init__(
+ self,
+ cache_dir: Optional[str] = None,
+ dataset_name: Optional[str] = None,
+ config_name: Optional[str] = None,
+ hash: Optional[str] = None,
+ base_path: Optional[str] = None,
+ info: Optional[DatasetInfo] = None,
+ features: Optional[Features] = None,
+ token: Optional[Union[bool, str]] = None,
+ repo_id: Optional[str] = None,
+ data_files: Optional[Union[str, list, dict, DataFilesDict]] = None,
+ data_dir: Optional[str] = None,
+ storage_options: Optional[dict] = None,
+ writer_batch_size: Optional[int] = None,
+ config_id: Optional[str] = None,
+ **config_kwargs,
+ ):
+ # DatasetBuilder name
+ self.name: str = camelcase_to_snakecase(self.__module__.split(".")[-1])
+ self.hash: Optional[str] = hash
+ self.base_path = base_path
+ self.token = token
+ self.repo_id = repo_id
+ self.storage_options = storage_options or {}
+ self.dataset_name = camelcase_to_snakecase(dataset_name) if dataset_name else self.name
+ self._writer_batch_size = writer_batch_size or self.DEFAULT_WRITER_BATCH_SIZE
+
+ if data_files is not None and not isinstance(data_files, DataFilesDict):
+ data_files = DataFilesDict.from_patterns(
+ sanitize_patterns(data_files),
+ base_path=base_path,
+ download_config=DownloadConfig(token=token, storage_options=self.storage_options),
+ )
+
+ # Prepare config: DatasetConfig contains name, version and description but can be extended by each dataset
+ if "features" in inspect.signature(self.BUILDER_CONFIG_CLASS.__init__).parameters and features is not None:
+ config_kwargs["features"] = features
+ if data_files is not None:
+ config_kwargs["data_files"] = data_files
+ if data_dir is not None:
+ config_kwargs["data_dir"] = data_dir
+ self.config_kwargs = config_kwargs
+ self.config, self.config_id = self._create_builder_config(
+ config_name=config_name,
+ custom_features=features,
+ config_id=config_id,
+ **config_kwargs,
+ )
+
+ # prepare info: DatasetInfo are a standardized dataclass across all datasets
+ # Prefill datasetinfo
+ if info is None:
+ info = self._info()
+ info.builder_name = self.name
+ info.dataset_name = self.dataset_name
+ info.config_name = self.config.name
+ info.version = self.config.version
+ self.info = info
+ # update info with user specified infos
+ if features is not None:
+ self.info.features = features
+
+ # Prepare data dirs:
+ # cache_dir can be a remote bucket on GCS or S3
+ self._cache_dir_root = str(cache_dir or config.HF_DATASETS_CACHE)
+ self._cache_dir_root = (
+ self._cache_dir_root if is_remote_url(self._cache_dir_root) else os.path.expanduser(self._cache_dir_root)
+ )
+ self._cache_downloaded_dir = (
+ posixpath.join(self._cache_dir_root, config.DOWNLOADED_DATASETS_DIR)
+ if cache_dir
+ else str(config.DOWNLOADED_DATASETS_PATH)
+ )
+ self._cache_downloaded_dir = (
+ self._cache_downloaded_dir
+ if is_remote_url(self._cache_downloaded_dir)
+ else os.path.expanduser(self._cache_downloaded_dir)
+ )
+
+ # In case there exists a legacy cache directory
+ self._legacy_relative_data_dir = None
+
+ self._cache_dir = self._build_cache_dir()
+ if not is_remote_url(self._cache_dir_root):
+ os.makedirs(self._cache_dir_root, exist_ok=True)
+ lock_path = os.path.join(
+ self._cache_dir_root, Path(self._cache_dir).as_posix().replace("/", "_") + ".lock"
+ )
+ with FileLock(lock_path):
+ if os.path.exists(self._cache_dir): # check if data exist
+ if len(os.listdir(self._cache_dir)) > 0:
+ if os.path.exists(os.path.join(self._cache_dir, config.DATASET_INFO_FILENAME)):
+ logger.debug("Overwrite dataset info from restored data version if exists.")
+ self.info = DatasetInfo.from_directory(self._cache_dir)
+ else: # dir exists but no data, remove the empty dir as data aren't available anymore
+ logger.warning(
+ f"Old caching folder {self._cache_dir} for dataset {self.dataset_name} exists but no data were found. Removing it. "
+ )
+ os.rmdir(self._cache_dir)
+
+ # Store in the cache by default unless the user specifies a custom output_dir to download_and_prepare
+ self._output_dir = self._cache_dir
+ self._fs: fsspec.AbstractFileSystem = fsspec.filesystem("file")
+
+ # Set download manager
+ self.dl_manager = None
+
+ # Set to True by "datasets-cli test" to generate file checksums for (deprecated) dataset_infos.json independently of verification_mode value.
+ self._record_infos = False
+
+ # Set in `.download_and_prepare` once the format of the generated dataset is known
+ self._file_format = None
+
+ # Enable streaming (e.g. it patches "open" to work with remote files)
+ extend_dataset_builder_for_streaming(self)
+
+ def __getstate__(self):
+ return self.__dict__
+
+ def __setstate__(self, d):
+ self.__dict__ = d
+ # Re-enable streaming, since patched functions are not kept when pickling
+ extend_dataset_builder_for_streaming(self)
+
+ # Must be set for datasets that use 'data_dir' functionality - the ones
+ # that require users to do additional steps to download the data
+ # (this is usually due to some external regulations / rules).
+ # This field should contain a string with user instructions, including
+ # the list of files that should be present. It will be
+ # displayed in the dataset documentation.
+ @property
+ def manual_download_instructions(self) -> Optional[str]:
+ return None
+
+ def _check_legacy_cache(self) -> Optional[str]:
+ """Check for the old cache directory template {cache_dir}/{namespace}___{builder_name} from 2.13"""
+ if (
+ self.__module__.startswith("datasets.")
+ and not is_remote_url(self._cache_dir_root)
+ and self.config.name == "default"
+ ):
+ from .packaged_modules import _PACKAGED_DATASETS_MODULES
+
+ namespace = self.repo_id.split("/")[0] if self.repo_id and self.repo_id.count("/") > 0 else None
+ config_name = self.repo_id.replace("/", "--") if self.repo_id is not None else self.dataset_name
+ config_id = config_name + self.config_id[len(self.config.name) :]
+ hash = _PACKAGED_DATASETS_MODULES.get(self.name, "missing")[1]
+ legacy_relative_data_dir = posixpath.join(
+ self.dataset_name if namespace is None else f"{namespace}___{self.dataset_name}",
+ config_id,
+ "0.0.0",
+ hash,
+ )
+ legacy_cache_dir = posixpath.join(self._cache_dir_root, legacy_relative_data_dir)
+ if os.path.isdir(legacy_cache_dir):
+ return legacy_relative_data_dir
+
+ def _check_legacy_cache2(self, dataset_module: "DatasetModule") -> Optional[str]:
+ """Check for the old cache directory template {cache_dir}/{namespace}___{dataset_name}/{config_name}-xxx from 2.14 and 2.15"""
+ if (
+ self.__module__.startswith("datasets.")
+ and not is_remote_url(self._cache_dir_root)
+ and not (set(self.config_kwargs) - {"data_files", "data_dir"})
+ ):
+ from .packaged_modules import _PACKAGED_DATASETS_MODULES_2_15_HASHES
+ from .utils._dill import Pickler
+
+ def update_hash_with_config_parameters(hash: str, config_parameters: dict) -> str:
+ """
+ Used to update hash of packaged modules which is used for creating unique cache directories to reflect
+ different config parameters which are passed in metadata from readme.
+ """
+ params_to_exclude = {"config_name", "version", "description"}
+ params_to_add_to_hash = {
+ param: value
+ for param, value in sorted(config_parameters.items())
+ if param not in params_to_exclude
+ }
+ m = Hasher()
+ m.update(hash)
+ m.update(params_to_add_to_hash)
+ return m.hexdigest()
+
+ namespace = self.repo_id.split("/")[0] if self.repo_id and self.repo_id.count("/") > 0 else None
+ with patch.object(Pickler, "_legacy_no_dict_keys_sorting", True):
+ config_id = self.config.name + "-" + Hasher.hash({"data_files": self.config.data_files})
+ hash = _PACKAGED_DATASETS_MODULES_2_15_HASHES.get(self.name, "missing")
+ if (
+ dataset_module.builder_configs_parameters.metadata_configs
+ and self.config.name in dataset_module.builder_configs_parameters.metadata_configs
+ ):
+ hash = update_hash_with_config_parameters(
+ hash, dataset_module.builder_configs_parameters.metadata_configs[self.config.name]
+ )
+ legacy_relative_data_dir = posixpath.join(
+ self.dataset_name if namespace is None else f"{namespace}___{self.dataset_name}",
+ config_id,
+ "0.0.0",
+ hash,
+ )
+ legacy_cache_dir = posixpath.join(self._cache_dir_root, legacy_relative_data_dir)
+ if os.path.isdir(legacy_cache_dir):
+ return legacy_relative_data_dir
+
+ def _create_builder_config(
+ self, config_name=None, custom_features=None, config_id=None, **config_kwargs
+ ) -> tuple[BuilderConfig, str]:
+ """Create and validate BuilderConfig object as well as a unique config id for this config.
+ Raises ValueError if there are multiple builder configs and config_name and DEFAULT_CONFIG_NAME are None.
+ config_kwargs override the defaults kwargs in config
+ """
+ builder_config = None
+
+ # try default config
+ if config_name is None and self.BUILDER_CONFIGS:
+ if self.DEFAULT_CONFIG_NAME is not None:
+ builder_config = self.builder_configs.get(self.DEFAULT_CONFIG_NAME)
+ logger.info(f"No config specified, defaulting to: {self.dataset_name}/{builder_config.name}")
+ else:
+ if len(self.BUILDER_CONFIGS) > 1:
+ if not config_kwargs:
+ example_of_usage = (
+ f"load_dataset('{self.repo_id or self.dataset_name}', '{self.BUILDER_CONFIGS[0].name}')"
+ )
+ raise ValueError(
+ "Config name is missing."
+ f"\nPlease pick one among the available configs: {list(self.builder_configs.keys())}"
+ + f"\nExample of usage:\n\t`{example_of_usage}`"
+ )
+ else:
+ builder_config = self.BUILDER_CONFIGS[0]
+ logger.info(
+ f"No config specified, defaulting to the single config: {self.dataset_name}/{builder_config.name}"
+ )
+
+ # try to get config by name
+ if isinstance(config_name, str):
+ builder_config = self.builder_configs.get(config_name)
+ if builder_config is None and self.BUILDER_CONFIGS:
+ raise ValueError(
+ f"BuilderConfig '{config_name}' not found. Available: {list(self.builder_configs.keys())}"
+ )
+
+ # if not using an existing config, then create a new config on the fly
+ if not builder_config:
+ if config_name is not None:
+ config_kwargs["name"] = config_name
+ elif self.DEFAULT_CONFIG_NAME and not config_kwargs:
+ # Use DEFAULT_CONFIG_NAME only if no config_kwargs are passed
+ config_kwargs["name"] = self.DEFAULT_CONFIG_NAME
+ if "version" not in config_kwargs and hasattr(self, "VERSION") and self.VERSION:
+ config_kwargs["version"] = self.VERSION
+ builder_config = self.BUILDER_CONFIG_CLASS(**config_kwargs)
+
+ # otherwise use the config_kwargs to overwrite the attributes
+ else:
+ builder_config = copy.deepcopy(builder_config) if config_kwargs else builder_config
+ for key, value in config_kwargs.items():
+ if value is not None:
+ if not hasattr(builder_config, key):
+ raise ValueError(f"BuilderConfig {builder_config} doesn't have a '{key}' key.")
+ setattr(builder_config, key, value)
+
+ if not builder_config.name:
+ raise ValueError(f"BuilderConfig must have a name, got {builder_config.name}")
+
+ # resolve data files if needed
+ builder_config._resolve_data_files(
+ base_path=self.base_path,
+ download_config=DownloadConfig(token=self.token, storage_options=self.storage_options),
+ )
+
+ # compute the config id that is going to be used for caching
+ if config_id is None:
+ config_id = builder_config.create_config_id(
+ config_kwargs,
+ custom_features=custom_features,
+ )
+ is_custom = (config_id not in self.builder_configs) and config_id != "default"
+ if is_custom:
+ logger.info(f"Using custom data configuration {config_id}")
+ else:
+ if (
+ builder_config.name in self.builder_configs
+ and builder_config != self.builder_configs[builder_config.name]
+ ):
+ raise ValueError(
+ "Cannot name a custom BuilderConfig the same as an available "
+ f"BuilderConfig. Change the name. Available BuilderConfigs: {list(self.builder_configs.keys())}"
+ )
+ if not builder_config.version:
+ raise ValueError(f"BuilderConfig {builder_config.name} must have a version")
+
+ return builder_config, config_id
+
+ @classproperty
+ @classmethod
+ @memoize()
+ def builder_configs(cls) -> dict[str, BuilderConfig]:
+ """Dictionary of pre-defined configurations for this builder class."""
+ configs = {config.name: config for config in cls.BUILDER_CONFIGS}
+ if len(configs) != len(cls.BUILDER_CONFIGS):
+ names = [config.name for config in cls.BUILDER_CONFIGS]
+ raise ValueError(f"Names in BUILDER_CONFIGS must not be duplicated. Got {names}")
+ return configs
+
+ @property
+ def cache_dir(self):
+ return self._cache_dir
+
+ def _use_legacy_cache_dir_if_possible(self, dataset_module: "DatasetModule"):
+ # Check for the legacy cache directory template (datasets<3.0.0)
+ self._legacy_relative_data_dir = (
+ self._check_legacy_cache2(dataset_module) or self._check_legacy_cache() or None
+ )
+ self._cache_dir = self._build_cache_dir()
+ self._output_dir = self._cache_dir
+
+ def _relative_data_dir(self, with_version=True, with_hash=True) -> str:
+ """Relative path of this dataset in cache_dir:
+ Will be:
+ self.dataset_name/self.config.version/self.hash/
+ or if a repo_id with a namespace has been specified:
+ self.namespace___self.dataset_name/self.config.version/self.hash/
+ If any of these element is missing or if ``with_version=False`` the corresponding subfolders are dropped.
+ """
+ if self._legacy_relative_data_dir is not None and with_version and with_hash:
+ return self._legacy_relative_data_dir
+
+ namespace = self.repo_id.split("/")[0] if self.repo_id and self.repo_id.count("/") > 0 else None
+ builder_data_dir = self.dataset_name if namespace is None else f"{namespace}___{self.dataset_name}"
+ builder_data_dir = posixpath.join(builder_data_dir, self.config_id)
+ if with_version:
+ builder_data_dir = posixpath.join(builder_data_dir, str(self.config.version))
+ if with_hash and self.hash and isinstance(self.hash, str):
+ builder_data_dir = posixpath.join(builder_data_dir, self.hash)
+ return builder_data_dir
+
+ def _build_cache_dir(self):
+ """Return the data directory for the current version."""
+ builder_data_dir = posixpath.join(self._cache_dir_root, self._relative_data_dir(with_version=False))
+ version_data_dir = posixpath.join(self._cache_dir_root, self._relative_data_dir(with_version=True))
+
+ def _other_versions_on_disk():
+ """Returns previous versions on disk."""
+ if not os.path.exists(builder_data_dir):
+ return []
+
+ version_dirnames = []
+ for dir_name in os.listdir(builder_data_dir):
+ try:
+ version_dirnames.append((utils.Version(dir_name), dir_name))
+ except ValueError: # Invalid version (ex: incomplete data dir)
+ pass
+ version_dirnames.sort(reverse=True)
+ return version_dirnames
+
+ # Check and warn if other versions exist
+ if not is_remote_url(builder_data_dir):
+ version_dirs = _other_versions_on_disk()
+ if version_dirs:
+ other_version = version_dirs[0][0]
+ if other_version != self.config.version:
+ warn_msg = (
+ f"Found a different version {str(other_version)} of dataset {self.dataset_name} in "
+ f"cache_dir {self._cache_dir_root}. Using currently defined version "
+ f"{str(self.config.version)}."
+ )
+ logger.warning(warn_msg)
+
+ return version_data_dir
+
+ @abc.abstractmethod
+ def _info(self) -> DatasetInfo:
+ """Construct the DatasetInfo object. See `DatasetInfo` for details.
+
+ Warning: This function is only called once and the result is cached for all
+ following .info() calls.
+
+ Returns:
+ info: (DatasetInfo) The dataset information
+ """
+ raise NotImplementedError
+
+ @classmethod
+ def get_imported_module_dir(cls):
+ """Return the path of the module of this class or subclass."""
+ return os.path.dirname(inspect.getfile(inspect.getmodule(cls)))
+
+ def _rename(self, src: str, dst: str):
+ rename(self._fs, src, dst)
+
+ def download_and_prepare(
+ self,
+ output_dir: Optional[str] = None,
+ download_config: Optional[DownloadConfig] = None,
+ download_mode: Optional[Union[DownloadMode, str]] = None,
+ verification_mode: Optional[Union[VerificationMode, str]] = None,
+ dl_manager: Optional[DownloadManager] = None,
+ base_path: Optional[str] = None,
+ file_format: str = "arrow",
+ max_shard_size: Optional[Union[int, str]] = None,
+ num_proc: Optional[int] = None,
+ storage_options: Optional[dict] = None,
+ **download_and_prepare_kwargs,
+ ):
+ """Downloads and prepares dataset for reading.
+
+ Args:
+ output_dir (`str`, *optional*):
+ Output directory for the dataset.
+ Default to this builder's `cache_dir`, which is inside `~/.cache/huggingface/datasets` by default.
+
+
+ download_config (`DownloadConfig`, *optional*):
+ Specific download configuration parameters.
+ download_mode ([`DownloadMode`] or `str`, *optional*):
+ Select the download/generate mode, default to `REUSE_DATASET_IF_EXISTS`.
+ verification_mode ([`VerificationMode`] or `str`, defaults to `BASIC_CHECKS`):
+ Verification mode determining the checks to run on the downloaded/processed dataset information (checksums/size/splits/...).
+
+
+ dl_manager (`DownloadManager`, *optional*):
+ Specific `DownloadManger` to use.
+ base_path (`str`, *optional*):
+ Base path for relative paths that are used to download files. This can be a remote url.
+ If not specified, the value of the `base_path` attribute (`self.base_path`) will be used instead.
+ file_format (`str`, *optional*):
+ Format of the data files in which the dataset will be written.
+ Supported formats: "arrow", "parquet". Default to "arrow" format.
+ If the format is "parquet", then image and audio data are embedded into the Parquet files instead of pointing to local files.
+
+
+ max_shard_size (`Union[str, int]`, *optional*):
+ Maximum number of bytes written per shard, default is "500MB".
+ The size is based on uncompressed data size, so in practice your shard files may be smaller than
+ `max_shard_size` thanks to Parquet compression for example.
+
+
+ num_proc (`int`, *optional*, defaults to `None`):
+ Number of processes when downloading and generating the dataset locally.
+ Multiprocessing is disabled by default.
+
+
+ storage_options (`dict`, *optional*):
+ Key/value pairs to be passed on to the caching file-system backend, if any.
+
+
+ **download_and_prepare_kwargs (additional keyword arguments): Keyword arguments.
+
+ Example:
+
+ Download and prepare the dataset as Arrow files that can be loaded as a Dataset using `builder.as_dataset()`:
+
+ ```py
+ >>> from datasets import load_dataset_builder
+ >>> builder = load_dataset_builder("cornell-movie-review-data/rotten_tomatoes")
+ >>> builder.download_and_prepare()
+ ```
+
+ Download and prepare the dataset as sharded Parquet files locally:
+
+ ```py
+ >>> from datasets import load_dataset_builder
+ >>> builder = load_dataset_builder("cornell-movie-review-data/rotten_tomatoes")
+ >>> builder.download_and_prepare("./output_dir", file_format="parquet")
+ ```
+
+ Download and prepare the dataset as sharded Parquet files in a cloud storage:
+
+ ```py
+ >>> from datasets import load_dataset_builder
+ >>> storage_options = {"key": aws_access_key_id, "secret": aws_secret_access_key}
+ >>> builder = load_dataset_builder("cornell-movie-review-data/rotten_tomatoes")
+ >>> builder.download_and_prepare("s3://my-bucket/my_rotten_tomatoes", storage_options=storage_options, file_format="parquet")
+ ```
+ """
+ output_dir = output_dir if output_dir is not None else self._cache_dir
+ # output_dir can be a remote bucket on GCS or S3
+ fs, output_dir = url_to_fs(output_dir, **(storage_options or {}))
+ self._fs = fs
+ self._output_dir = output_dir if not is_remote_filesystem(self._fs) else self._fs.unstrip_protocol(output_dir)
+
+ download_mode = DownloadMode(download_mode or DownloadMode.REUSE_DATASET_IF_EXISTS)
+ verification_mode = VerificationMode(verification_mode or VerificationMode.BASIC_CHECKS)
+ base_path = base_path if base_path is not None else self.base_path
+
+ if file_format is not None and file_format not in ["arrow", "parquet"]:
+ raise ValueError(f"Unsupported file_format: {file_format}. Expected 'arrow' or 'parquet'")
+ self._file_format = file_format
+
+ if self._fs._strip_protocol(self._output_dir) == "":
+ # We don't support the root directory, because it has no dirname,
+ # and we need a dirname to use a .incomplete directory
+ # when the dataset is being written
+ raise RuntimeError(
+ f"Unable to download and prepare the dataset at the root {self._output_dir}. "
+ f"Please specify a subdirectory, e.g. '{self._output_dir + self.dataset_name}'"
+ )
+
+ if dl_manager is None:
+ if download_config is None:
+ download_config = DownloadConfig(
+ cache_dir=self._cache_downloaded_dir,
+ force_download=download_mode == DownloadMode.FORCE_REDOWNLOAD,
+ force_extract=download_mode == DownloadMode.FORCE_REDOWNLOAD,
+ use_etag=False,
+ num_proc=num_proc,
+ token=self.token,
+ storage_options=self.storage_options,
+ ) # We don't use etag for data files to speed up the process
+
+ dl_manager = DownloadManager(
+ dataset_name=self.dataset_name,
+ download_config=download_config,
+ data_dir=self.config.data_dir,
+ base_path=base_path,
+ record_checksums=(self._record_infos or verification_mode == VerificationMode.ALL_CHECKS),
+ )
+
+ is_local = not is_remote_filesystem(self._fs)
+ self.dl_manager = dl_manager
+
+ # Prevent parallel local disk operations
+ if is_local:
+ # Create parent directory of the output_dir to put the lock file in there
+ Path(self._output_dir).parent.mkdir(parents=True, exist_ok=True)
+ lock_path = self._output_dir + "_builder.lock"
+
+ # File locking only with local paths; no file locking on GCS or S3
+ with FileLock(lock_path) if is_local else contextlib.nullcontext():
+ # Check if the data already exists
+ data_exists = self._fs.exists(posixpath.join(self._output_dir, config.DATASET_INFO_FILENAME))
+ if data_exists and download_mode == DownloadMode.REUSE_DATASET_IF_EXISTS:
+ logger.info(f"Found cached dataset {self.dataset_name} ({self._output_dir})")
+ # We need to update the info in case some splits were added in the meantime
+ # for example when calling load_dataset from multiple workers.
+ self.info = self._load_info()
+ self.download_post_processing_resources(dl_manager)
+ return
+
+ logger.info(f"Generating dataset {self.dataset_name} ({self._output_dir})")
+ if is_local: # if cache dir is local, check for available space
+ if not has_sufficient_disk_space(
+ self.info.size_in_bytes or 0, directory=Path(self._output_dir).parent
+ ):
+ raise OSError(
+ f"Not enough disk space. Needed: {size_str(self.info.size_in_bytes or 0)} (download: {size_str(self.info.download_size or 0)}, generated: {size_str(self.info.dataset_size or 0)}, post-processed: {size_str(self.info.post_processing_size or 0)})"
+ )
+
+ @contextlib.contextmanager
+ def incomplete_dir(dirname):
+ """Create temporary dir for dirname and rename on exit."""
+ if not is_local:
+ self._fs.makedirs(dirname, exist_ok=True)
+ yield dirname
+ else:
+ tmp_dir = dirname + ".incomplete"
+ os.makedirs(tmp_dir, exist_ok=True)
+ try:
+ yield tmp_dir
+ if os.path.isdir(dirname):
+ shutil.rmtree(dirname)
+ # LocalFileSystem.mv does copy + rm, it is more efficient to simply rename a local directory
+ shutil.move(tmp_dir, dirname)
+ finally:
+ if os.path.exists(tmp_dir):
+ shutil.rmtree(tmp_dir)
+
+ # Print is intentional: we want this to always go to stdout so user has
+ # information needed to cancel download/preparation if needed.
+ # This comes right before the progress bar.
+ if self.info.size_in_bytes:
+ logger.info(
+ f"Downloading and preparing dataset {self.dataset_name}/{self.config.name} "
+ f"(download: {size_str(self.info.download_size)}, generated: {size_str(self.info.dataset_size)}, "
+ f"post-processed: {size_str(self.info.post_processing_size)}, "
+ f"total: {size_str(self.info.size_in_bytes)}) to {self._output_dir}..."
+ )
+ else:
+ _dest = self._fs._strip_protocol(self._output_dir) if is_local else self._output_dir
+ logger.info(f"Downloading and preparing dataset {self.dataset_name}/{self.config.name} to {_dest}...")
+
+ self._check_manual_download(dl_manager)
+
+ # Create a tmp dir and rename to self._output_dir on successful exit.
+ with incomplete_dir(self._output_dir) as tmp_output_dir:
+ # Temporarily assign _output_dir to tmp_data_dir to avoid having to forward
+ # it to every sub function.
+ with temporary_assignment(self, "_output_dir", tmp_output_dir):
+ prepare_split_kwargs = {"file_format": file_format}
+ if max_shard_size is not None:
+ prepare_split_kwargs["max_shard_size"] = max_shard_size
+ if num_proc is not None:
+ prepare_split_kwargs["num_proc"] = num_proc
+ self._download_and_prepare(
+ dl_manager=dl_manager,
+ verification_mode=verification_mode,
+ **prepare_split_kwargs,
+ **download_and_prepare_kwargs,
+ )
+ # Sync info
+ self.info.dataset_size = sum(split.num_bytes for split in self.info.splits.values())
+ self.info.download_checksums = dl_manager.get_recorded_sizes_checksums()
+ if self.info.download_size is not None:
+ self.info.size_in_bytes = self.info.dataset_size + self.info.download_size
+ # Save info
+ self._save_info()
+
+ # Download post processing resources
+ self.download_post_processing_resources(dl_manager)
+
+ logger.info(
+ f"Dataset {self.dataset_name} downloaded and prepared to {self._output_dir}. "
+ f"Subsequent calls will reuse this data."
+ )
+
+ def _check_manual_download(self, dl_manager):
+ if self.manual_download_instructions is not None and dl_manager.manual_dir is None:
+ raise ManualDownloadError(
+ textwrap.dedent(
+ f"""\
+ The dataset {self.dataset_name} with config {self.config.name} requires manual data.
+ Please follow the manual download instructions:
+ {self.manual_download_instructions}
+ Manual data can be loaded with:
+ datasets.load_dataset("{self.repo_id or self.dataset_name}", data_dir="")"""
+ )
+ )
+
+ def _download_and_prepare(self, dl_manager, verification_mode, **prepare_split_kwargs):
+ """Downloads and prepares dataset for reading.
+
+ This is the internal implementation to overwrite called when user calls
+ `download_and_prepare`. It should download all required data and generate
+ the pre-processed datasets files.
+
+ Args:
+ dl_manager ([`DownloadManager`]):
+ `DownloadManager` used to download and cache data.
+ verification_mode ([`VerificationMode`]):
+ if `ALL_CHECKS`, perform all the verifications including checksums.
+ if `BASIC_CHECKS`, do not perform checksums, only perform split tests.
+ if `NO_CHECKS`, do not perform any verification.
+ prepare_split_kwargs: Additional options, such as `file_format`, `max_shard_size`
+ """
+ # Generating data for all splits
+ split_dict = SplitDict(dataset_name=self.dataset_name)
+ split_generators_kwargs = self._make_split_generators_kwargs(prepare_split_kwargs)
+ split_generators = self._split_generators(dl_manager, **split_generators_kwargs)
+
+ # Checksums verification
+ if verification_mode == VerificationMode.ALL_CHECKS and dl_manager.record_checksums:
+ verify_checksums(
+ self.info.download_checksums, dl_manager.get_recorded_sizes_checksums(), "dataset source files"
+ )
+
+ # Build splits
+ for split_generator in split_generators:
+ if str(split_generator.split_info.name).lower() == "all":
+ raise ValueError(
+ "`all` is a special split keyword corresponding to the "
+ "union of all splits, so cannot be used as key in "
+ "._split_generator()."
+ )
+
+ logger.info(f"Generating {split_generator.split_info.name} split")
+ split_dict.add(split_generator.split_info)
+
+ try:
+ # Prepare split will record examples associated to the split
+ self._prepare_split(split_generator, **prepare_split_kwargs)
+ except OSError as e:
+ raise OSError(
+ "Cannot find data file. "
+ + (self.manual_download_instructions or "")
+ + "\nOriginal error:\n"
+ + str(e)
+ ) from None
+ # If check_duplicates is set to True , then except DuplicatedKeysError
+ except DuplicatedKeysError as e:
+ raise DuplicatedKeysError(
+ e.key,
+ e.duplicate_key_indices,
+ fix_msg=f"To avoid duplicate keys, please fix the dataset splits for {self.name}",
+ ) from None
+ dl_manager.manage_extracted_files()
+
+ if verification_mode == VerificationMode.BASIC_CHECKS or verification_mode == VerificationMode.ALL_CHECKS:
+ verify_splits(self.info.splits, split_dict)
+
+ # Update the info object with the splits.
+ self.info.splits = split_dict
+ self.info.download_size = dl_manager.downloaded_size
+
+ def download_post_processing_resources(self, dl_manager):
+ for split in self.info.splits or []:
+ for resource_name, resource_file_name in self._post_processing_resources(split).items():
+ if not not is_remote_filesystem(self._fs):
+ raise NotImplementedError(f"Post processing is not supported on filesystem {self._fs}")
+ if os.sep in resource_file_name:
+ raise ValueError(f"Resources shouldn't be in a sub-directory: {resource_file_name}")
+ resource_path = os.path.join(self._output_dir, resource_file_name)
+ if not os.path.exists(resource_path):
+ downloaded_resource_path = self._download_post_processing_resources(
+ split, resource_name, dl_manager
+ )
+ if downloaded_resource_path:
+ logger.info(f"Downloaded post-processing resource {resource_name} as {resource_file_name}")
+ shutil.move(downloaded_resource_path, resource_path)
+
+ def _load_info(self) -> DatasetInfo:
+ return DatasetInfo.from_directory(self._output_dir, storage_options=self._fs.storage_options)
+
+ def _save_info(self):
+ file_lock = (
+ FileLock(self._output_dir + "_info.lock")
+ if not is_remote_filesystem(self._fs)
+ else contextlib.nullcontext()
+ )
+ with file_lock:
+ self.info.write_to_directory(self._output_dir, storage_options=self._fs.storage_options)
+
+ def _make_split_generators_kwargs(self, prepare_split_kwargs):
+ """Get kwargs for `self._split_generators()` from `prepare_split_kwargs`."""
+ del prepare_split_kwargs
+ return {}
+
+ def as_dataset(
+ self,
+ split: Optional[Union[str, Split, list[str], list[Split]]] = None,
+ run_post_process=True,
+ verification_mode: Optional[Union[VerificationMode, str]] = None,
+ in_memory=False,
+ ) -> Union[Dataset, DatasetDict]:
+ """Return a Dataset for the specified split.
+
+ Args:
+ split (`datasets.Split`):
+ Which subset of the data to return.
+ run_post_process (`bool`, defaults to `True`):
+ Whether to run post-processing dataset transforms and/or add
+ indexes.
+ verification_mode ([`VerificationMode`] or `str`, defaults to `BASIC_CHECKS`):
+ Verification mode determining the checks to run on the
+ downloaded/processed dataset information (checksums/size/splits/...).
+
+
+ in_memory (`bool`, defaults to `False`):
+ Whether to copy the data in-memory.
+
+ Returns:
+ datasets.Dataset
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset_builder
+ >>> builder = load_dataset_builder('cornell-movie-review-data/rotten_tomatoes')
+ >>> builder.download_and_prepare()
+ >>> ds = builder.as_dataset(split='train')
+ >>> ds
+ Dataset({
+ features: ['text', 'label'],
+ num_rows: 8530
+ })
+ ```
+ """
+ if self._file_format is not None and self._file_format != "arrow":
+ raise FileFormatError('Loading a dataset not written in the "arrow" format is not supported.')
+ if is_remote_filesystem(self._fs):
+ raise NotImplementedError(f"Loading a dataset cached in a {type(self._fs).__name__} is not supported.")
+ if not os.path.exists(self._output_dir):
+ raise FileNotFoundError(
+ f"Dataset {self.dataset_name}: could not find data in {self._output_dir}. Please make sure to call "
+ "builder.download_and_prepare(), or use "
+ "datasets.load_dataset() before trying to access the Dataset object."
+ )
+
+ logger.debug(f"Constructing Dataset for split {split or ', '.join(self.info.splits)}, from {self._output_dir}")
+
+ # By default, return all splits
+ if split is None:
+ split = {s: s for s in self.info.splits}
+
+ verification_mode = VerificationMode(verification_mode or VerificationMode.BASIC_CHECKS)
+
+ # Create a dataset for each of the given splits
+ datasets = map_nested(
+ partial(
+ self._build_single_dataset,
+ run_post_process=run_post_process,
+ verification_mode=verification_mode,
+ in_memory=in_memory,
+ ),
+ split,
+ map_tuple=True,
+ disable_tqdm=True,
+ )
+ if isinstance(datasets, dict):
+ datasets = DatasetDict(datasets)
+ return datasets
+
+ def _build_single_dataset(
+ self,
+ split: Union[str, ReadInstruction, Split],
+ run_post_process: bool,
+ verification_mode: VerificationMode,
+ in_memory: bool = False,
+ ):
+ """as_dataset for a single split."""
+ if not isinstance(split, ReadInstruction):
+ split = str(split)
+ if split == "all":
+ split = "+".join(self.info.splits.keys())
+ split = Split(split)
+
+ # Build base dataset
+ ds = self._as_dataset(
+ split=split,
+ in_memory=in_memory,
+ )
+ if run_post_process:
+ for resource_file_name in self._post_processing_resources(split).values():
+ if os.sep in resource_file_name:
+ raise ValueError(f"Resources shouldn't be in a sub-directory: {resource_file_name}")
+ resources_paths = {
+ resource_name: os.path.join(self._output_dir, resource_file_name)
+ for resource_name, resource_file_name in self._post_processing_resources(split).items()
+ }
+ post_processed = self._post_process(ds, resources_paths)
+ if post_processed is not None:
+ ds = post_processed
+ recorded_checksums = {}
+ record_checksums = False
+ for resource_name, resource_path in resources_paths.items():
+ size_checksum = get_size_checksum_dict(resource_path)
+ recorded_checksums[resource_name] = size_checksum
+ if verification_mode == VerificationMode.ALL_CHECKS and record_checksums:
+ if self.info.post_processed is None or self.info.post_processed.resources_checksums is None:
+ expected_checksums = None
+ else:
+ expected_checksums = self.info.post_processed.resources_checksums.get(split)
+ verify_checksums(expected_checksums, recorded_checksums, "post processing resources")
+ if self.info.post_processed is None:
+ self.info.post_processed = PostProcessedInfo()
+ if self.info.post_processed.resources_checksums is None:
+ self.info.post_processed.resources_checksums = {}
+ self.info.post_processed.resources_checksums[str(split)] = recorded_checksums
+ self.info.post_processing_size = sum(
+ checksums_dict["num_bytes"]
+ for split_checksums_dicts in self.info.post_processed.resources_checksums.values()
+ for checksums_dict in split_checksums_dicts.values()
+ )
+ if self.info.dataset_size is not None and self.info.download_size is not None:
+ self.info.size_in_bytes = (
+ self.info.dataset_size + self.info.download_size + self.info.post_processing_size
+ )
+ self._save_info()
+ ds._info.post_processed = self.info.post_processed
+ ds._info.post_processing_size = self.info.post_processing_size
+ ds._info.size_in_bytes = self.info.size_in_bytes
+ if self.info.post_processed.features is not None:
+ if self.info.post_processed.features.type != ds.features.type:
+ raise ValueError(
+ f"Post-processed features info don't match the dataset:\nGot\n{self.info.post_processed.features}\nbut expected something like\n{ds.features}"
+ )
+ else:
+ ds.info.features = self.info.post_processed.features
+
+ return ds
+
+ def _as_dataset(self, split: Union[ReadInstruction, Split] = Split.TRAIN, in_memory: bool = False) -> Dataset:
+ """Constructs a `Dataset`.
+
+ This is the internal implementation to overwrite called when user calls
+ `as_dataset`. It should read the pre-processed datasets files and generate
+ the `Dataset` object.
+
+ Args:
+ split (`datasets.Split`):
+ which subset of the data to read.
+ in_memory (`bool`, defaults to `False`):
+ Whether to copy the data in-memory.
+
+ Returns:
+ `Dataset`
+ """
+ cache_dir = self._fs._strip_protocol(self._output_dir)
+ dataset_name = self.dataset_name
+ if self._check_legacy_cache():
+ dataset_name = self.name
+ dataset_kwargs = ArrowReader(cache_dir, self.info).read(
+ name=dataset_name,
+ instructions=split,
+ split_infos=self.info.splits.values(),
+ in_memory=in_memory,
+ )
+ fingerprint = self._get_dataset_fingerprint(split)
+ return Dataset(fingerprint=fingerprint, **dataset_kwargs)
+
+ def _get_dataset_fingerprint(self, split: Union[ReadInstruction, Split]) -> str:
+ """The dataset fingerprint is the hash of the relative directory dataset_name/config_name/version/hash, as well as the split specs."""
+ hasher = Hasher()
+ hasher.update(Path(self._relative_data_dir()).as_posix())
+ hasher.update(str(split)) # for example: train, train+test, train[:10%], test[:33%](pct1_dropremainder)
+ fingerprint = hasher.hexdigest()
+ return fingerprint
+
+ def as_streaming_dataset(
+ self,
+ split: Optional[str] = None,
+ base_path: Optional[str] = None,
+ ) -> Union[dict[str, IterableDataset], IterableDataset]:
+ if is_remote_filesystem(self._fs):
+ raise NotImplementedError(
+ f"Loading a streaming dataset cached in a {type(self._fs).__name__} is not supported yet."
+ )
+
+ dl_manager = StreamingDownloadManager(
+ base_path=base_path or self.base_path,
+ download_config=DownloadConfig(token=self.token, storage_options=self.storage_options),
+ dataset_name=self.dataset_name,
+ data_dir=self.config.data_dir,
+ )
+ self._check_manual_download(dl_manager)
+ splits_generators = {sg.name: sg for sg in self._split_generators(dl_manager)}
+ # By default, return all splits
+ if split is None:
+ splits_generator = splits_generators
+ elif split in splits_generators:
+ splits_generator = splits_generators[split]
+ else:
+ raise ValueError(f"Bad split: {split}. Available splits: {list(splits_generators)}")
+
+ # Create a dataset for each of the given splits
+ datasets = map_nested(
+ self._as_streaming_dataset_single,
+ splits_generator,
+ map_tuple=True,
+ )
+ if isinstance(datasets, dict):
+ datasets = IterableDatasetDict(datasets)
+ return datasets
+
+ def _as_streaming_dataset_single(
+ self,
+ splits_generator,
+ ) -> IterableDataset:
+ ex_iterable = self._get_examples_iterable_for_split(splits_generator)
+ # add auth to be able to access and decode audio/image files from private repositories.
+ token_per_repo_id = {self.repo_id: self.token} if self.repo_id else {}
+ return IterableDataset(
+ ex_iterable, info=self.info, split=splits_generator.name, token_per_repo_id=token_per_repo_id
+ )
+
+ def _post_process(self, dataset: Dataset, resources_paths: Mapping[str, str]) -> Optional[Dataset]:
+ """Run dataset transforms or add indexes"""
+ return None
+
+ def _post_processing_resources(self, split: str) -> dict[str, str]:
+ """Mapping resource_name -> resource_file_name"""
+ return {}
+
+ def _download_post_processing_resources(
+ self, split: str, resource_name: str, dl_manager: DownloadManager
+ ) -> Optional[str]:
+ """Download the resource using the download manager and return the downloaded path."""
+ return None
+
+ @abc.abstractmethod
+ def _split_generators(self, dl_manager: Union[DownloadManager, StreamingDownloadManager]):
+ """Specify feature dictionary generators and dataset splits.
+
+ This function returns a list of `SplitGenerator`s defining how to generate
+ data and what splits to use.
+
+ Example:
+
+ return [
+ datasets.SplitGenerator(
+ name=datasets.Split.TRAIN,
+ gen_kwargs={'file': 'train_data.zip'},
+ ),
+ datasets.SplitGenerator(
+ name=datasets.Split.TEST,
+ gen_kwargs={'file': 'test_data.zip'},
+ ),
+ ]
+
+ The above code will first call `_generate_examples(file='train_data.zip')`
+ to write the train data, then `_generate_examples(file='test_data.zip')` to
+ write the test data.
+
+ Datasets are typically split into different subsets to be used at various
+ stages of training and evaluation.
+
+ Note that for datasets without a `VALIDATION` split, you can use a
+ fraction of the `TRAIN` data for evaluation as you iterate on your model
+ so as not to overfit to the `TEST` data.
+
+ For downloads and extractions, use the given `download_manager`.
+ Note that the `DownloadManager` caches downloads, so it is fine to have each
+ generator attempt to download the source data.
+
+ A good practice is to download all data in this function, and then
+ distribute the relevant parts to each split with the `gen_kwargs` argument
+
+ Args:
+ dl_manager (`Union[DownloadManager, StreamingDownloadManager]`):
+ Download manager to download the data
+
+ Returns:
+ `list`.
+ """
+ raise NotImplementedError()
+
+ @abc.abstractmethod
+ def _prepare_split(
+ self,
+ split_generator: SplitGenerator,
+ file_format: str = "arrow",
+ max_shard_size: Optional[Union[str, int]] = None,
+ num_proc: Optional[int] = None,
+ **kwargs,
+ ):
+ """Generate the examples and record them on disk.
+
+ Args:
+ split_generator (`SplitGenerator`):
+ Split generator to process
+ file_format (`str`, *optional*):
+ format of the data files in which the dataset will be written.
+ Supported formats: "arrow", "parquet". Default to "arrow" format.
+ max_shard_size (`Union[str, int]`, *optional*):
+ Maximum number of bytes written per shard, default is "500MB".
+ The size is based on uncompressed data size, so in practice your shard files may be smaller than
+ `max_shard_size` thanks to Parquet compression for example.
+ num_proc (`int`, *optional*, defaults to `None`):
+ Number of processes when downloading and generating the dataset locally.
+ Multiprocessing is disabled by default.
+
+
+ **kwargs: Additional kwargs forwarded from _download_and_prepare
+ """
+ raise NotImplementedError()
+
+ def _get_examples_iterable_for_split(self, split_generator: SplitGenerator) -> ExamplesIterable:
+ """Generate the examples on the fly.
+
+ Args:
+ split_generator (`SplitGenerator`):
+ Split generator to process
+ """
+ raise NotImplementedError()
+
+
+class GeneratorBasedBuilder(DatasetBuilder):
+ """Base class for datasets with data generation based on dict generators.
+
+ `GeneratorBasedBuilder` is a convenience class that abstracts away much
+ of the data writing and reading of `DatasetBuilder`. It expects subclasses to
+ implement generators of feature dictionaries across the dataset splits
+ (`_split_generators`). See the method docstrings for details.
+ """
+
+ @abc.abstractmethod
+ def _generate_examples(self, **kwargs):
+ """Default function generating examples for each `SplitGenerator`.
+
+ This function preprocess the examples from the raw data to the preprocessed
+ dataset files.
+ This function is called once for each `SplitGenerator` defined in
+ `_split_generators`. The examples yielded here will be written on
+ disk.
+
+ Args:
+ **kwargs (additional keyword arguments):
+ Arguments forwarded from the SplitGenerator.gen_kwargs
+
+ Yields:
+ key: `str` or `int`, a unique deterministic example identification key.
+ * Unique: An error will be raised if two examples are yield with the
+ same key.
+ * Deterministic: When generating the dataset twice, the same example
+ should have the same key.
+ Good keys can be the image id, or line number if examples are extracted
+ from a text file.
+ The key will be hashed and sorted to shuffle examples deterministically,
+ such as generating the dataset multiple times keep examples in the
+ same order.
+ example: `dict`, a feature dictionary
+ ready to be encoded and written to disk. The example will be
+ encoded with `self.info.features.encode_example({...})`.
+ """
+ raise NotImplementedError()
+
+ def _prepare_split(
+ self,
+ split_generator: SplitGenerator,
+ check_duplicate_keys: bool,
+ file_format="arrow",
+ num_proc: Optional[int] = None,
+ max_shard_size: Optional[Union[int, str]] = None,
+ ):
+ max_shard_size = convert_file_size_to_int(max_shard_size or config.MAX_SHARD_SIZE)
+
+ if self.info.splits is not None:
+ split_info = self.info.splits[split_generator.name]
+ else:
+ split_info = split_generator.split_info
+
+ SUFFIX = "-JJJJJ-SSSSS-of-NNNNN"
+ fname = f"{self.dataset_name}-{split_generator.name}{SUFFIX}.{file_format}"
+ fpath = posixpath.join(self._output_dir, fname)
+
+ if num_proc and num_proc > 1:
+ num_input_shards = _number_of_shards_in_gen_kwargs(split_generator.gen_kwargs)
+ if num_input_shards <= 1:
+ logger.warning(
+ f"Setting num_proc from {num_proc} back to 1 for the {split_info.name} split to disable multiprocessing as it only contains one shard."
+ )
+ num_proc = 1
+ elif num_input_shards < num_proc:
+ logger.warning(
+ f"Setting num_proc from {num_proc} to {num_input_shards} for the {split_info.name} split as it only contains {num_input_shards} shards."
+ )
+ num_proc = num_input_shards
+
+ pbar = hf_tqdm(
+ unit=" examples",
+ total=split_info.num_examples,
+ desc=f"Generating {split_info.name} split",
+ )
+
+ _prepare_split_args = {
+ "fpath": fpath,
+ "file_format": file_format,
+ "max_shard_size": max_shard_size,
+ "split_info": split_info,
+ "check_duplicate_keys": check_duplicate_keys,
+ }
+
+ if num_proc is None or num_proc == 1:
+ result = None
+ gen_kwargs = split_generator.gen_kwargs
+ job_id = 0
+ with pbar:
+ for job_id, done, content in self._prepare_split_single(
+ gen_kwargs=gen_kwargs, job_id=job_id, **_prepare_split_args
+ ):
+ if done:
+ result = content
+ else:
+ pbar.update(content)
+ # wrapping everything into lists for consistency with the multiprocessed code path
+ assert result is not None, "Failed to retrieve results from prepare_split"
+ examples_per_job, bytes_per_job, features_per_job, shards_per_job, shard_lengths_per_job = (
+ [item] for item in result
+ )
+ else:
+ kwargs_per_job = [
+ {"gen_kwargs": gen_kwargs, "job_id": job_id, **_prepare_split_args}
+ for job_id, gen_kwargs in enumerate(
+ _split_gen_kwargs(split_generator.gen_kwargs, max_num_jobs=num_proc)
+ )
+ ]
+ num_jobs = len(kwargs_per_job)
+
+ examples_per_job = [None] * num_jobs
+ bytes_per_job = [None] * num_jobs
+ features_per_job = [None] * num_jobs
+ shards_per_job = [None] * num_jobs
+ shard_lengths_per_job = [None] * num_jobs
+
+ with Pool(num_proc) as pool:
+ with pbar:
+ for job_id, done, content in iflatmap_unordered(
+ pool, self._prepare_split_single, kwargs_iterable=kwargs_per_job
+ ):
+ if done:
+ # the content is the result of the job
+ (
+ examples_per_job[job_id],
+ bytes_per_job[job_id],
+ features_per_job[job_id],
+ shards_per_job[job_id],
+ shard_lengths_per_job[job_id],
+ ) = content
+ else:
+ # the content is the number of examples progress update
+ pbar.update(content)
+
+ assert None not in examples_per_job, (
+ f"Failed to retrieve results from prepare_split: result list {examples_per_job} still contains None - at least one worker failed to return its results"
+ )
+
+ total_shards = sum(shards_per_job)
+ total_num_examples = sum(examples_per_job)
+ total_num_bytes = sum(bytes_per_job)
+ features = features_per_job[0]
+
+ split_generator.split_info.num_examples = total_num_examples
+ split_generator.split_info.num_bytes = total_num_bytes
+
+ # should rename everything at the end
+ logger.debug(f"Renaming {total_shards} shards.")
+ if total_shards > 1:
+ # use the -SSSSS-of-NNNNN pattern
+
+ def _rename_shard(shard_and_job: tuple[int]):
+ shard_id, job_id = shard_and_job
+ global_shard_id = sum(shards_per_job[:job_id]) + shard_id
+ self._rename(
+ fpath.replace("SSSSS", f"{shard_id:05d}").replace("JJJJJ", f"{job_id:05d}"),
+ fpath.replace("JJJJJ-SSSSS", f"{global_shard_id:05d}").replace("NNNNN", f"{total_shards:05d}"),
+ )
+
+ shards_and_jobs = [
+ (shard_id, job_id)
+ for job_id, num_shards in enumerate(shards_per_job)
+ for shard_id in range(num_shards)
+ ]
+ thread_map(_rename_shard, shards_and_jobs, disable=True, max_workers=64)
+
+ split_generator.split_info.shard_lengths = [
+ shard_length for shard_lengths in shard_lengths_per_job for shard_length in shard_lengths
+ ]
+ else:
+ # don't use any pattern
+ shard_id, job_id = 0, 0
+ self._rename(
+ fpath.replace("SSSSS", f"{shard_id:05d}").replace("JJJJJ", f"{job_id:05d}"),
+ fpath.replace(SUFFIX, ""),
+ )
+
+ if self.info.features is None:
+ self.info.features = features
+
+ def _prepare_split_single(
+ self,
+ gen_kwargs: dict,
+ fpath: str,
+ file_format: str,
+ max_shard_size: int,
+ split_info: SplitInfo,
+ check_duplicate_keys: bool,
+ job_id: int,
+ ) -> Iterable[tuple[int, bool, Union[int, tuple]]]:
+ generator = self._generate_examples(**gen_kwargs)
+ writer_class = ParquetWriter if file_format == "parquet" else ArrowWriter
+ embed_local_files = file_format == "parquet"
+ shard_lengths = []
+ total_num_examples, total_num_bytes = 0, 0
+
+ shard_id = 0
+ num_examples_progress_update = 0
+ try:
+ writer = writer_class(
+ features=self.info.features,
+ path=fpath.replace("SSSSS", f"{shard_id:05d}").replace("JJJJJ", f"{job_id:05d}"),
+ writer_batch_size=self._writer_batch_size,
+ hash_salt=split_info.name,
+ check_duplicates=check_duplicate_keys,
+ storage_options=self._fs.storage_options,
+ embed_local_files=embed_local_files,
+ )
+ try:
+ _time = time.time()
+ for key, record in generator:
+ if max_shard_size is not None and writer._num_bytes > max_shard_size:
+ num_examples, num_bytes = writer.finalize()
+ writer.close()
+ shard_lengths.append(num_examples)
+ total_num_examples += num_examples
+ total_num_bytes += num_bytes
+ shard_id += 1
+ writer = writer_class(
+ features=writer._features,
+ path=fpath.replace("SSSSS", f"{shard_id:05d}").replace("JJJJJ", f"{job_id:05d}"),
+ writer_batch_size=self._writer_batch_size,
+ hash_salt=split_info.name,
+ check_duplicates=check_duplicate_keys,
+ storage_options=self._fs.storage_options,
+ embed_local_files=embed_local_files,
+ )
+ example = self.info.features.encode_example(record) if self.info.features is not None else record
+ writer.write(example, key)
+ num_examples_progress_update += 1
+ if time.time() > _time + config.PBAR_REFRESH_TIME_INTERVAL:
+ _time = time.time()
+ yield job_id, False, num_examples_progress_update
+ num_examples_progress_update = 0
+ finally:
+ yield job_id, False, num_examples_progress_update
+ num_shards = shard_id + 1
+ num_examples, num_bytes = writer.finalize()
+ writer.close()
+ shard_lengths.append(num_examples)
+ total_num_examples += num_examples
+ total_num_bytes += num_bytes
+ except Exception as e:
+ # Ignore the writer's error for no examples written to the file if this error was caused by the error in _generate_examples before the first example was yielded
+ if isinstance(e, SchemaInferenceError) and e.__context__ is not None:
+ e = e.__context__
+ raise DatasetGenerationError("An error occurred while generating the dataset") from e
+
+ yield job_id, True, (total_num_examples, total_num_bytes, writer._features, num_shards, shard_lengths)
+
+ def _download_and_prepare(self, dl_manager, verification_mode, **prepare_splits_kwargs):
+ super()._download_and_prepare(
+ dl_manager,
+ verification_mode,
+ check_duplicate_keys=verification_mode == VerificationMode.BASIC_CHECKS
+ or verification_mode == VerificationMode.ALL_CHECKS,
+ **prepare_splits_kwargs,
+ )
+
+ def _get_examples_iterable_for_split(self, split_generator: SplitGenerator) -> ExamplesIterable:
+ return ExamplesIterable(self._generate_examples, split_generator.gen_kwargs)
+
+
+class ArrowBasedBuilder(DatasetBuilder):
+ """Base class for datasets with data generation based on Arrow loading functions (CSV/JSON/Parquet)."""
+
+ @abc.abstractmethod
+ def _generate_tables(self, **kwargs):
+ """Default function generating examples for each `SplitGenerator`.
+
+ This function preprocess the examples from the raw data to the preprocessed
+ dataset files.
+ This function is called once for each `SplitGenerator` defined in
+ `_split_generators`. The examples yielded here will be written on
+ disk.
+
+ Args:
+ **kwargs (additional keyword arguments):
+ Arguments forwarded from the SplitGenerator.gen_kwargs
+
+ Yields:
+ key: `str` or `int`, a unique deterministic example identification key.
+ * Unique: An error will be raised if two examples are yield with the
+ same key.
+ * Deterministic: When generating the dataset twice, the same example
+ should have the same key.
+ Good keys can be the image id, or line number if examples are extracted
+ from a text file.
+ The key will be hashed and sorted to shuffle examples deterministically,
+ such as generating the dataset multiple times keep examples in the
+ same order.
+ example: `pyarrow.Table`, a feature table
+ ready to be encoded and written to disk.
+ """
+ raise NotImplementedError()
+
+ def _prepare_split(
+ self,
+ split_generator: SplitGenerator,
+ file_format: str = "arrow",
+ num_proc: Optional[int] = None,
+ max_shard_size: Optional[Union[str, int]] = None,
+ ):
+ max_shard_size = convert_file_size_to_int(max_shard_size or config.MAX_SHARD_SIZE)
+
+ try:
+ split_info = self.info.splits[split_generator.name]
+ except Exception:
+ split_info = split_generator.split_info
+
+ SUFFIX = "-JJJJJ-SSSSS-of-NNNNN"
+ fname = f"{self.dataset_name}-{split_generator.name}{SUFFIX}.{file_format}"
+ fpath = posixpath.join(self._output_dir, fname)
+
+ if num_proc and num_proc > 1:
+ num_input_shards = _number_of_shards_in_gen_kwargs(split_generator.gen_kwargs)
+ if num_input_shards <= 1:
+ logger.warning(
+ f"Setting num_proc from {num_proc} back to 1 for the {split_info.name} split to disable multiprocessing as it only contains one shard."
+ )
+ num_proc = 1
+ elif num_input_shards < num_proc:
+ logger.warning(
+ f"Setting num_proc from {num_proc} to {num_input_shards} for the {split_info.name} split as it only contains {num_input_shards} shards."
+ )
+ num_proc = num_input_shards
+
+ pbar = hf_tqdm(
+ unit=" examples",
+ total=split_info.num_examples,
+ desc=f"Generating {split_info.name} split",
+ )
+
+ _prepare_split_args = {
+ "fpath": fpath,
+ "file_format": file_format,
+ "max_shard_size": max_shard_size,
+ }
+
+ if num_proc is None or num_proc == 1:
+ result = None
+ gen_kwargs = split_generator.gen_kwargs
+ job_id = 0
+ with pbar:
+ for job_id, done, content in self._prepare_split_single(
+ gen_kwargs=gen_kwargs, job_id=job_id, **_prepare_split_args
+ ):
+ if done:
+ result = content
+ else:
+ pbar.update(content)
+ # wrapping everything into lists for consistency with the multiprocessed code path
+ assert result is not None, "Failed to retrieve results from prepare_split"
+ examples_per_job, bytes_per_job, features_per_job, shards_per_job, shard_lengths_per_job = (
+ [item] for item in result
+ )
+ else:
+ kwargs_per_job = [
+ {"gen_kwargs": gen_kwargs, "job_id": job_id, **_prepare_split_args}
+ for job_id, gen_kwargs in enumerate(
+ _split_gen_kwargs(split_generator.gen_kwargs, max_num_jobs=num_proc)
+ )
+ ]
+ num_jobs = len(kwargs_per_job)
+
+ examples_per_job = [None] * num_jobs
+ bytes_per_job = [None] * num_jobs
+ features_per_job = [None] * num_jobs
+ shards_per_job = [None] * num_jobs
+ shard_lengths_per_job = [None] * num_jobs
+
+ with Pool(num_proc) as pool:
+ with pbar:
+ for job_id, done, content in iflatmap_unordered(
+ pool, self._prepare_split_single, kwargs_iterable=kwargs_per_job
+ ):
+ if done:
+ # the content is the result of the job
+ (
+ examples_per_job[job_id],
+ bytes_per_job[job_id],
+ features_per_job[job_id],
+ shards_per_job[job_id],
+ shard_lengths_per_job[job_id],
+ ) = content
+ else:
+ # the content is the number of examples progress update
+ pbar.update(content)
+
+ assert None not in examples_per_job, (
+ f"Failed to retrieve results from prepare_split: result list {examples_per_job} still contains None - at least one worker failed to return its results"
+ )
+
+ total_shards = sum(shards_per_job)
+ total_num_examples = sum(examples_per_job)
+ total_num_bytes = sum(bytes_per_job)
+ features = features_per_job[0]
+
+ split_generator.split_info.num_examples = total_num_examples
+ split_generator.split_info.num_bytes = total_num_bytes
+
+ # should rename everything at the end
+ logger.debug(f"Renaming {total_shards} shards.")
+ if total_shards > 1:
+ # use the -SSSSS-of-NNNNN pattern
+
+ def _rename_shard(shard_id_and_job: tuple[int]):
+ shard_id, job_id = shard_id_and_job
+ global_shard_id = sum(shards_per_job[:job_id]) + shard_id
+ self._rename(
+ fpath.replace("SSSSS", f"{shard_id:05d}").replace("JJJJJ", f"{job_id:05d}"),
+ fpath.replace("JJJJJ-SSSSS", f"{global_shard_id:05d}").replace("NNNNN", f"{total_shards:05d}"),
+ )
+
+ shard_ids_and_jobs = [
+ (shard_id, job_id)
+ for job_id, num_shards in enumerate(shards_per_job)
+ for shard_id in range(num_shards)
+ ]
+ thread_map(_rename_shard, shard_ids_and_jobs, disable=True, max_workers=64)
+
+ split_generator.split_info.shard_lengths = [
+ shard_length for shard_lengths in shard_lengths_per_job for shard_length in shard_lengths
+ ]
+ else:
+ # don't use any pattern
+ shard_id, job_id = 0, 0
+ self._rename(
+ fpath.replace("SSSSS", f"{shard_id:05d}").replace("JJJJJ", f"{job_id:05d}"),
+ fpath.replace(SUFFIX, ""),
+ )
+
+ if self.info.features is None:
+ self.info.features = features
+
+ def _prepare_split_single(
+ self, gen_kwargs: dict, fpath: str, file_format: str, max_shard_size: int, job_id: int
+ ) -> Iterable[tuple[int, bool, Union[int, tuple]]]:
+ gen_kwargs = {k: tracked_list(v) if isinstance(v, list) else v for k, v in gen_kwargs.items()}
+ generator = self._generate_tables(**gen_kwargs)
+ writer_class = ParquetWriter if file_format == "parquet" else ArrowWriter
+ embed_local_files = file_format == "parquet"
+ shard_lengths = []
+ total_num_examples, total_num_bytes = 0, 0
+
+ shard_id = 0
+ num_examples_progress_update = 0
+ try:
+ writer = writer_class(
+ features=self.info.features,
+ path=fpath.replace("SSSSS", f"{shard_id:05d}").replace("JJJJJ", f"{job_id:05d}"),
+ writer_batch_size=self._writer_batch_size,
+ storage_options=self._fs.storage_options,
+ embed_local_files=embed_local_files,
+ )
+ try:
+ _time = time.time()
+ for _, table in generator:
+ if max_shard_size is not None and writer._num_bytes > max_shard_size:
+ num_examples, num_bytes = writer.finalize()
+ writer.close()
+ shard_lengths.append(num_examples)
+ total_num_examples += num_examples
+ total_num_bytes += num_bytes
+ shard_id += 1
+ writer = writer_class(
+ features=writer._features,
+ path=fpath.replace("SSSSS", f"{shard_id:05d}").replace("JJJJJ", f"{job_id:05d}"),
+ writer_batch_size=self._writer_batch_size,
+ storage_options=self._fs.storage_options,
+ embed_local_files=embed_local_files,
+ )
+ try:
+ writer.write_table(table)
+ except CastError as cast_error:
+ raise DatasetGenerationCastError.from_cast_error(
+ cast_error=cast_error,
+ builder_name=self.info.builder_name,
+ gen_kwargs=gen_kwargs,
+ token=self.token,
+ )
+ num_examples_progress_update += len(table)
+ if time.time() > _time + config.PBAR_REFRESH_TIME_INTERVAL:
+ _time = time.time()
+ yield job_id, False, num_examples_progress_update
+ num_examples_progress_update = 0
+ finally:
+ yield job_id, False, num_examples_progress_update
+ num_shards = shard_id + 1
+ num_examples, num_bytes = writer.finalize()
+ writer.close()
+ shard_lengths.append(num_examples)
+ total_num_examples += num_examples
+ total_num_bytes += num_bytes
+ except Exception as e:
+ # Ignore the writer's error for no examples written to the file if this error was caused by the error in _generate_examples before the first example was yielded
+ if isinstance(e, SchemaInferenceError) and e.__context__ is not None:
+ e = e.__context__
+ if isinstance(e, DatasetGenerationError):
+ raise
+ raise DatasetGenerationError("An error occurred while generating the dataset") from e
+
+ yield job_id, True, (total_num_examples, total_num_bytes, writer._features, num_shards, shard_lengths)
+
+ def _get_examples_iterable_for_split(self, split_generator: SplitGenerator) -> ExamplesIterable:
+ return ArrowExamplesIterable(self._generate_tables, kwargs=split_generator.gen_kwargs)
diff --git a/venv/lib/python3.10/site-packages/datasets/combine.py b/venv/lib/python3.10/site-packages/datasets/combine.py
new file mode 100644
index 0000000000000000000000000000000000000000..91a6457c02cdb097554826f1319670e42bfc9ba0
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/combine.py
@@ -0,0 +1,223 @@
+from typing import Optional, TypeVar
+
+from .arrow_dataset import Dataset, _concatenate_map_style_datasets, _interleave_map_style_datasets
+from .dataset_dict import DatasetDict, IterableDatasetDict
+from .info import DatasetInfo
+from .iterable_dataset import IterableDataset, _concatenate_iterable_datasets, _interleave_iterable_datasets
+from .splits import NamedSplit
+from .utils import logging
+from .utils.py_utils import Literal
+
+
+logger = logging.get_logger(__name__)
+
+
+DatasetType = TypeVar("DatasetType", Dataset, IterableDataset)
+
+
+def interleave_datasets(
+ datasets: list[DatasetType],
+ probabilities: Optional[list[float]] = None,
+ seed: Optional[int] = None,
+ info: Optional[DatasetInfo] = None,
+ split: Optional[NamedSplit] = None,
+ stopping_strategy: Literal[
+ "first_exhausted", "all_exhausted", "all_exhausted_without_replacement"
+ ] = "first_exhausted",
+) -> DatasetType:
+ """
+ Interleave several datasets (sources) into a single dataset.
+ The new dataset is constructed by alternating between the sources to get the examples.
+
+ You can use this function on a list of [`Dataset`] objects, or on a list of [`IterableDataset`] objects.
+
+ - If `probabilities` is `None` (default) the new dataset is constructed by cycling between each source to get the examples.
+ - If `probabilities` is not `None`, the new dataset is constructed by getting examples from a random source at a time according to the provided probabilities.
+
+ The resulting dataset ends when one of the source datasets runs out of examples except when `oversampling` is `True`,
+ in which case, the resulting dataset ends when all datasets have ran out of examples at least one time.
+
+ Note for iterable datasets:
+
+ In a distributed setup or in PyTorch DataLoader workers, the stopping strategy is applied per process.
+ Therefore the "first_exhausted" strategy on an sharded iterable dataset can generate less samples in total (up to 1 missing sample per subdataset per worker).
+
+ Args:
+ datasets (`List[Dataset]` or `List[IterableDataset]`):
+ List of datasets to interleave.
+ probabilities (`List[float]`, *optional*, defaults to `None`):
+ If specified, the new dataset is constructed by sampling
+ examples from one source at a time according to these probabilities.
+ seed (`int`, *optional*, defaults to `None`):
+ The random seed used to choose a source for each example.
+ info ([`DatasetInfo`], *optional*):
+ Dataset information, like description, citation, etc.
+
+ split ([`NamedSplit`], *optional*):
+ Name of the dataset split.
+
+ stopping_strategy (`str`, defaults to `first_exhausted`):
+ Three strategies are proposed right now, `first_exhausted`, `all_exhausted` and `all_exhausted_without_replacement`.
+ By default, `first_exhausted` is an undersampling strategy, i.e the dataset construction is stopped as soon as one dataset has ran out of samples.
+ If the strategy is `all_exhausted`, we use an oversampling strategy, i.e the dataset construction is stopped as soon as every samples of every dataset has been added at least once.
+ When strategy is `all_exhausted_without_replacement` we make sure that each sample in each dataset is sampled only once.
+ Note that if the strategy is `all_exhausted`, the interleaved dataset size can get enormous:
+ - with no probabilities, the resulting dataset will have `max_length_datasets*nb_dataset` samples.
+ - with given probabilities, the resulting dataset will have more samples if some datasets have really low probability of visiting.
+ Returns:
+ [`Dataset`] or [`IterableDataset`]: Return type depends on the input `datasets`
+ parameter. `Dataset` if the input is a list of `Dataset`, `IterableDataset` if the input is a list of
+ `IterableDataset`.
+
+ Example:
+
+ For regular datasets (map-style):
+
+ ```python
+ >>> from datasets import Dataset, interleave_datasets
+ >>> d1 = Dataset.from_dict({"a": [0, 1, 2]})
+ >>> d2 = Dataset.from_dict({"a": [10, 11, 12]})
+ >>> d3 = Dataset.from_dict({"a": [20, 21, 22]})
+ >>> dataset = interleave_datasets([d1, d2, d3], probabilities=[0.7, 0.2, 0.1], seed=42, stopping_strategy="all_exhausted")
+ >>> dataset["a"]
+ [10, 0, 11, 1, 2, 20, 12, 10, 0, 1, 2, 21, 0, 11, 1, 2, 0, 1, 12, 2, 10, 0, 22]
+ >>> dataset = interleave_datasets([d1, d2, d3], probabilities=[0.7, 0.2, 0.1], seed=42)
+ >>> dataset["a"]
+ [10, 0, 11, 1, 2]
+ >>> dataset = interleave_datasets([d1, d2, d3])
+ >>> dataset["a"]
+ [0, 10, 20, 1, 11, 21, 2, 12, 22]
+ >>> dataset = interleave_datasets([d1, d2, d3], stopping_strategy="all_exhausted")
+ >>> dataset["a"]
+ [0, 10, 20, 1, 11, 21, 2, 12, 22]
+ >>> d1 = Dataset.from_dict({"a": [0, 1, 2]})
+ >>> d2 = Dataset.from_dict({"a": [10, 11, 12, 13]})
+ >>> d3 = Dataset.from_dict({"a": [20, 21, 22, 23, 24]})
+ >>> dataset = interleave_datasets([d1, d2, d3])
+ >>> dataset["a"]
+ [0, 10, 20, 1, 11, 21, 2, 12, 22]
+ >>> dataset = interleave_datasets([d1, d2, d3], stopping_strategy="all_exhausted")
+ >>> dataset["a"]
+ [0, 10, 20, 1, 11, 21, 2, 12, 22, 0, 13, 23, 1, 10, 24]
+ >>> dataset = interleave_datasets([d1, d2, d3], probabilities=[0.7, 0.2, 0.1], seed=42)
+ >>> dataset["a"]
+ [10, 0, 11, 1, 2]
+ >>> dataset = interleave_datasets([d1, d2, d3], probabilities=[0.7, 0.2, 0.1], seed=42, stopping_strategy="all_exhausted")
+ >>> dataset["a"]
+ [10, 0, 11, 1, 2, 20, 12, 13, ..., 0, 1, 2, 0, 24]
+ For datasets in streaming mode (iterable):
+
+ >>> from datasets import interleave_datasets
+ >>> d1 = load_dataset('allenai/c4', 'es', split='train', streaming=True)
+ >>> d2 = load_dataset('allenai/c4', 'fr', split='train', streaming=True)
+ >>> dataset = interleave_datasets([d1, d2])
+ >>> iterator = iter(dataset)
+ >>> next(iterator)
+ {'text': 'Comprar Zapatillas para niña en chancla con goma por...'}
+ >>> next(iterator)
+ {'text': 'Le sacre de philippe ier, 23 mai 1059 - Compte Rendu...'
+ ```
+ """
+ from .arrow_dataset import Dataset
+ from .iterable_dataset import IterableDataset
+
+ if not datasets:
+ raise ValueError("Unable to interleave an empty list of datasets.")
+ for i, dataset in enumerate(datasets):
+ if not isinstance(dataset, (Dataset, IterableDataset)):
+ if isinstance(dataset, (DatasetDict, IterableDatasetDict)):
+ if not dataset:
+ raise ValueError(
+ f"Expected a list of Dataset objects or a list of IterableDataset objects, but element at position {i} "
+ "is an empty dataset dictionary."
+ )
+ raise ValueError(
+ f"Dataset at position {i} has at least one split: {list(dataset)}\n"
+ f"Please pick one to interleave with the other datasets, for example: dataset['{next(iter(dataset))}']"
+ )
+ raise ValueError(
+ f"Expected a list of Dataset objects or a list of IterableDataset objects, but element at position {i} is a {type(dataset).__name__}."
+ )
+ if i == 0:
+ dataset_type, other_type = (
+ (Dataset, IterableDataset) if isinstance(dataset, Dataset) else (IterableDataset, Dataset)
+ )
+ elif not isinstance(dataset, dataset_type):
+ raise ValueError(
+ f"Unable to interleave a {dataset_type.__name__} (at position 0) with a {other_type.__name__} (at position {i}). Expected a list of Dataset objects or a list of IterableDataset objects."
+ )
+ if stopping_strategy not in ["first_exhausted", "all_exhausted", "all_exhausted_without_replacement"]:
+ raise ValueError(f"{stopping_strategy} is not supported. Please enter a valid stopping_strategy.")
+ if dataset_type is Dataset:
+ return _interleave_map_style_datasets(
+ datasets, probabilities, seed, info=info, split=split, stopping_strategy=stopping_strategy
+ )
+ else:
+ return _interleave_iterable_datasets(
+ datasets,
+ probabilities,
+ seed,
+ info=info,
+ split=split,
+ stopping_strategy=stopping_strategy,
+ )
+
+
+def concatenate_datasets(
+ dsets: list[DatasetType],
+ info: Optional[DatasetInfo] = None,
+ split: Optional[NamedSplit] = None,
+ axis: int = 0,
+) -> DatasetType:
+ """
+ Converts a list of [`Dataset`] with the same schema into a single [`Dataset`].
+
+ Args:
+ dsets (`List[datasets.Dataset]`):
+ List of Datasets to concatenate.
+ info (`DatasetInfo`, *optional*):
+ Dataset information, like description, citation, etc.
+ split (`NamedSplit`, *optional*):
+ Name of the dataset split.
+ axis (`{0, 1}`, defaults to `0`):
+ Axis to concatenate over, where `0` means over rows (vertically) and `1` means over columns
+ (horizontally).
+
+
+
+ Example:
+
+ ```py
+ >>> ds3 = concatenate_datasets([ds1, ds2])
+ ```
+ """
+
+ if not dsets:
+ raise ValueError("Unable to concatenate an empty list of datasets.")
+ for i, dataset in enumerate(dsets):
+ if not isinstance(dataset, (Dataset, IterableDataset)):
+ if isinstance(dataset, (DatasetDict, IterableDatasetDict)):
+ if not dataset:
+ raise ValueError(
+ f"Expected a list of Dataset objects or a list of IterableDataset objects, but element at position {i} "
+ "is an empty dataset dictionary."
+ )
+ raise ValueError(
+ f"Dataset at position {i} has at least one split: {list(dataset)}\n"
+ f"Please pick one to interleave with the other datasets, for example: dataset['{next(iter(dataset))}']"
+ )
+ raise ValueError(
+ f"Expected a list of Dataset objects or a list of IterableDataset objects, but element at position {i} is a {type(dataset).__name__}."
+ )
+ if i == 0:
+ dataset_type, other_type = (
+ (Dataset, IterableDataset) if isinstance(dataset, Dataset) else (IterableDataset, Dataset)
+ )
+ elif not isinstance(dataset, dataset_type):
+ raise ValueError(
+ f"Unable to interleave a {dataset_type.__name__} (at position 0) with a {other_type.__name__} (at position {i}). Expected a list of Dataset objects or a list of IterableDataset objects."
+ )
+ if dataset_type is Dataset:
+ return _concatenate_map_style_datasets(dsets, info=info, split=split, axis=axis)
+ else:
+ return _concatenate_iterable_datasets(dsets, info=info, split=split, axis=axis)
diff --git a/venv/lib/python3.10/site-packages/datasets/commands/__init__.py b/venv/lib/python3.10/site-packages/datasets/commands/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..905e753955a348a8e486302e1b6f5e8f53ec7bf4
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/commands/__init__.py
@@ -0,0 +1,13 @@
+from abc import ABC, abstractmethod
+from argparse import ArgumentParser
+
+
+class BaseDatasetsCLICommand(ABC):
+ @staticmethod
+ @abstractmethod
+ def register_subcommand(parser: ArgumentParser):
+ raise NotImplementedError()
+
+ @abstractmethod
+ def run(self):
+ raise NotImplementedError()
diff --git a/venv/lib/python3.10/site-packages/datasets/commands/__pycache__/__init__.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/commands/__pycache__/__init__.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..eeabc9448c1efd1bf2272e8e5df5640e12d149cd
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/commands/__pycache__/__init__.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/commands/__pycache__/datasets_cli.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/commands/__pycache__/datasets_cli.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..c2d1b9a174480d65719d60801b676b4799130626
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/commands/__pycache__/datasets_cli.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/commands/__pycache__/delete_from_hub.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/commands/__pycache__/delete_from_hub.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..f52470acebdb773686f17e89242eedbd27279ebf
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/commands/__pycache__/delete_from_hub.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/commands/__pycache__/env.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/commands/__pycache__/env.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..088483aa52f39b8fc32ccd9504dad72a6712a703
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/commands/__pycache__/env.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/commands/__pycache__/test.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/commands/__pycache__/test.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..e3a9618467ca7ed21a25ce23b2dfd1017738ce17
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/commands/__pycache__/test.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/commands/datasets_cli.py b/venv/lib/python3.10/site-packages/datasets/commands/datasets_cli.py
new file mode 100644
index 0000000000000000000000000000000000000000..1d04d87ede1bfddc0886c0ae1686fa02ebd21768
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/commands/datasets_cli.py
@@ -0,0 +1,39 @@
+#!/usr/bin/env python
+from argparse import ArgumentParser
+
+from datasets.commands.delete_from_hub import DeleteFromHubCommand
+from datasets.commands.env import EnvironmentCommand
+from datasets.commands.test import TestCommand
+from datasets.utils.logging import set_verbosity_info
+
+
+def parse_unknown_args(unknown_args):
+ return {key.lstrip("-"): value for key, value in zip(unknown_args[::2], unknown_args[1::2])}
+
+
+def main():
+ parser = ArgumentParser(
+ "HuggingFace Datasets CLI tool", usage="datasets-cli []", allow_abbrev=False
+ )
+ commands_parser = parser.add_subparsers(help="datasets-cli command helpers")
+ set_verbosity_info()
+
+ # Register commands
+ EnvironmentCommand.register_subcommand(commands_parser)
+ TestCommand.register_subcommand(commands_parser)
+ DeleteFromHubCommand.register_subcommand(commands_parser)
+
+ # Parse args
+ args, unknown_args = parser.parse_known_args()
+ if not hasattr(args, "func"):
+ parser.print_help()
+ exit(1)
+ kwargs = parse_unknown_args(unknown_args)
+
+ # Run
+ service = args.func(args, **kwargs)
+ service.run()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/venv/lib/python3.10/site-packages/datasets/commands/delete_from_hub.py b/venv/lib/python3.10/site-packages/datasets/commands/delete_from_hub.py
new file mode 100644
index 0000000000000000000000000000000000000000..126a029c246ab584df47c4ba1b908a798b9003b6
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/commands/delete_from_hub.py
@@ -0,0 +1,42 @@
+from argparse import ArgumentParser
+from typing import Optional
+
+from datasets.commands import BaseDatasetsCLICommand
+from datasets.hub import delete_from_hub
+
+
+def _command_factory(args):
+ return DeleteFromHubCommand(
+ args.dataset_id,
+ args.config_name,
+ args.token,
+ args.revision,
+ )
+
+
+class DeleteFromHubCommand(BaseDatasetsCLICommand):
+ @staticmethod
+ def register_subcommand(parser):
+ parser: ArgumentParser = parser.add_parser("delete_from_hub", help="Delete dataset config from the Hub")
+ parser.add_argument(
+ "dataset_id", help="source dataset ID, e.g. USERNAME/DATASET_NAME or ORGANIZATION/DATASET_NAME"
+ )
+ parser.add_argument("config_name", help="config name to delete")
+ parser.add_argument("--token", help="access token to the Hugging Face Hub")
+ parser.add_argument("--revision", help="source revision")
+ parser.set_defaults(func=_command_factory)
+
+ def __init__(
+ self,
+ dataset_id: str,
+ config_name: str,
+ token: Optional[str],
+ revision: Optional[str],
+ ):
+ self._dataset_id = dataset_id
+ self._config_name = config_name
+ self._token = token
+ self._revision = revision
+
+ def run(self) -> None:
+ _ = delete_from_hub(self._dataset_id, self._config_name, revision=self._revision, token=self._token)
diff --git a/venv/lib/python3.10/site-packages/datasets/commands/env.py b/venv/lib/python3.10/site-packages/datasets/commands/env.py
new file mode 100644
index 0000000000000000000000000000000000000000..40b2a3654c8f1c6c080222a80b12d108972a5dc9
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/commands/env.py
@@ -0,0 +1,41 @@
+import platform
+from argparse import ArgumentParser
+
+import fsspec
+import huggingface_hub
+import pandas
+import pyarrow
+
+from datasets import __version__ as version
+from datasets.commands import BaseDatasetsCLICommand
+
+
+def info_command_factory(_):
+ return EnvironmentCommand()
+
+
+class EnvironmentCommand(BaseDatasetsCLICommand):
+ @staticmethod
+ def register_subcommand(parser: ArgumentParser):
+ download_parser = parser.add_parser("env", help="Print relevant system environment info.")
+ download_parser.set_defaults(func=info_command_factory)
+
+ def run(self):
+ info = {
+ "`datasets` version": version,
+ "Platform": platform.platform(),
+ "Python version": platform.python_version(),
+ "`huggingface_hub` version": huggingface_hub.__version__,
+ "PyArrow version": pyarrow.__version__,
+ "Pandas version": pandas.__version__,
+ "`fsspec` version": fsspec.__version__,
+ }
+
+ print("\nCopy-and-paste the text below in your GitHub issue.\n")
+ print(self.format_dict(info))
+
+ return info
+
+ @staticmethod
+ def format_dict(d):
+ return "\n".join([f"- {prop}: {val}" for prop, val in d.items()]) + "\n"
diff --git a/venv/lib/python3.10/site-packages/datasets/commands/test.py b/venv/lib/python3.10/site-packages/datasets/commands/test.py
new file mode 100644
index 0000000000000000000000000000000000000000..e54169c300c189a681011fa88e7b30b1fbfdf227
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/commands/test.py
@@ -0,0 +1,180 @@
+import logging
+import os
+from argparse import ArgumentParser
+from collections.abc import Generator
+from shutil import rmtree
+
+import datasets.config
+from datasets.builder import DatasetBuilder
+from datasets.commands import BaseDatasetsCLICommand
+from datasets.download.download_manager import DownloadMode
+from datasets.info import DatasetInfosDict
+from datasets.load import dataset_module_factory, get_dataset_builder_class
+from datasets.utils.info_utils import VerificationMode
+from datasets.utils.logging import ERROR, get_logger
+
+
+logger = get_logger(__name__)
+
+
+def _test_command_factory(args):
+ return TestCommand(
+ args.dataset,
+ args.name,
+ args.cache_dir,
+ args.data_dir,
+ args.all_configs,
+ args.save_info or args.save_infos,
+ args.ignore_verifications,
+ args.force_redownload,
+ args.clear_cache,
+ args.num_proc,
+ )
+
+
+class TestCommand(BaseDatasetsCLICommand):
+ __test__ = False # to tell pytest it's not a test class
+
+ @staticmethod
+ def register_subcommand(parser: ArgumentParser):
+ test_parser = parser.add_parser("test", help="Test dataset loading.")
+ test_parser.add_argument("--name", type=str, default=None, help="Dataset processing name")
+ test_parser.add_argument(
+ "--cache_dir",
+ type=str,
+ default=None,
+ help="Cache directory where the datasets are stored.",
+ )
+ test_parser.add_argument(
+ "--data_dir",
+ type=str,
+ default=None,
+ help="Can be used to specify a manual directory to get the files from.",
+ )
+ test_parser.add_argument("--all_configs", action="store_true", help="Test all dataset configurations")
+ test_parser.add_argument(
+ "--save_info", action="store_true", help="Save the dataset infos in the dataset card (README.md)"
+ )
+ test_parser.add_argument(
+ "--ignore_verifications",
+ action="store_true",
+ help="Run the test without checksums and splits checks.",
+ )
+ test_parser.add_argument("--force_redownload", action="store_true", help="Force dataset redownload")
+ test_parser.add_argument(
+ "--clear_cache",
+ action="store_true",
+ help="Remove downloaded files and cached datasets after each config test",
+ )
+ test_parser.add_argument("--num_proc", type=int, default=None, help="Number of processes")
+ # aliases
+ test_parser.add_argument("--save_infos", action="store_true", help="alias to save_info")
+ test_parser.add_argument("dataset", type=str, help="Name of the dataset to download")
+ test_parser.set_defaults(func=_test_command_factory)
+
+ def __init__(
+ self,
+ dataset: str,
+ name: str,
+ cache_dir: str,
+ data_dir: str,
+ all_configs: bool,
+ save_infos: bool,
+ ignore_verifications: bool,
+ force_redownload: bool,
+ clear_cache: bool,
+ num_proc: int,
+ ):
+ self._dataset = dataset
+ self._name = name
+ self._cache_dir = cache_dir
+ self._data_dir = data_dir
+ self._all_configs = all_configs
+ self._save_infos = save_infos
+ self._ignore_verifications = ignore_verifications
+ self._force_redownload = force_redownload
+ self._clear_cache = clear_cache
+ self._num_proc = num_proc
+ if clear_cache and not cache_dir:
+ print(
+ "When --clear_cache is used, specifying a cache directory is mandatory.\n"
+ "The 'download' folder of the cache directory and the dataset builder cache will be deleted after each configuration test.\n"
+ "Please provide a --cache_dir that will be used to test the dataset."
+ )
+ exit(1)
+ if save_infos:
+ self._ignore_verifications = True
+
+ def run(self):
+ logging.getLogger("filelock").setLevel(ERROR)
+ if self._name is not None and self._all_configs:
+ print("Both parameters `config` and `all_configs` can't be used at once.")
+ exit(1)
+ path, config_name = self._dataset, self._name
+ module = dataset_module_factory(path)
+ builder_cls = get_dataset_builder_class(module)
+ n_builders = len(builder_cls.BUILDER_CONFIGS) if self._all_configs and builder_cls.BUILDER_CONFIGS else 1
+
+ def get_builders() -> Generator[DatasetBuilder, None, None]:
+ if self._all_configs and builder_cls.BUILDER_CONFIGS:
+ for i, config in enumerate(builder_cls.BUILDER_CONFIGS):
+ if "config_name" in module.builder_kwargs:
+ yield builder_cls(
+ cache_dir=self._cache_dir,
+ data_dir=self._data_dir,
+ **module.builder_kwargs,
+ )
+ else:
+ yield builder_cls(
+ config_name=config.name,
+ cache_dir=self._cache_dir,
+ data_dir=self._data_dir,
+ **module.builder_kwargs,
+ )
+ else:
+ if "config_name" in module.builder_kwargs:
+ yield builder_cls(cache_dir=self._cache_dir, data_dir=self._data_dir, **module.builder_kwargs)
+ else:
+ yield builder_cls(
+ config_name=config_name,
+ cache_dir=self._cache_dir,
+ data_dir=self._data_dir,
+ **module.builder_kwargs,
+ )
+
+ for j, builder in enumerate(get_builders()):
+ print(f"Testing builder '{builder.config.name}' ({j + 1}/{n_builders})")
+ builder._record_infos = os.path.exists(
+ os.path.join(builder.get_imported_module_dir(), datasets.config.DATASETDICT_INFOS_FILENAME)
+ ) # record checksums only if we need to update a (deprecated) dataset_infos.json
+ builder.download_and_prepare(
+ download_mode=DownloadMode.REUSE_CACHE_IF_EXISTS
+ if not self._force_redownload
+ else DownloadMode.FORCE_REDOWNLOAD,
+ verification_mode=VerificationMode.NO_CHECKS
+ if self._ignore_verifications
+ else VerificationMode.ALL_CHECKS,
+ num_proc=self._num_proc,
+ )
+ builder.as_dataset()
+
+ # If save_infos=True, we create the dataset card (README.md)
+ # The dataset_infos are saved in the YAML part of the README.md
+ # This is to allow the user to upload them on HF afterwards.
+ if self._save_infos:
+ save_infos_dir = os.path.basename(path) if not os.path.isdir(path) else path
+ os.makedirs(save_infos_dir, exist_ok=True)
+ DatasetInfosDict(**{builder.config.name: builder.info}).write_to_directory(save_infos_dir)
+ print(f"Dataset card saved at {os.path.join(save_infos_dir, datasets.config.REPOCARD_FILENAME)}")
+
+ # If clear_cache=True, the download folder and the dataset builder cache directory are deleted
+ if self._clear_cache:
+ if os.path.isdir(builder._cache_dir):
+ logger.warning(f"Clearing cache at {builder._cache_dir}")
+ rmtree(builder._cache_dir)
+ download_dir = os.path.join(self._cache_dir, datasets.config.DOWNLOADED_DATASETS_DIR)
+ if os.path.isdir(download_dir):
+ logger.warning(f"Clearing cache at {download_dir}")
+ rmtree(download_dir)
+
+ print("Test successful.")
diff --git a/venv/lib/python3.10/site-packages/datasets/config.py b/venv/lib/python3.10/site-packages/datasets/config.py
new file mode 100644
index 0000000000000000000000000000000000000000..b64126827277d30295a23a580f3cf552d02ff6cd
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/config.py
@@ -0,0 +1,271 @@
+import importlib
+import importlib.metadata
+import logging
+import os
+import platform
+from pathlib import Path
+from typing import Optional
+
+from huggingface_hub import constants
+from packaging import version
+
+
+logger = logging.getLogger(__name__.split(".", 1)[0]) # to avoid circular import from .utils.logging
+
+# Datasets
+S3_DATASETS_BUCKET_PREFIX = "https://s3.amazonaws.com/datasets.huggingface.co/datasets/datasets"
+CLOUDFRONT_DATASETS_DISTRIB_PREFIX = "https://cdn-datasets.huggingface.co/datasets/datasets"
+REPO_DATASETS_URL = "https://raw.githubusercontent.com/huggingface/datasets/{revision}/datasets/{path}/{name}"
+
+# Hub
+HF_ENDPOINT = os.environ.get("HF_ENDPOINT", "https://huggingface.co")
+HUB_DATASETS_URL = HF_ENDPOINT + "/datasets/{repo_id}/resolve/{revision}/{path}"
+HUB_DATASETS_HFFS_URL = "hf://datasets/{repo_id}@{revision}/{path}"
+HUB_DEFAULT_VERSION = "main"
+
+PY_VERSION = version.parse(platform.python_version())
+
+# General environment variables accepted values for booleans
+ENV_VARS_TRUE_VALUES = {"1", "ON", "YES", "TRUE"}
+ENV_VARS_FALSE_VALUES = {"0", "OFF", "NO", "FALSE"}
+ENV_VARS_TRUE_AND_AUTO_VALUES = ENV_VARS_TRUE_VALUES.union({"AUTO"})
+ENV_VARS_FALSE_AND_AUTO_VALUES = ENV_VARS_FALSE_VALUES.union({"AUTO"})
+
+
+# Imports
+DILL_VERSION = version.parse(importlib.metadata.version("dill"))
+FSSPEC_VERSION = version.parse(importlib.metadata.version("fsspec"))
+PANDAS_VERSION = version.parse(importlib.metadata.version("pandas"))
+PYARROW_VERSION = version.parse(importlib.metadata.version("pyarrow"))
+HF_HUB_VERSION = version.parse(importlib.metadata.version("huggingface_hub"))
+
+USE_TF = os.environ.get("USE_TF", "AUTO").upper()
+USE_TORCH = os.environ.get("USE_TORCH", "AUTO").upper()
+USE_JAX = os.environ.get("USE_JAX", "AUTO").upper()
+
+TORCH_VERSION = "N/A"
+TORCH_AVAILABLE = False
+
+if USE_TORCH in ENV_VARS_TRUE_AND_AUTO_VALUES and USE_TF not in ENV_VARS_TRUE_VALUES:
+ TORCH_AVAILABLE = importlib.util.find_spec("torch") is not None
+ if TORCH_AVAILABLE:
+ try:
+ TORCH_VERSION = version.parse(importlib.metadata.version("torch"))
+ logger.debug(f"PyTorch version {TORCH_VERSION} available.")
+ except importlib.metadata.PackageNotFoundError:
+ pass
+else:
+ logger.info("Disabling PyTorch because USE_TF is set")
+
+POLARS_VERSION = "N/A"
+POLARS_AVAILABLE = importlib.util.find_spec("polars") is not None
+
+if POLARS_AVAILABLE:
+ try:
+ POLARS_VERSION = version.parse(importlib.metadata.version("polars"))
+ logger.debug(f"Polars version {POLARS_VERSION} available.")
+ except importlib.metadata.PackageNotFoundError:
+ pass
+
+
+DUCKDB_VERSION = "N/A"
+DUCKDB_AVAILABLE = importlib.util.find_spec("duckdb") is not None
+
+if DUCKDB_AVAILABLE:
+ try:
+ DUCKDB_VERSION = version.parse(importlib.metadata.version("duckdb"))
+ logger.debug(f"Duckdb version {DUCKDB_VERSION} available.")
+ except importlib.metadata.PackageNotFoundError:
+ pass
+
+TF_VERSION = "N/A"
+TF_AVAILABLE = False
+
+if USE_TF in ENV_VARS_TRUE_AND_AUTO_VALUES and USE_TORCH not in ENV_VARS_TRUE_VALUES:
+ TF_AVAILABLE = importlib.util.find_spec("tensorflow") is not None
+ if TF_AVAILABLE:
+ # For the metadata, we have to look for both tensorflow and tensorflow-cpu
+ for package in [
+ "tensorflow",
+ "tensorflow-cpu",
+ "tensorflow-gpu",
+ "tf-nightly",
+ "tf-nightly-cpu",
+ "tf-nightly-gpu",
+ "intel-tensorflow",
+ "tensorflow-rocm",
+ "tensorflow-macos",
+ ]:
+ try:
+ TF_VERSION = version.parse(importlib.metadata.version(package))
+ except importlib.metadata.PackageNotFoundError:
+ continue
+ else:
+ break
+ else:
+ TF_AVAILABLE = False
+ if TF_AVAILABLE:
+ if TF_VERSION.major < 2:
+ logger.info(f"TensorFlow found but with version {TF_VERSION}. `datasets` requires version 2 minimum.")
+ TF_AVAILABLE = False
+ else:
+ logger.info(f"TensorFlow version {TF_VERSION} available.")
+else:
+ logger.info("Disabling Tensorflow because USE_TORCH is set")
+
+
+JAX_VERSION = "N/A"
+JAX_AVAILABLE = False
+
+if USE_JAX in ENV_VARS_TRUE_AND_AUTO_VALUES:
+ JAX_AVAILABLE = importlib.util.find_spec("jax") is not None and importlib.util.find_spec("jaxlib") is not None
+ if JAX_AVAILABLE:
+ try:
+ JAX_VERSION = version.parse(importlib.metadata.version("jax"))
+ logger.info(f"JAX version {JAX_VERSION} available.")
+ except importlib.metadata.PackageNotFoundError:
+ pass
+else:
+ logger.info("Disabling JAX because USE_JAX is set to False")
+
+
+# Optional tools for data loading
+SQLALCHEMY_AVAILABLE = importlib.util.find_spec("sqlalchemy") is not None
+
+# Optional tools for feature decoding
+PIL_AVAILABLE = importlib.util.find_spec("PIL") is not None
+IS_OPUS_SUPPORTED = True
+IS_MP3_SUPPORTED = True
+TORCHCODEC_AVAILABLE = importlib.util.find_spec("torchcodec") is not None
+TORCHVISION_AVAILABLE = importlib.util.find_spec("torchvision") is not None
+PDFPLUMBER_AVAILABLE = importlib.util.find_spec("pdfplumber") is not None
+NIBABEL_AVAILABLE = importlib.util.find_spec("nibabel") is not None
+
+# Optional compression tools
+RARFILE_AVAILABLE = importlib.util.find_spec("rarfile") is not None
+ZSTANDARD_AVAILABLE = importlib.util.find_spec("zstandard") is not None
+LZ4_AVAILABLE = importlib.util.find_spec("lz4") is not None
+PY7ZR_AVAILABLE = importlib.util.find_spec("py7zr") is not None
+
+# Cache location
+DEFAULT_XDG_CACHE_HOME = "~/.cache"
+XDG_CACHE_HOME = os.getenv("XDG_CACHE_HOME", DEFAULT_XDG_CACHE_HOME)
+DEFAULT_HF_CACHE_HOME = os.path.join(XDG_CACHE_HOME, "huggingface")
+HF_CACHE_HOME = os.path.expanduser(os.getenv("HF_HOME", DEFAULT_HF_CACHE_HOME))
+
+DEFAULT_HF_DATASETS_CACHE = os.path.join(HF_CACHE_HOME, "datasets")
+HF_DATASETS_CACHE = Path(os.getenv("HF_DATASETS_CACHE", DEFAULT_HF_DATASETS_CACHE))
+
+DEFAULT_HF_MODULES_CACHE = os.path.join(HF_CACHE_HOME, "modules")
+HF_MODULES_CACHE = Path(os.getenv("HF_MODULES_CACHE", DEFAULT_HF_MODULES_CACHE))
+
+DOWNLOADED_DATASETS_DIR = "downloads"
+DEFAULT_DOWNLOADED_DATASETS_PATH = os.path.join(HF_DATASETS_CACHE, DOWNLOADED_DATASETS_DIR)
+DOWNLOADED_DATASETS_PATH = Path(os.getenv("HF_DATASETS_DOWNLOADED_DATASETS_PATH", DEFAULT_DOWNLOADED_DATASETS_PATH))
+
+EXTRACTED_DATASETS_DIR = "extracted"
+DEFAULT_EXTRACTED_DATASETS_PATH = os.path.join(DEFAULT_DOWNLOADED_DATASETS_PATH, EXTRACTED_DATASETS_DIR)
+EXTRACTED_DATASETS_PATH = Path(os.getenv("HF_DATASETS_EXTRACTED_DATASETS_PATH", DEFAULT_EXTRACTED_DATASETS_PATH))
+
+# Download count for the website
+HF_UPDATE_DOWNLOAD_COUNTS = (
+ os.environ.get("HF_UPDATE_DOWNLOAD_COUNTS", "AUTO").upper() in ENV_VARS_TRUE_AND_AUTO_VALUES
+)
+
+# For downloads and to check remote files metadata
+HF_DATASETS_MULTITHREADING_MAX_WORKERS = 16
+
+# Dataset viewer API
+USE_PARQUET_EXPORT = True
+
+# Batch size constants. For more info, see:
+# https://github.com/apache/arrow/blob/master/docs/source/cpp/arrays.rst#size-limitations-and-recommendations)
+DEFAULT_MAX_BATCH_SIZE = 1000
+
+DEFAULT_CDC_OPTIONS = {"min_chunk_size": 256 * 1024, "max_chunk_size": 1024 * 1024, "norm_level": 0}
+
+# Size of the preloaded record batch in `Dataset.__iter__`
+ARROW_READER_BATCH_SIZE_IN_DATASET_ITER = 10
+
+# Max uncompressed shard size in bytes (e.g. to shard parquet datasets in push_to_hub or download_and_prepare)
+MAX_SHARD_SIZE = "500MB"
+
+# Max uncompressed row group size in bytes (e.g. for parquet files in push_to_hub or download_and_prepare)
+MAX_ROW_GROUP_SIZE = "100MB"
+
+# Parquet configuration
+PARQUET_ROW_GROUP_SIZE_FOR_AUDIO_DATASETS = None
+PARQUET_ROW_GROUP_SIZE_FOR_IMAGE_DATASETS = None
+PARQUET_ROW_GROUP_SIZE_FOR_BINARY_DATASETS = None
+PARQUET_ROW_GROUP_SIZE_FOR_VIDEO_DATASETS = None
+
+# Arrow configuration
+ARROW_RECORD_BATCH_SIZE_FOR_AUDIO_DATASETS = 100
+ARROW_RECORD_BATCH_SIZE_FOR_IMAGE_DATASETS = 100
+ARROW_RECORD_BATCH_SIZE_FOR_BINARY_DATASETS = 100
+ARROW_RECORD_BATCH_SIZE_FOR_VIDEO_DATASETS = 10
+
+# Offline mode
+_offline = os.environ.get("HF_DATASETS_OFFLINE")
+HF_HUB_OFFLINE = constants.HF_HUB_OFFLINE if _offline is None else _offline.upper() in ENV_VARS_TRUE_VALUES
+HF_DATASETS_OFFLINE = HF_HUB_OFFLINE # kept for backward-compatibility
+
+# Here, `True` will disable progress bars globally without possibility of enabling it
+# programmatically. `False` will enable them without possibility of disabling them.
+# If environment variable is not set (None), then the user is free to enable/disable
+# them programmatically.
+# TL;DR: env variable has priority over code
+__HF_DATASETS_DISABLE_PROGRESS_BARS = os.environ.get("HF_DATASETS_DISABLE_PROGRESS_BARS")
+HF_DATASETS_DISABLE_PROGRESS_BARS: Optional[bool] = (
+ __HF_DATASETS_DISABLE_PROGRESS_BARS.upper() in ENV_VARS_TRUE_VALUES
+ if __HF_DATASETS_DISABLE_PROGRESS_BARS is not None
+ else None
+)
+
+# In-memory
+DEFAULT_IN_MEMORY_MAX_SIZE = 0 # Disabled
+IN_MEMORY_MAX_SIZE = float(os.environ.get("HF_DATASETS_IN_MEMORY_MAX_SIZE", DEFAULT_IN_MEMORY_MAX_SIZE))
+
+# File names
+DATASET_ARROW_FILENAME = "dataset.arrow"
+DATASET_INDICES_FILENAME = "indices.arrow"
+DATASET_STATE_JSON_FILENAME = "state.json"
+DATASET_INFO_FILENAME = "dataset_info.json"
+DATASETDICT_INFOS_FILENAME = "dataset_infos.json"
+LICENSE_FILENAME = "LICENSE"
+DATASETDICT_JSON_FILENAME = "dataset_dict.json"
+METADATA_CONFIGS_FIELD = "configs"
+REPOCARD_FILENAME = "README.md"
+REPOYAML_FILENAME = ".huggingface.yaml"
+
+MODULE_NAME_FOR_DYNAMIC_MODULES = "datasets_modules"
+
+MAX_DATASET_CONFIG_ID_READABLE_LENGTH = 255
+
+# Temporary cache directory prefix
+TEMP_CACHE_DIR_PREFIX = "hf_datasets-"
+
+# Streaming
+STREAMING_READ_MAX_RETRIES = 20
+STREAMING_READ_RETRY_INTERVAL = 5
+STREAMING_READ_SERVER_UNAVAILABLE_RETRY_INTERVAL = 20
+STREAMING_READ_RATE_LIMIT_RETRY_INTERVAL = 60
+STREAMING_OPEN_MAX_RETRIES = 20
+STREAMING_OPEN_RETRY_INTERVAL = 5
+
+# Datasets repositories exploration
+DATA_FILES_MAX_NUMBER_FOR_MODULE_INFERENCE = 200
+GLOBBED_DATA_FILES_MAX_NUMBER_FOR_MODULE_INFERENCE = 10
+ARCHIVED_DATA_FILES_MAX_NUMBER_FOR_MODULE_INFERENCE = 200
+
+# Async map functions
+MAX_NUM_RUNNING_ASYNC_MAP_FUNCTIONS_IN_PARALLEL = 1000
+
+# Progress bars
+PBAR_REFRESH_TIME_INTERVAL = 0.05 # 20 progress updates per sec
+
+# Maximum number of uploaded files per commit
+UPLOADS_MAX_NUMBER_PER_COMMIT = 50
+
+# Backward compatibility
+MAX_TABLE_NBYTES_FOR_PICKLING = 4 << 30
diff --git a/venv/lib/python3.10/site-packages/datasets/data_files.py b/venv/lib/python3.10/site-packages/datasets/data_files.py
new file mode 100644
index 0000000000000000000000000000000000000000..96d4daea52d3b9c5a840d097703f99603fc77563
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/data_files.py
@@ -0,0 +1,811 @@
+import os
+import re
+from functools import partial
+from glob import has_magic
+from pathlib import Path, PurePath
+from typing import Callable, Optional, Union
+
+import huggingface_hub
+from fsspec.core import url_to_fs
+from huggingface_hub import HfFileSystem
+from packaging import version
+from tqdm.contrib.concurrent import thread_map
+
+from . import config
+from .download import DownloadConfig
+from .naming import _split_re
+from .splits import Split
+from .utils import logging
+from .utils import tqdm as hf_tqdm
+from .utils.file_utils import _prepare_path_and_storage_options, is_local_path, is_relative_path, xbasename, xjoin
+from .utils.py_utils import string_to_dict
+
+
+SingleOriginMetadata = Union[tuple[str, str], tuple[str], tuple[()]]
+
+
+SANITIZED_DEFAULT_SPLIT = str(Split.TRAIN)
+
+
+logger = logging.get_logger(__name__)
+
+
+class Url(str):
+ pass
+
+
+class EmptyDatasetError(FileNotFoundError):
+ pass
+
+
+SPLIT_PATTERN_SHARDED = "data/{split}-[0-9][0-9][0-9][0-9][0-9]-of-[0-9][0-9][0-9][0-9][0-9]*.*"
+
+SPLIT_KEYWORDS = {
+ Split.TRAIN: ["train", "training"],
+ Split.VALIDATION: ["validation", "valid", "dev", "val"],
+ Split.TEST: ["test", "testing", "eval", "evaluation"],
+}
+NON_WORDS_CHARS = "-._ 0-9"
+if config.FSSPEC_VERSION < version.parse("2023.9.0"):
+ KEYWORDS_IN_FILENAME_BASE_PATTERNS = ["**[{sep}/]{keyword}[{sep}]*", "{keyword}[{sep}]*"]
+ KEYWORDS_IN_DIR_NAME_BASE_PATTERNS = [
+ "{keyword}/**",
+ "{keyword}[{sep}]*/**",
+ "**[{sep}/]{keyword}/**",
+ "**[{sep}/]{keyword}[{sep}]*/**",
+ ]
+elif config.FSSPEC_VERSION < version.parse("2023.12.0"):
+ KEYWORDS_IN_FILENAME_BASE_PATTERNS = ["**/*[{sep}/]{keyword}[{sep}]*", "{keyword}[{sep}]*"]
+ KEYWORDS_IN_DIR_NAME_BASE_PATTERNS = [
+ "{keyword}/**/*",
+ "{keyword}[{sep}]*/**/*",
+ "**/*[{sep}/]{keyword}/**/*",
+ "**/*[{sep}/]{keyword}[{sep}]*/**/*",
+ ]
+else:
+ KEYWORDS_IN_FILENAME_BASE_PATTERNS = ["**/{keyword}[{sep}]*", "**/*[{sep}]{keyword}[{sep}]*"]
+ KEYWORDS_IN_DIR_NAME_BASE_PATTERNS = [
+ "**/{keyword}/**",
+ "**/{keyword}[{sep}]*/**",
+ "**/*[{sep}]{keyword}/**",
+ "**/*[{sep}]{keyword}[{sep}]*/**",
+ ]
+
+DEFAULT_SPLITS = [Split.TRAIN, Split.VALIDATION, Split.TEST]
+DEFAULT_PATTERNS_SPLIT_IN_FILENAME = {
+ split: [
+ pattern.format(keyword=keyword, sep=NON_WORDS_CHARS)
+ for keyword in SPLIT_KEYWORDS[split]
+ for pattern in KEYWORDS_IN_FILENAME_BASE_PATTERNS
+ ]
+ for split in DEFAULT_SPLITS
+}
+DEFAULT_PATTERNS_SPLIT_IN_DIR_NAME = {
+ split: [
+ pattern.format(keyword=keyword, sep=NON_WORDS_CHARS)
+ for keyword in SPLIT_KEYWORDS[split]
+ for pattern in KEYWORDS_IN_DIR_NAME_BASE_PATTERNS
+ ]
+ for split in DEFAULT_SPLITS
+}
+
+
+DEFAULT_PATTERNS_ALL = {
+ Split.TRAIN: ["**"],
+}
+
+ALL_SPLIT_PATTERNS = [SPLIT_PATTERN_SHARDED]
+ALL_DEFAULT_PATTERNS = [
+ DEFAULT_PATTERNS_SPLIT_IN_DIR_NAME,
+ DEFAULT_PATTERNS_SPLIT_IN_FILENAME,
+ DEFAULT_PATTERNS_ALL,
+]
+WILDCARD_CHARACTERS = "*[]"
+FILES_TO_IGNORE = [
+ "README.md",
+ "config.json",
+ "dataset_info.json",
+ "dataset_infos.json",
+ "dummy_data.zip",
+ "dataset_dict.json",
+]
+
+
+def contains_wildcards(pattern: str) -> bool:
+ return any(wildcard_character in pattern for wildcard_character in WILDCARD_CHARACTERS)
+
+
+def sanitize_patterns(patterns: Union[dict, list, str]) -> dict[str, Union[list[str], "DataFilesList"]]:
+ """
+ Take the data_files patterns from the user, and format them into a dictionary.
+ Each key is the name of the split, and each value is a list of data files patterns (paths or urls).
+ The default split is "train".
+
+ Returns:
+ patterns: dictionary of split_name -> list of patterns
+ """
+ if isinstance(patterns, dict):
+ return {str(key): value if isinstance(value, list) else [value] for key, value in patterns.items()}
+ elif isinstance(patterns, str):
+ return {SANITIZED_DEFAULT_SPLIT: [patterns]}
+ elif isinstance(patterns, list):
+ if any(isinstance(pattern, dict) for pattern in patterns):
+ for pattern in patterns:
+ if not (
+ isinstance(pattern, dict)
+ and len(pattern) == 2
+ and "split" in pattern
+ and isinstance(pattern.get("path"), (str, list))
+ ):
+ raise ValueError(
+ f"Expected each split to have a 'path' key which can be a string or a list of strings, but got {pattern}"
+ )
+ splits = [pattern["split"] for pattern in patterns]
+ if len(set(splits)) != len(splits):
+ raise ValueError(f"Some splits are duplicated in data_files: {splits}")
+ return {
+ str(pattern["split"]): pattern["path"] if isinstance(pattern["path"], list) else [pattern["path"]]
+ for pattern in patterns
+ }
+ else:
+ return {SANITIZED_DEFAULT_SPLIT: patterns}
+ else:
+ return sanitize_patterns(list(patterns))
+
+
+def _is_inside_unrequested_special_dir(matched_rel_path: str, pattern: str) -> bool:
+ """
+ When a path matches a pattern, we additionally check if it's inside a special directory
+ we ignore by default (if it starts with a double underscore).
+
+ Users can still explicitly request a filepath inside such a directory if "__pycache__" is
+ mentioned explicitly in the requested pattern.
+
+ Some examples:
+
+ base directory:
+
+ ./
+ └── __pycache__
+ └── b.txt
+
+ >>> _is_inside_unrequested_special_dir("__pycache__/b.txt", "**")
+ True
+ >>> _is_inside_unrequested_special_dir("__pycache__/b.txt", "*/b.txt")
+ True
+ >>> _is_inside_unrequested_special_dir("__pycache__/b.txt", "__pycache__/*")
+ False
+ >>> _is_inside_unrequested_special_dir("__pycache__/b.txt", "__*/*")
+ False
+ """
+ # We just need to check if every special directories from the path is present explicitly in the pattern.
+ # Since we assume that the path matches the pattern, it's equivalent to counting that both
+ # the parent path and the parent pattern have the same number of special directories.
+ data_dirs_to_ignore_in_path = [part for part in PurePath(matched_rel_path).parent.parts if part.startswith("__")]
+ data_dirs_to_ignore_in_pattern = [part for part in PurePath(pattern).parent.parts if part.startswith("__")]
+ return len(data_dirs_to_ignore_in_path) != len(data_dirs_to_ignore_in_pattern)
+
+
+def _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(matched_rel_path: str, pattern: str) -> bool:
+ """
+ When a path matches a pattern, we additionally check if it's a hidden file or if it's inside
+ a hidden directory we ignore by default, i.e. if the file name or a parent directory name starts with a dot.
+
+ Users can still explicitly request a filepath that is hidden or is inside a hidden directory
+ if the hidden part is mentioned explicitly in the requested pattern.
+
+ Some examples:
+
+ base directory:
+
+ ./
+ └── .hidden_file.txt
+
+ >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_file.txt", "**")
+ True
+ >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_file.txt", ".*")
+ False
+
+ base directory:
+
+ ./
+ └── .hidden_dir
+ └── a.txt
+
+ >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/a.txt", "**")
+ True
+ >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/a.txt", ".*/*")
+ False
+ >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/a.txt", ".hidden_dir/*")
+ False
+
+ base directory:
+
+ ./
+ └── .hidden_dir
+ └── .hidden_file.txt
+
+ >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/.hidden_file.txt", "**")
+ True
+ >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/.hidden_file.txt", ".*/*")
+ True
+ >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/.hidden_file.txt", ".*/.*")
+ False
+ >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/.hidden_file.txt", ".hidden_dir/*")
+ True
+ >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/.hidden_file.txt", ".hidden_dir/.*")
+ False
+ """
+ # We just need to check if every hidden part from the path is present explicitly in the pattern.
+ # Since we assume that the path matches the pattern, it's equivalent to counting that both
+ # the path and the pattern have the same number of hidden parts.
+ hidden_directories_in_path = [
+ part for part in PurePath(matched_rel_path).parts if part.startswith(".") and not set(part) == {"."}
+ ]
+ hidden_directories_in_pattern = [
+ part for part in PurePath(pattern).parts if part.startswith(".") and not set(part) == {"."}
+ ]
+ return len(hidden_directories_in_path) != len(hidden_directories_in_pattern)
+
+
+def _get_data_files_patterns(pattern_resolver: Callable[[str], list[str]]) -> dict[str, list[str]]:
+ """
+ Get the default pattern from a directory or repository by testing all the supported patterns.
+ The first patterns to return a non-empty list of data files is returned.
+
+ In order, it first tests if SPLIT_PATTERN_SHARDED works, otherwise it tests the patterns in ALL_DEFAULT_PATTERNS.
+ """
+ # first check the split patterns like data/{split}-00000-of-00001.parquet
+ for split_pattern in ALL_SPLIT_PATTERNS:
+ pattern = split_pattern.replace("{split}", "*")
+ try:
+ data_files = pattern_resolver(pattern)
+ except FileNotFoundError:
+ continue
+ if len(data_files) > 0:
+ splits: set[str] = set()
+ for p in data_files:
+ p_parts = string_to_dict(xbasename(p), xbasename(split_pattern))
+ assert p_parts is not None
+ splits.add(p_parts["split"])
+
+ if any(not re.match(_split_re, split) for split in splits):
+ raise ValueError(f"Split name should match '{_split_re}'' but got '{splits}'.")
+ sorted_splits = [str(split) for split in DEFAULT_SPLITS if split in splits] + sorted(
+ splits - {str(split) for split in DEFAULT_SPLITS}
+ )
+ return {split: [split_pattern.format(split=split)] for split in sorted_splits}
+ # then check the default patterns based on train/valid/test splits
+ for patterns_dict in ALL_DEFAULT_PATTERNS:
+ non_empty_splits = []
+ for split, patterns in patterns_dict.items():
+ for pattern in patterns:
+ try:
+ data_files = pattern_resolver(pattern)
+ except FileNotFoundError:
+ continue
+ if len(data_files) > 0:
+ non_empty_splits.append(split)
+ break
+ if non_empty_splits:
+ return {split: patterns_dict[split] for split in non_empty_splits}
+ raise FileNotFoundError(f"Couldn't resolve pattern {pattern} with resolver {pattern_resolver}")
+
+
+def resolve_pattern(
+ pattern: str,
+ base_path: str,
+ allowed_extensions: Optional[list[str]] = None,
+ download_config: Optional[DownloadConfig] = None,
+) -> list[str]:
+ """
+ Resolve the paths and URLs of the data files from the pattern passed by the user.
+
+ You can use patterns to resolve multiple local files. Here are a few examples:
+ - *.csv to match all the CSV files at the first level
+ - **.csv to match all the CSV files at any level
+ - data/* to match all the files inside "data"
+ - data/** to match all the files inside "data" and its subdirectories
+
+ The patterns are resolved using the fsspec glob. In fsspec>=2023.12.0 this is equivalent to
+ Python's glob.glob, Path.glob, Path.match and fnmatch where ** is unsupported with a prefix/suffix
+ other than a forward slash /.
+
+ More generally:
+ - '*' matches any character except a forward-slash (to match just the file or directory name)
+ - '**' matches any character including a forward-slash /
+
+ Hidden files and directories (i.e. whose names start with a dot) are ignored, unless they are explicitly requested.
+ The same applies to special directories that start with a double underscore like "__pycache__".
+ You can still include one if the pattern explicitly mentions it:
+ - to include a hidden file: "*/.hidden.txt" or "*/.*"
+ - to include a hidden directory: ".hidden/*" or ".*/*"
+ - to include a special directory: "__special__/*" or "__*/*"
+
+ Example::
+
+ >>> from datasets.data_files import resolve_pattern
+ >>> base_path = "."
+ >>> resolve_pattern("docs/**/*.py", base_path)
+ [/Users/mariosasko/Desktop/projects/datasets/docs/source/_config.py']
+
+ Args:
+ pattern (str): Unix pattern or paths or URLs of the data files to resolve.
+ The paths can be absolute or relative to base_path.
+ Remote filesystems using fsspec are supported, e.g. with the hf:// protocol.
+ base_path (str): Base path to use when resolving relative paths.
+ allowed_extensions (Optional[list], optional): White-list of file extensions to use. Defaults to None (all extensions).
+ For example: allowed_extensions=[".csv", ".json", ".txt", ".parquet"]
+ download_config ([`DownloadConfig`], *optional*): Specific download configuration parameters.
+ Returns:
+ List[str]: List of paths or URLs to the local or remote files that match the patterns.
+ """
+ if is_relative_path(pattern):
+ pattern = xjoin(base_path, pattern)
+ elif is_local_path(pattern):
+ base_path = os.path.splitdrive(pattern)[0] + os.sep
+ else:
+ base_path = ""
+ pattern, storage_options = _prepare_path_and_storage_options(pattern, download_config=download_config)
+ fs, fs_pattern = url_to_fs(pattern, **storage_options)
+ files_to_ignore = set(FILES_TO_IGNORE) - {xbasename(pattern)}
+ protocol = (
+ pattern.split("://")[0]
+ if "://" in pattern
+ else (fs.protocol if isinstance(fs.protocol, str) else fs.protocol[0])
+ )
+ protocol_prefix = protocol + "://" if protocol != "file" else ""
+ glob_kwargs = {}
+ if protocol == "hf":
+ # 10 times faster glob with detail=True (ignores costly info like lastCommit)
+ glob_kwargs["expand_info"] = False
+ matched_paths = [
+ filepath if "://" in filepath else protocol_prefix + filepath
+ for filepath, info in fs.glob(pattern, detail=True, **glob_kwargs).items()
+ if (info["type"] == "file" or (info.get("islink") and os.path.isfile(os.path.realpath(filepath))))
+ and (xbasename(filepath) not in files_to_ignore)
+ and not _is_inside_unrequested_special_dir(filepath, fs_pattern)
+ and not _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(filepath, fs_pattern)
+ ] # ignore .ipynb and __pycache__, but keep /../
+ if allowed_extensions is not None:
+ out = [
+ filepath
+ for filepath in matched_paths
+ if any("." + suffix in allowed_extensions for suffix in xbasename(filepath).split(".")[1:])
+ ]
+ if len(out) < len(matched_paths):
+ invalid_matched_files = list(set(matched_paths) - set(out))
+ logger.info(
+ f"Some files matched the pattern '{pattern}' but don't have valid data file extensions: {invalid_matched_files}"
+ )
+ else:
+ out = matched_paths
+ if not out:
+ error_msg = f"Unable to find '{pattern}'"
+ if allowed_extensions is not None:
+ error_msg += f" with any supported extension {list(allowed_extensions)}"
+ raise FileNotFoundError(error_msg)
+ return out
+
+
+def get_data_patterns(base_path: str, download_config: Optional[DownloadConfig] = None) -> dict[str, list[str]]:
+ """
+ Get the default pattern from a directory testing all the supported patterns.
+ The first patterns to return a non-empty list of data files is returned.
+
+ Some examples of supported patterns:
+
+ Input:
+
+ my_dataset_repository/
+ ├── README.md
+ └── dataset.csv
+
+ Output:
+
+ {'train': ['**']}
+
+ Input:
+
+ my_dataset_repository/
+ ├── README.md
+ ├── train.csv
+ └── test.csv
+
+ my_dataset_repository/
+ ├── README.md
+ └── data/
+ ├── train.csv
+ └── test.csv
+
+ my_dataset_repository/
+ ├── README.md
+ ├── train_0.csv
+ ├── train_1.csv
+ ├── train_2.csv
+ ├── train_3.csv
+ ├── test_0.csv
+ └── test_1.csv
+
+ Output:
+
+ {'train': ['**/train[-._ 0-9]*', '**/*[-._ 0-9]train[-._ 0-9]*', '**/training[-._ 0-9]*', '**/*[-._ 0-9]training[-._ 0-9]*'],
+ 'test': ['**/test[-._ 0-9]*', '**/*[-._ 0-9]test[-._ 0-9]*', '**/testing[-._ 0-9]*', '**/*[-._ 0-9]testing[-._ 0-9]*', ...]}
+
+ Input:
+
+ my_dataset_repository/
+ ├── README.md
+ └── data/
+ ├── train/
+ │ ├── shard_0.csv
+ │ ├── shard_1.csv
+ │ ├── shard_2.csv
+ │ └── shard_3.csv
+ └── test/
+ ├── shard_0.csv
+ └── shard_1.csv
+
+ Output:
+
+ {'train': ['**/train/**', '**/train[-._ 0-9]*/**', '**/*[-._ 0-9]train/**', '**/*[-._ 0-9]train[-._ 0-9]*/**', ...],
+ 'test': ['**/test/**', '**/test[-._ 0-9]*/**', '**/*[-._ 0-9]test/**', '**/*[-._ 0-9]test[-._ 0-9]*/**', ...]}
+
+ Input:
+
+ my_dataset_repository/
+ ├── README.md
+ └── data/
+ ├── train-00000-of-00003.csv
+ ├── train-00001-of-00003.csv
+ ├── train-00002-of-00003.csv
+ ├── test-00000-of-00001.csv
+ ├── random-00000-of-00003.csv
+ ├── random-00001-of-00003.csv
+ └── random-00002-of-00003.csv
+
+ Output:
+
+ {'train': ['data/train-[0-9][0-9][0-9][0-9][0-9]-of-[0-9][0-9][0-9][0-9][0-9]*.*'],
+ 'test': ['data/test-[0-9][0-9][0-9][0-9][0-9]-of-[0-9][0-9][0-9][0-9][0-9]*.*'],
+ 'random': ['data/random-[0-9][0-9][0-9][0-9][0-9]-of-[0-9][0-9][0-9][0-9][0-9]*.*']}
+
+ In order, it first tests if SPLIT_PATTERN_SHARDED works, otherwise it tests the patterns in ALL_DEFAULT_PATTERNS.
+ """
+ resolver = partial(resolve_pattern, base_path=base_path, download_config=download_config)
+ try:
+ return _get_data_files_patterns(resolver)
+ except FileNotFoundError:
+ raise EmptyDatasetError(f"The directory at {base_path} doesn't contain any data files") from None
+
+
+def _get_single_origin_metadata(
+ data_file: str,
+ download_config: Optional[DownloadConfig] = None,
+) -> SingleOriginMetadata:
+ data_file, storage_options = _prepare_path_and_storage_options(data_file, download_config=download_config)
+ fs, *_ = url_to_fs(data_file, **storage_options)
+ if isinstance(fs, HfFileSystem):
+ resolved_path = fs.resolve_path(data_file)
+ return resolved_path.repo_id, resolved_path.revision
+ elif data_file.startswith(config.HF_ENDPOINT):
+ hffs = HfFileSystem(endpoint=config.HF_ENDPOINT, token=download_config.token)
+ data_file = "hf://" + data_file[len(config.HF_ENDPOINT) + 1 :].replace("/resolve/", "@", 1)
+ resolved_path = hffs.resolve_path(data_file)
+ return resolved_path.repo_id, resolved_path.revision
+ info = fs.info(data_file)
+ # s3fs uses "ETag", gcsfs uses "etag", and for local we simply check mtime
+ for key in ["ETag", "etag", "mtime"]:
+ if key in info:
+ return (str(info[key]),)
+ return ()
+
+
+def _get_origin_metadata(
+ data_files: list[str],
+ download_config: Optional[DownloadConfig] = None,
+ max_workers: Optional[int] = None,
+) -> list[SingleOriginMetadata]:
+ max_workers = max_workers if max_workers is not None else config.HF_DATASETS_MULTITHREADING_MAX_WORKERS
+ if all("hf://" in data_file for data_file in data_files):
+ # No need for multithreading here since the origin metadata of HF files
+ # is (repo_id, revision) and is cached after first .info() call.
+ return [
+ _get_single_origin_metadata(data_file, download_config=download_config)
+ for data_file in hf_tqdm(
+ data_files,
+ desc="Resolving data files",
+ # set `disable=None` rather than `disable=False` by default to disable progress bar when no TTY attached
+ disable=len(data_files) <= 16 or None,
+ )
+ ]
+ return thread_map(
+ partial(_get_single_origin_metadata, download_config=download_config),
+ data_files,
+ max_workers=max_workers,
+ tqdm_class=hf_tqdm,
+ desc="Resolving data files",
+ # set `disable=None` rather than `disable=False` by default to disable progress bar when no TTY attached
+ disable=len(data_files) <= 16 or None,
+ )
+
+
+class DataFilesList(list[str]):
+ """
+ List of data files (absolute local paths or URLs).
+ It has two construction methods given the user's data files patterns:
+ - ``from_hf_repo``: resolve patterns inside a dataset repository
+ - ``from_local_or_remote``: resolve patterns from a local path
+
+ Moreover, DataFilesList has an additional attribute ``origin_metadata``.
+ It can store:
+ - the last modified time of local files
+ - ETag of remote files
+ - commit sha of a dataset repository
+
+ Thanks to this additional attribute, it is possible to hash the list
+ and get a different hash if and only if at least one file changed.
+ This is useful for caching Dataset objects that are obtained from a list of data files.
+ """
+
+ def __init__(self, data_files: list[str], origin_metadata: list[SingleOriginMetadata]) -> None:
+ super().__init__(data_files)
+ self.origin_metadata = origin_metadata
+
+ def __add__(self, other: "DataFilesList") -> "DataFilesList":
+ return DataFilesList([*self, *other], self.origin_metadata + other.origin_metadata)
+
+ @classmethod
+ def from_hf_repo(
+ cls,
+ patterns: list[str],
+ dataset_info: huggingface_hub.hf_api.DatasetInfo,
+ base_path: Optional[str] = None,
+ allowed_extensions: Optional[list[str]] = None,
+ download_config: Optional[DownloadConfig] = None,
+ ) -> "DataFilesList":
+ base_path = f"hf://datasets/{dataset_info.id}@{dataset_info.sha}/{base_path or ''}".rstrip("/")
+ return cls.from_patterns(
+ patterns, base_path=base_path, allowed_extensions=allowed_extensions, download_config=download_config
+ )
+
+ @classmethod
+ def from_local_or_remote(
+ cls,
+ patterns: list[str],
+ base_path: Optional[str] = None,
+ allowed_extensions: Optional[list[str]] = None,
+ download_config: Optional[DownloadConfig] = None,
+ ) -> "DataFilesList":
+ base_path = base_path if base_path is not None else Path().resolve().as_posix()
+ return cls.from_patterns(
+ patterns, base_path=base_path, allowed_extensions=allowed_extensions, download_config=download_config
+ )
+
+ @classmethod
+ def from_patterns(
+ cls,
+ patterns: list[str],
+ base_path: Optional[str] = None,
+ allowed_extensions: Optional[list[str]] = None,
+ download_config: Optional[DownloadConfig] = None,
+ ) -> "DataFilesList":
+ base_path = base_path if base_path is not None else Path().resolve().as_posix()
+ data_files = []
+ for pattern in patterns:
+ try:
+ data_files.extend(
+ resolve_pattern(
+ pattern,
+ base_path=base_path,
+ allowed_extensions=allowed_extensions,
+ download_config=download_config,
+ )
+ )
+ except FileNotFoundError:
+ if not has_magic(pattern):
+ raise
+ origin_metadata = _get_origin_metadata(data_files, download_config=download_config)
+ return cls(data_files, origin_metadata)
+
+ def filter(
+ self, *, extensions: Optional[list[str]] = None, file_names: Optional[list[str]] = None
+ ) -> "DataFilesList":
+ patterns = []
+ if extensions:
+ ext_pattern = "|".join(re.escape(ext) for ext in extensions)
+ patterns.append(re.compile(f".*({ext_pattern})(\\..+)?$"))
+ if file_names:
+ fn_pattern = "|".join(re.escape(fn) for fn in file_names)
+ patterns.append(re.compile(rf".*[\/]?({fn_pattern})$"))
+ if patterns:
+ return DataFilesList(
+ [data_file for data_file in self if any(pattern.match(data_file) for pattern in patterns)],
+ origin_metadata=self.origin_metadata,
+ )
+ else:
+ return DataFilesList(list(self), origin_metadata=self.origin_metadata)
+
+
+class DataFilesDict(dict[str, DataFilesList]):
+ """
+ Dict of split_name -> list of data files (absolute local paths or URLs).
+ It has two construction methods given the user's data files patterns :
+ - ``from_hf_repo``: resolve patterns inside a dataset repository
+ - ``from_local_or_remote``: resolve patterns from a local path
+
+ Moreover, each list is a DataFilesList. It is possible to hash the dictionary
+ and get a different hash if and only if at least one file changed.
+ For more info, see [`DataFilesList`].
+
+ This is useful for caching Dataset objects that are obtained from a list of data files.
+
+ Changing the order of the keys of this dictionary also doesn't change its hash.
+ """
+
+ @classmethod
+ def from_local_or_remote(
+ cls,
+ patterns: dict[str, Union[list[str], DataFilesList]],
+ base_path: Optional[str] = None,
+ allowed_extensions: Optional[list[str]] = None,
+ download_config: Optional[DownloadConfig] = None,
+ ) -> "DataFilesDict":
+ out = cls()
+ for key, patterns_for_key in patterns.items():
+ out[key] = (
+ patterns_for_key
+ if isinstance(patterns_for_key, DataFilesList)
+ else DataFilesList.from_local_or_remote(
+ patterns_for_key,
+ base_path=base_path,
+ allowed_extensions=allowed_extensions,
+ download_config=download_config,
+ )
+ )
+ return out
+
+ @classmethod
+ def from_hf_repo(
+ cls,
+ patterns: dict[str, Union[list[str], DataFilesList]],
+ dataset_info: huggingface_hub.hf_api.DatasetInfo,
+ base_path: Optional[str] = None,
+ allowed_extensions: Optional[list[str]] = None,
+ download_config: Optional[DownloadConfig] = None,
+ ) -> "DataFilesDict":
+ out = cls()
+ for key, patterns_for_key in patterns.items():
+ out[key] = (
+ patterns_for_key
+ if isinstance(patterns_for_key, DataFilesList)
+ else DataFilesList.from_hf_repo(
+ patterns_for_key,
+ dataset_info=dataset_info,
+ base_path=base_path,
+ allowed_extensions=allowed_extensions,
+ download_config=download_config,
+ )
+ )
+ return out
+
+ @classmethod
+ def from_patterns(
+ cls,
+ patterns: dict[str, Union[list[str], DataFilesList]],
+ base_path: Optional[str] = None,
+ allowed_extensions: Optional[list[str]] = None,
+ download_config: Optional[DownloadConfig] = None,
+ ) -> "DataFilesDict":
+ out = cls()
+ for key, patterns_for_key in patterns.items():
+ out[key] = (
+ patterns_for_key
+ if isinstance(patterns_for_key, DataFilesList)
+ else DataFilesList.from_patterns(
+ patterns_for_key,
+ base_path=base_path,
+ allowed_extensions=allowed_extensions,
+ download_config=download_config,
+ )
+ )
+ return out
+
+ def filter(
+ self, *, extensions: Optional[list[str]] = None, file_names: Optional[list[str]] = None
+ ) -> "DataFilesDict":
+ out = type(self)()
+ for key, data_files_list in self.items():
+ out[key] = data_files_list.filter(extensions=extensions, file_names=file_names)
+ return out
+
+
+class DataFilesPatternsList(list[str]):
+ """
+ List of data files patterns (absolute local paths or URLs).
+ For each pattern there should also be a list of allowed extensions
+ to keep, or a None ot keep all the files for the pattern.
+ """
+
+ def __init__(
+ self,
+ patterns: list[str],
+ allowed_extensions: list[Optional[list[str]]],
+ ):
+ super().__init__(patterns)
+ self.allowed_extensions = allowed_extensions
+
+ def __add__(self, other):
+ return DataFilesList([*self, *other], self.allowed_extensions + other.allowed_extensions)
+
+ @classmethod
+ def from_patterns(
+ cls, patterns: list[str], allowed_extensions: Optional[list[str]] = None
+ ) -> "DataFilesPatternsList":
+ return cls(patterns, [allowed_extensions] * len(patterns))
+
+ def resolve(
+ self,
+ base_path: str,
+ download_config: Optional[DownloadConfig] = None,
+ ) -> "DataFilesList":
+ base_path = base_path if base_path is not None else Path().resolve().as_posix()
+ data_files = []
+ for pattern, allowed_extensions in zip(self, self.allowed_extensions):
+ try:
+ data_files.extend(
+ resolve_pattern(
+ pattern,
+ base_path=base_path,
+ allowed_extensions=allowed_extensions,
+ download_config=download_config,
+ )
+ )
+ except FileNotFoundError:
+ if not has_magic(pattern):
+ raise
+ origin_metadata = _get_origin_metadata(data_files, download_config=download_config)
+ return DataFilesList(data_files, origin_metadata)
+
+ def filter_extensions(self, extensions: list[str]) -> "DataFilesPatternsList":
+ return DataFilesPatternsList(
+ self, [allowed_extensions + extensions for allowed_extensions in self.allowed_extensions]
+ )
+
+
+class DataFilesPatternsDict(dict[str, DataFilesPatternsList]):
+ """
+ Dict of split_name -> list of data files patterns (absolute local paths or URLs).
+ """
+
+ @classmethod
+ def from_patterns(
+ cls, patterns: dict[str, list[str]], allowed_extensions: Optional[list[str]] = None
+ ) -> "DataFilesPatternsDict":
+ out = cls()
+ for key, patterns_for_key in patterns.items():
+ out[key] = (
+ patterns_for_key
+ if isinstance(patterns_for_key, DataFilesPatternsList)
+ else DataFilesPatternsList.from_patterns(
+ patterns_for_key,
+ allowed_extensions=allowed_extensions,
+ )
+ )
+ return out
+
+ def resolve(
+ self,
+ base_path: str,
+ download_config: Optional[DownloadConfig] = None,
+ ) -> "DataFilesDict":
+ out = DataFilesDict()
+ for key, data_files_patterns_list in self.items():
+ out[key] = data_files_patterns_list.resolve(base_path, download_config)
+ return out
+
+ def filter_extensions(self, extensions: list[str]) -> "DataFilesPatternsDict":
+ out = type(self)()
+ for key, data_files_patterns_list in self.items():
+ out[key] = data_files_patterns_list.filter_extensions(extensions)
+ return out
diff --git a/venv/lib/python3.10/site-packages/datasets/dataset_dict.py b/venv/lib/python3.10/site-packages/datasets/dataset_dict.py
new file mode 100644
index 0000000000000000000000000000000000000000..63a93429c4527a823ac3a5c3f5c0dab31e15ad54
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/dataset_dict.py
@@ -0,0 +1,2852 @@
+import contextlib
+import copy
+import fnmatch
+import itertools
+import json
+import math
+import posixpath
+import random
+import re
+import time
+from collections.abc import Sequence
+from functools import partial
+from pathlib import Path
+from typing import Callable, Optional, Union
+
+import fsspec
+import numpy as np
+from fsspec.core import url_to_fs
+from huggingface_hub import (
+ CommitInfo,
+ CommitOperationAdd,
+ CommitOperationDelete,
+ DatasetCard,
+ DatasetCardData,
+ HfApi,
+)
+from huggingface_hub.hf_api import RepoFile
+from huggingface_hub.utils import HfHubHTTPError, RepositoryNotFoundError
+
+from . import config
+from .arrow_dataset import (
+ PUSH_TO_HUB_WITHOUT_METADATA_CONFIGS_SPLIT_PATTERN_SHARDED,
+ Dataset,
+)
+from .features import Features
+from .features.features import FeatureType
+from .info import DatasetInfo, DatasetInfosDict
+from .iterable_dataset import IterableDataset
+from .naming import _split_re
+from .splits import NamedSplit, Split, SplitDict, SplitInfo
+from .table import Table
+from .utils import logging
+from .utils.doc_utils import is_documented_by
+from .utils.metadata import MetadataConfigs
+from .utils.py_utils import asdict, glob_pattern_to_regex, string_to_dict
+from .utils.typing import PathLike
+
+
+logger = logging.get_logger(__name__)
+
+
+class bind(partial):
+ def __call__(self, *fn_args, **fn_kwargs):
+ return self.func(*fn_args, *self.args, **fn_kwargs)
+
+
+class DatasetDict(dict[Union[str, NamedSplit], "Dataset"]):
+ """A dictionary (dict of str: datasets.Dataset) with dataset transforms methods (map, filter, etc.)"""
+
+ def _check_values_type(self):
+ for dataset in self.values():
+ if not isinstance(dataset, Dataset):
+ raise TypeError(f"Values in `DatasetDict` should be of type `Dataset` but got type '{type(dataset)}'")
+
+ def _check_values_features(self):
+ items = list(self.items())
+ for item_a, item_b in zip(items[:-1], items[1:]):
+ if item_a[1].features != item_b[1].features:
+ raise ValueError(
+ f"All datasets in `DatasetDict` should have the same features but features for '{item_a[0]}' and '{item_b[0]}' don't match: {item_a[1].features} != {item_b[1].features}"
+ )
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ # Here `del` is used to del the pyarrow tables. This properly closes the files used for memory mapped tables
+ for dataset in self.values():
+ if hasattr(dataset, "_data"):
+ del dataset._data
+ if hasattr(dataset, "_indices"):
+ del dataset._indices
+
+ def __getitem__(self, k) -> Dataset:
+ if isinstance(k, (str, NamedSplit)) or len(self) == 0:
+ return super().__getitem__(k)
+ else:
+ available_suggested_splits = [
+ split for split in (Split.TRAIN, Split.TEST, Split.VALIDATION) if split in self
+ ]
+ suggested_split = available_suggested_splits[0] if available_suggested_splits else list(self)[0]
+ raise KeyError(
+ f"Invalid key: {k}. Please first select a split. For example: "
+ f"`my_dataset_dictionary['{suggested_split}'][{k}]`. "
+ f"Available splits: {sorted(self)}"
+ )
+
+ @property
+ def data(self) -> dict[str, Table]:
+ """The Apache Arrow tables backing each split.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes")
+ >>> ds.data
+ ```
+ """
+ self._check_values_type()
+ return {k: dataset.data for k, dataset in self.items()}
+
+ @property
+ def cache_files(self) -> dict[str, dict]:
+ """The cache files containing the Apache Arrow table backing each split.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes")
+ >>> ds.cache_files
+ {'test': [{'filename': '/root/.cache/huggingface/datasets/rotten_tomatoes_movie_review/default/1.0.0/40d411e45a6ce3484deed7cc15b82a53dad9a72aafd9f86f8f227134bec5ca46/rotten_tomatoes_movie_review-test.arrow'}],
+ 'train': [{'filename': '/root/.cache/huggingface/datasets/rotten_tomatoes_movie_review/default/1.0.0/40d411e45a6ce3484deed7cc15b82a53dad9a72aafd9f86f8f227134bec5ca46/rotten_tomatoes_movie_review-train.arrow'}],
+ 'validation': [{'filename': '/root/.cache/huggingface/datasets/rotten_tomatoes_movie_review/default/1.0.0/40d411e45a6ce3484deed7cc15b82a53dad9a72aafd9f86f8f227134bec5ca46/rotten_tomatoes_movie_review-validation.arrow'}]}
+ ```
+ """
+ self._check_values_type()
+ return {k: dataset.cache_files for k, dataset in self.items()}
+
+ @property
+ def num_columns(self) -> dict[str, int]:
+ """Number of columns in each split of the dataset.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes")
+ >>> ds.num_columns
+ {'test': 2, 'train': 2, 'validation': 2}
+ ```
+ """
+ self._check_values_type()
+ return {k: dataset.num_columns for k, dataset in self.items()}
+
+ @property
+ def num_rows(self) -> dict[str, int]:
+ """Number of rows in each split of the dataset.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes")
+ >>> ds.num_rows
+ {'test': 1066, 'train': 8530, 'validation': 1066}
+ ```
+ """
+ self._check_values_type()
+ return {k: dataset.num_rows for k, dataset in self.items()}
+
+ @property
+ def column_names(self) -> dict[str, list[str]]:
+ """Names of the columns in each split of the dataset.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes")
+ >>> ds.column_names
+ {'test': ['text', 'label'],
+ 'train': ['text', 'label'],
+ 'validation': ['text', 'label']}
+ ```
+ """
+ self._check_values_type()
+ return {k: dataset.column_names for k, dataset in self.items()}
+
+ @property
+ def shape(self) -> dict[str, tuple[int]]:
+ """Shape of each split of the dataset (number of rows, number of columns).
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes")
+ >>> ds.shape
+ {'test': (1066, 2), 'train': (8530, 2), 'validation': (1066, 2)}
+ ```
+ """
+ self._check_values_type()
+ return {k: dataset.shape for k, dataset in self.items()}
+
+ def flatten(self, max_depth=16) -> "DatasetDict":
+ """Flatten the Apache Arrow Table of each split (nested features are flatten).
+ Each column with a struct type is flattened into one column per struct field.
+ Other columns are left unchanged.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("rajpurkar/squad")
+ >>> ds["train"].features
+ {'id': Value('string'),
+ 'title': Value('string'),
+ 'context': Value('string'),
+ 'question': Value('string'),
+ 'answers.text': List(Value('string')),
+ 'answers.answer_start': List(Value('int32'))}
+ >>> ds.flatten()
+ DatasetDict({
+ train: Dataset({
+ features: ['id', 'title', 'context', 'question', 'answers.text', 'answers.answer_start'],
+ num_rows: 87599
+ })
+ validation: Dataset({
+ features: ['id', 'title', 'context', 'question', 'answers.text', 'answers.answer_start'],
+ num_rows: 10570
+ })
+ })
+ ```
+ """
+ self._check_values_type()
+ return DatasetDict({k: dataset.flatten(max_depth=max_depth) for k, dataset in self.items()})
+
+ def unique(self, column: str) -> dict[str, list]:
+ """Return a list of the unique elements in a column for each split.
+
+ This is implemented in the low-level backend and as such, very fast.
+
+ Args:
+ column (`str`):
+ column name (list all the column names with [`~datasets.DatasetDict.column_names`])
+
+ Returns:
+ Dict[`str`, `list`]: Dictionary of unique elements in the given column.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes")
+ >>> ds.unique("label")
+ {'test': [1, 0], 'train': [1, 0], 'validation': [1, 0]}
+ ```
+ """
+ self._check_values_type()
+ return {k: dataset.unique(column) for k, dataset in self.items()}
+
+ def cleanup_cache_files(self) -> dict[str, int]:
+ """Clean up all cache files in the dataset cache directory, excepted the currently used cache file if there is one.
+ Be careful when running this command that no other process is currently using other cache files.
+
+ Return:
+ `Dict` with the number of removed files for each split
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes")
+ >>> ds.cleanup_cache_files()
+ {'test': 0, 'train': 0, 'validation': 0}
+ ```
+ """
+ self._check_values_type()
+ return {k: dataset.cleanup_cache_files() for k, dataset in self.items()}
+
+ def __repr__(self):
+ repr = "\n".join([f"{k}: {v}" for k, v in self.items()])
+ repr = re.sub(r"^", " " * 4, repr, count=0, flags=re.M)
+ return f"DatasetDict({{\n{repr}\n}})"
+
+ def cast(self, features: Features) -> "DatasetDict":
+ """
+ Cast the dataset to a new set of features.
+ The transformation is applied to all the datasets of the dataset dictionary.
+
+ Args:
+ features ([`Features`]):
+ New features to cast the dataset to.
+ The name and order of the fields in the features must match the current column names.
+ The type of the data must also be convertible from one type to the other.
+ For non-trivial conversion, e.g. `string` <-> `ClassLabel` you should use [`~DatasetDict.map`] to update the dataset.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset, ClassLabel, Value
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes")
+ >>> ds["train"].features
+ {'label': ClassLabel(names=['neg', 'pos']),
+ 'text': Value('string')}
+ >>> new_features = ds["train"].features.copy()
+ >>> new_features['label'] = ClassLabel(names=['bad', 'good'])
+ >>> new_features['text'] = Value('large_string')
+ >>> ds = ds.cast(new_features)
+ >>> ds["train"].features
+ {'label': ClassLabel(names=['bad', 'good']),
+ 'text': Value('large_string')}
+ ```
+ """
+ self._check_values_type()
+ return DatasetDict({k: dataset.cast(features=features) for k, dataset in self.items()})
+
+ def cast_column(self, column: str, feature) -> "DatasetDict":
+ """Cast column to feature for decoding.
+
+ Args:
+ column (`str`):
+ Column name.
+ feature ([`Feature`]):
+ Target feature.
+
+ Returns:
+ [`DatasetDict`]
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset, ClassLabel
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes")
+ >>> ds["train"].features
+ {'label': ClassLabel(names=['neg', 'pos']),
+ 'text': Value('string')}
+ >>> ds = ds.cast_column('label', ClassLabel(names=['bad', 'good']))
+ >>> ds["train"].features
+ {'label': ClassLabel(names=['bad', 'good']),
+ 'text': Value('string')}
+ ```
+ """
+ self._check_values_type()
+ return DatasetDict({k: dataset.cast_column(column=column, feature=feature) for k, dataset in self.items()})
+
+ def remove_columns(self, column_names: Union[str, list[str]]) -> "DatasetDict":
+ """
+ Remove one or several column(s) from each split in the dataset
+ and the features associated to the column(s).
+
+ The transformation is applied to all the splits of the dataset dictionary.
+
+ You can also remove a column using [`~DatasetDict.map`] with `remove_columns` but the present method
+ doesn't copy the data of the remaining columns and is thus faster.
+
+ Args:
+ column_names (`Union[str, list[str]]`):
+ Name of the column(s) to remove.
+
+ Returns:
+ [`DatasetDict`]: A copy of the dataset object without the columns to remove.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes")
+ >>> ds = ds.remove_columns("label")
+ DatasetDict({
+ train: Dataset({
+ features: ['text'],
+ num_rows: 8530
+ })
+ validation: Dataset({
+ features: ['text'],
+ num_rows: 1066
+ })
+ test: Dataset({
+ features: ['text'],
+ num_rows: 1066
+ })
+ })
+ ```
+ """
+ self._check_values_type()
+ return DatasetDict({k: dataset.remove_columns(column_names=column_names) for k, dataset in self.items()})
+
+ def rename_column(self, original_column_name: str, new_column_name: str) -> "DatasetDict":
+ """
+ Rename a column in the dataset and move the features associated to the original column under the new column name.
+ The transformation is applied to all the datasets of the dataset dictionary.
+
+ You can also rename a column using [`~DatasetDict.map`] with `remove_columns` but the present method:
+ - takes care of moving the original features under the new column name.
+ - doesn't copy the data to a new dataset and is thus much faster.
+
+ Args:
+ original_column_name (`str`):
+ Name of the column to rename.
+ new_column_name (`str`):
+ New name for the column.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes")
+ >>> ds = ds.rename_column("label", "label_new")
+ DatasetDict({
+ train: Dataset({
+ features: ['text', 'label_new'],
+ num_rows: 8530
+ })
+ validation: Dataset({
+ features: ['text', 'label_new'],
+ num_rows: 1066
+ })
+ test: Dataset({
+ features: ['text', 'label_new'],
+ num_rows: 1066
+ })
+ })
+ ```
+ """
+ self._check_values_type()
+ return DatasetDict(
+ {
+ k: dataset.rename_column(
+ original_column_name=original_column_name,
+ new_column_name=new_column_name,
+ )
+ for k, dataset in self.items()
+ }
+ )
+
+ def rename_columns(self, column_mapping: dict[str, str]) -> "DatasetDict":
+ """
+ Rename several columns in the dataset, and move the features associated to the original columns under
+ the new column names.
+ The transformation is applied to all the datasets of the dataset dictionary.
+
+ Args:
+ column_mapping (`Dict[str, str]`):
+ A mapping of columns to rename to their new names.
+
+ Returns:
+ [`DatasetDict`]: A copy of the dataset with renamed columns.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes")
+ >>> ds.rename_columns({'text': 'text_new', 'label': 'label_new'})
+ DatasetDict({
+ train: Dataset({
+ features: ['text_new', 'label_new'],
+ num_rows: 8530
+ })
+ validation: Dataset({
+ features: ['text_new', 'label_new'],
+ num_rows: 1066
+ })
+ test: Dataset({
+ features: ['text_new', 'label_new'],
+ num_rows: 1066
+ })
+ })
+ ```
+ """
+ self._check_values_type()
+ return DatasetDict({k: dataset.rename_columns(column_mapping=column_mapping) for k, dataset in self.items()})
+
+ def select_columns(self, column_names: Union[str, list[str]]) -> "DatasetDict":
+ """Select one or several column(s) from each split in the dataset and
+ the features associated to the column(s).
+
+ The transformation is applied to all the splits of the dataset
+ dictionary.
+
+ Args:
+ column_names (`Union[str, list[str]]`):
+ Name of the column(s) to keep.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes")
+ >>> ds.select_columns("text")
+ DatasetDict({
+ train: Dataset({
+ features: ['text'],
+ num_rows: 8530
+ })
+ validation: Dataset({
+ features: ['text'],
+ num_rows: 1066
+ })
+ test: Dataset({
+ features: ['text'],
+ num_rows: 1066
+ })
+ })
+ ```
+ """
+ self._check_values_type()
+ return DatasetDict({k: dataset.select_columns(column_names=column_names) for k, dataset in self.items()})
+
+ def class_encode_column(self, column: str, include_nulls: bool = False) -> "DatasetDict":
+ """Casts the given column as [`~datasets.features.ClassLabel`] and updates the tables.
+
+ Args:
+ column (`str`):
+ The name of the column to cast.
+ include_nulls (`bool`, defaults to `False`):
+ Whether to include null values in the class labels. If `True`, the null values will be encoded as the `"None"` class label.
+
+
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("boolq")
+ >>> ds["train"].features
+ {'answer': Value('bool'),
+ 'passage': Value('string'),
+ 'question': Value('string')}
+ >>> ds = ds.class_encode_column("answer")
+ >>> ds["train"].features
+ {'answer': ClassLabel(num_classes=2, names=['False', 'True']),
+ 'passage': Value('string'),
+ 'question': Value('string')}
+ ```
+ """
+ self._check_values_type()
+ return DatasetDict(
+ {k: dataset.class_encode_column(column=column, include_nulls=include_nulls) for k, dataset in self.items()}
+ )
+
+ @contextlib.contextmanager
+ def formatted_as(
+ self,
+ type: Optional[str] = None,
+ columns: Optional[list] = None,
+ output_all_columns: bool = False,
+ **format_kwargs,
+ ):
+ """To be used in a `with` statement. Set `__getitem__` return format (type and columns).
+ The transformation is applied to all the datasets of the dataset dictionary.
+
+ Args:
+ type (`str`, *optional*):
+ Either output type selected in `[None, 'numpy', 'torch', 'tensorflow', 'jax', 'arrow', 'pandas', 'polars']`.
+ `None` means `__getitem__` returns python objects (default).
+ columns (`list[str]`, *optional*):
+ Columns to format in the output.
+ `None` means `__getitem__` returns all columns (default).
+ output_all_columns (`bool`, defaults to False):
+ Keep un-formatted columns as well in the output (as python objects).
+ **format_kwargs (additional keyword arguments):
+ Keywords arguments passed to the convert function like `np.array`, `torch.tensor` or `tensorflow.ragged.constant`.
+ """
+ self._check_values_type()
+ old_format_type = {k: dataset._format_type for k, dataset in self.items()}
+ old_format_kwargs = {k: dataset._format_kwargs for k, dataset in self.items()}
+ old_format_columns = {k: dataset._format_columns for k, dataset in self.items()}
+ old_output_all_columns = {k: dataset._output_all_columns for k, dataset in self.items()}
+ try:
+ self.set_format(type, columns, output_all_columns, **format_kwargs)
+ yield
+ finally:
+ for k, dataset in self.items():
+ dataset.set_format(
+ old_format_type[k],
+ old_format_columns[k],
+ old_output_all_columns[k],
+ **old_format_kwargs[k],
+ )
+
+ def set_format(
+ self,
+ type: Optional[str] = None,
+ columns: Optional[list] = None,
+ output_all_columns: bool = False,
+ **format_kwargs,
+ ):
+ """Set `__getitem__` return format (type and columns).
+ The format is set for every dataset in the dataset dictionary.
+
+ Args:
+ type (`str`, *optional*):
+ Either output type selected in `[None, 'numpy', 'torch', 'tensorflow', 'jax', 'arrow', 'pandas', 'polars']`.
+ `None` means `__getitem__` returns python objects (default).
+ columns (`list[str]`, *optional*):
+ Columns to format in the output.
+ `None` means `__getitem__` returns all columns (default).
+ output_all_columns (`bool`, defaults to False):
+ Keep un-formatted columns as well in the output (as python objects),
+ **format_kwargs (additional keyword arguments):
+ Keywords arguments passed to the convert function like `np.array`, `torch.tensor` or `tensorflow.ragged.constant`.
+
+ It is possible to call `map` after calling `set_format`. Since `map` may add new columns, then the list of formatted columns
+ gets updated. In this case, if you apply `map` on a dataset to add a new column, then this column will be formatted:
+
+ `new formatted columns = (all columns - previously unformatted columns)`
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> from transformers import AutoTokenizer
+ >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
+ >>> ds = ds.map(lambda x: tokenizer(x["text"], truncation=True, padding=True), batched=True)
+ >>> ds.set_format(type="numpy", columns=['input_ids', 'token_type_ids', 'attention_mask', 'label'])
+ >>> ds["train"].format
+ {'columns': ['input_ids', 'token_type_ids', 'attention_mask', 'label'],
+ 'format_kwargs': {},
+ 'output_all_columns': False,
+ 'type': 'numpy'}
+ ```
+ """
+ self._check_values_type()
+ for dataset in self.values():
+ dataset.set_format(
+ type=type,
+ columns=columns,
+ output_all_columns=output_all_columns,
+ **format_kwargs,
+ )
+
+ def reset_format(self):
+ """Reset `__getitem__` return format to python objects and all columns.
+ The transformation is applied to all the datasets of the dataset dictionary.
+
+ Same as `self.set_format()`
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> from transformers import AutoTokenizer
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes")
+ >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
+ >>> ds = ds.map(lambda x: tokenizer(x["text"], truncation=True, padding=True), batched=True)
+ >>> ds.set_format(type="numpy", columns=['input_ids', 'token_type_ids', 'attention_mask', 'label'])
+ >>> ds["train"].format
+ {'columns': ['input_ids', 'token_type_ids', 'attention_mask', 'label'],
+ 'format_kwargs': {},
+ 'output_all_columns': False,
+ 'type': 'numpy'}
+ >>> ds.reset_format()
+ >>> ds["train"].format
+ {'columns': ['text', 'label', 'input_ids', 'token_type_ids', 'attention_mask'],
+ 'format_kwargs': {},
+ 'output_all_columns': False,
+ 'type': None}
+ ```
+ """
+ self._check_values_type()
+ for dataset in self.values():
+ dataset.set_format()
+
+ def set_transform(
+ self,
+ transform: Optional[Callable],
+ columns: Optional[list] = None,
+ output_all_columns: bool = False,
+ ):
+ """Set ``__getitem__`` return format using this transform. The transform is applied on-the-fly on batches when ``__getitem__`` is called.
+ The transform is set for every dataset in the dataset dictionary
+ As :func:`datasets.Dataset.set_format`, this can be reset using :func:`datasets.Dataset.reset_format`
+
+ Args:
+ transform (`Callable`, optional): user-defined formatting transform, replaces the format defined by :func:`datasets.Dataset.set_format`
+ A formatting function is a callable that takes a batch (as a dict) as input and returns a batch.
+ This function is applied right before returning the objects in ``__getitem__``.
+ columns (`list[str]`, optional): columns to format in the output
+ If specified, then the input batch of the transform only contains those columns.
+ output_all_columns (`bool`, default to False): keep un-formatted columns as well in the output (as python objects)
+ If set to True, then the other un-formatted columns are kept with the output of the transform.
+
+ """
+ self._check_values_type()
+ for dataset in self.values():
+ dataset.set_format(
+ "custom",
+ columns=columns,
+ output_all_columns=output_all_columns,
+ transform=transform,
+ )
+
+ def with_format(
+ self,
+ type: Optional[str] = None,
+ columns: Optional[list] = None,
+ output_all_columns: bool = False,
+ **format_kwargs,
+ ) -> "DatasetDict":
+ """Set `__getitem__` return format (type and columns). The data formatting is applied on-the-fly.
+ The format `type` (for example "numpy") is used to format batches when using `__getitem__`.
+ The format is set for every dataset in the dataset dictionary.
+
+ It's also possible to use custom transforms for formatting using [`~datasets.Dataset.with_transform`].
+
+ Contrary to [`~datasets.DatasetDict.set_format`], `with_format` returns a new [`DatasetDict`] object with new [`Dataset`] objects.
+
+ Args:
+ type (`str`, *optional*):
+ Either output type selected in `[None, 'numpy', 'torch', 'tensorflow', 'jax', 'arrow', 'pandas', 'polars']`.
+ `None` means `__getitem__` returns python objects (default).
+ columns (`list[str]`, *optional*):
+ Columns to format in the output.
+ `None` means `__getitem__` returns all columns (default).
+ output_all_columns (`bool`, defaults to `False`):
+ Keep un-formatted columns as well in the output (as python objects).
+ **format_kwargs (additional keyword arguments):
+ Keywords arguments passed to the convert function like `np.array`, `torch.tensor` or `tensorflow.ragged.constant`.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> from transformers import AutoTokenizer
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes")
+ >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
+ >>> ds = ds.map(lambda x: tokenizer(x['text'], truncation=True, padding=True), batched=True)
+ >>> ds["train"].format
+ {'columns': ['text', 'label', 'input_ids', 'token_type_ids', 'attention_mask'],
+ 'format_kwargs': {},
+ 'output_all_columns': False,
+ 'type': None}
+ >>> ds = ds.with_format("torch")
+ >>> ds["train"].format
+ {'columns': ['text', 'label', 'input_ids', 'token_type_ids', 'attention_mask'],
+ 'format_kwargs': {},
+ 'output_all_columns': False,
+ 'type': 'torch'}
+ >>> ds["train"][0]
+ {'text': 'compassionately explores the seemingly irreconcilable situation between conservative christian parents and their estranged gay and lesbian children .',
+ 'label': tensor(1),
+ 'input_ids': tensor([ 101, 18027, 16310, 16001, 1103, 9321, 178, 11604, 7235, 6617,
+ 1742, 2165, 2820, 1206, 6588, 22572, 12937, 1811, 2153, 1105,
+ 1147, 12890, 19587, 6463, 1105, 15026, 1482, 119, 102, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0]),
+ 'token_type_ids': tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]),
+ 'attention_mask': tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])}
+ ```
+ """
+ dataset = copy.deepcopy(self)
+ dataset.set_format(
+ type=type,
+ columns=columns,
+ output_all_columns=output_all_columns,
+ **format_kwargs,
+ )
+ return dataset
+
+ def with_transform(
+ self,
+ transform: Optional[Callable],
+ columns: Optional[list] = None,
+ output_all_columns: bool = False,
+ ) -> "DatasetDict":
+ """Set `__getitem__` return format using this transform. The transform is applied on-the-fly on batches when `__getitem__` is called.
+ The transform is set for every dataset in the dataset dictionary
+
+ As [`~datasets.Dataset.set_format`], this can be reset using [`~datasets.Dataset.reset_format`].
+
+ Contrary to [`~datasets.DatasetDict.set_transform`], `with_transform` returns a new [`DatasetDict`] object with new [`Dataset`] objects.
+
+ Args:
+ transform (`Callable`, *optional*):
+ User-defined formatting transform, replaces the format defined by [`~datasets.Dataset.set_format`].
+ A formatting function is a callable that takes a batch (as a dict) as input and returns a batch.
+ This function is applied right before returning the objects in `__getitem__`.
+ columns (`list[str]`, *optional*):
+ Columns to format in the output.
+ If specified, then the input batch of the transform only contains those columns.
+ output_all_columns (`bool`, defaults to False):
+ Keep un-formatted columns as well in the output (as python objects).
+ If set to `True`, then the other un-formatted columns are kept with the output of the transform.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> from transformers import AutoTokenizer
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes")
+ >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
+ >>> def encode(example):
+ ... return tokenizer(example['text'], truncation=True, padding=True, return_tensors="pt")
+ >>> ds = ds.with_transform(encode)
+ >>> ds["train"][0]
+ {'attention_mask': tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1]),
+ 'input_ids': tensor([ 101, 1103, 2067, 1110, 17348, 1106, 1129, 1103, 6880, 1432,
+ 112, 188, 1207, 107, 14255, 1389, 107, 1105, 1115, 1119,
+ 112, 188, 1280, 1106, 1294, 170, 24194, 1256, 3407, 1190,
+ 170, 11791, 5253, 188, 1732, 7200, 10947, 12606, 2895, 117,
+ 179, 7766, 118, 172, 15554, 1181, 3498, 6961, 3263, 1137,
+ 188, 1566, 7912, 14516, 6997, 119, 102]),
+ 'token_type_ids': tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0])}
+ ```
+ """
+ dataset = copy.deepcopy(self)
+ dataset.set_transform(transform=transform, columns=columns, output_all_columns=output_all_columns)
+ return dataset
+
+ def map(
+ self,
+ function: Optional[Callable] = None,
+ with_indices: bool = False,
+ with_rank: bool = False,
+ with_split: bool = False,
+ input_columns: Optional[Union[str, list[str]]] = None,
+ batched: bool = False,
+ batch_size: Optional[int] = 1000,
+ drop_last_batch: bool = False,
+ remove_columns: Optional[Union[str, list[str]]] = None,
+ keep_in_memory: bool = False,
+ load_from_cache_file: Optional[bool] = None,
+ cache_file_names: Optional[dict[str, Optional[str]]] = None,
+ writer_batch_size: Optional[int] = 1000,
+ features: Optional[Features] = None,
+ disable_nullable: bool = False,
+ fn_kwargs: Optional[dict] = None,
+ num_proc: Optional[int] = None,
+ desc: Optional[str] = None,
+ try_original_type: Optional[bool] = True,
+ ) -> "DatasetDict":
+ """
+ Apply a function to all the examples in the table (individually or in batches) and update the table.
+ If your function returns a column that already exists, then it overwrites it.
+ The transformation is applied to all the datasets of the dataset dictionary.
+
+ You can specify whether the function should be batched or not with the `batched` parameter:
+
+ - If batched is `False`, then the function takes 1 example in and should return 1 example.
+ An example is a dictionary, e.g. `{"text": "Hello there !"}`.
+ - If batched is `True` and `batch_size` is 1, then the function takes a batch of 1 example as input and can return a batch with 1 or more examples.
+ A batch is a dictionary, e.g. a batch of 1 example is `{"text": ["Hello there !"]}`.
+ - If batched is `True` and `batch_size` is `n > 1`, then the function takes a batch of `n` examples as input and can return a batch with `n` examples, or with an arbitrary number of examples.
+ Note that the last batch may have less than `n` examples.
+ A batch is a dictionary, e.g. a batch of `n` examples is `{"text": ["Hello there !"] * n}`.
+
+ If the function is asynchronous, then `map` will run your function in parallel, with up to one thousand simultaneous calls.
+ It is recommended to use a `asyncio.Semaphore` in your function if you want to set a maximum number of operations that can run at the same time.
+
+ Args:
+ function (`callable`): with one of the following signature:
+ - `function(example: Dict[str, Any]) -> Dict[str, Any]` if `batched=False` and `with_indices=False`
+ - `function(example: Dict[str, Any], indices: int) -> Dict[str, Any]` if `batched=False` and `with_indices=True`
+ - `function(batch: Dict[str, list]) -> Dict[str, list]` if `batched=True` and `with_indices=False`
+ - `function(batch: Dict[str, list], indices: list[int]) -> Dict[str, list]` if `batched=True` and `with_indices=True`
+
+ For advanced usage, the function can also return a `pyarrow.Table`.
+ If the function is asynchronous, then `map` will run your function in parallel.
+ Moreover if your function returns nothing (`None`), then `map` will run your function and return the dataset unchanged.
+ If no function is provided, default to identity function: `lambda x: x`.
+ with_indices (`bool`, defaults to `False`):
+ Provide example indices to `function`. Note that in this case the signature of `function` should be `def function(example, idx): ...`.
+ with_rank (`bool`, defaults to `False`):
+ Provide process rank to `function`. Note that in this case the
+ signature of `function` should be `def function(example[, idx], rank): ...`.
+ with_split (`bool`, defaults to `False`):
+ Provide process split to `function`. Note that in this case the
+ signature of `function` should be `def function(example[, idx], split): ...`.
+ input_columns (`[Union[str, list[str]]]`, *optional*, defaults to `None`):
+ The columns to be passed into `function` as
+ positional arguments. If `None`, a dict mapping to all formatted columns is passed as one argument.
+ batched (`bool`, defaults to `False`):
+ Provide batch of examples to `function`.
+ batch_size (`int`, *optional*, defaults to `1000`):
+ Number of examples per batch provided to `function` if `batched=True`,
+ `batch_size <= 0` or `batch_size == None` then provide the full dataset as a single batch to `function`.
+ drop_last_batch (`bool`, defaults to `False`):
+ Whether a last batch smaller than the batch_size should be
+ dropped instead of being processed by the function.
+ remove_columns (`[Union[str, list[str]]]`, *optional*, defaults to `None`):
+ Remove a selection of columns while doing the mapping.
+ Columns will be removed before updating the examples with the output of `function`, i.e. if `function` is adding
+ columns with names in `remove_columns`, these columns will be kept.
+ keep_in_memory (`bool`, defaults to `False`):
+ Keep the dataset in memory instead of writing it to a cache file.
+ load_from_cache_file (`Optional[bool]`, defaults to `True` if caching is enabled):
+ If a cache file storing the current computation from `function`
+ can be identified, use it instead of recomputing.
+ cache_file_names (`[Dict[str, str]]`, *optional*, defaults to `None`):
+ Provide the name of a path for the cache file. It is used to store the
+ results of the computation instead of the automatically generated cache file name.
+ You have to provide one `cache_file_name` per dataset in the dataset dictionary.
+ writer_batch_size (`int`, default `1000`):
+ Number of rows per write operation for the cache file writer.
+ This value is a good trade-off between memory usage during the processing, and processing speed.
+ Higher value makes the processing do fewer lookups, lower value consume less temporary memory while running `map`.
+ features (`[datasets.Features]`, *optional*, defaults to `None`):
+ Use a specific [`Features`] to store the cache file
+ instead of the automatically generated one.
+ disable_nullable (`bool`, defaults to `False`):
+ Disallow null values in the table.
+ fn_kwargs (`Dict`, *optional*, defaults to `None`):
+ Keyword arguments to be passed to `function`
+ num_proc (`int`, *optional*, defaults to `None`):
+ The number of processes to use for multiprocessing.
+ - If `None` or `0`, no multiprocessing is used and the operation runs in the main process.
+ - If greater than `1`, one or multiple worker processes are used to process data in parallel.
+ Note: The function passed to `map()` must be picklable for multiprocessing to work correctly
+ (i.e., prefer functions defined at the top level of a module, not inside another function or class).
+ desc (`str`, *optional*, defaults to `None`):
+ Meaningful description to be displayed alongside with the progress bar while mapping examples.
+ try_original_type (`Optional[bool]`, defaults to `True`):
+ Try to keep the types of the original columns (e.g. int32 -> int32).
+ Set to False if you want to always infer new types.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes")
+ >>> def add_prefix(example):
+ ... example["text"] = "Review: " + example["text"]
+ ... return example
+ >>> ds = ds.map(add_prefix)
+ >>> ds["train"][0:3]["text"]
+ ['Review: the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .',
+ 'Review: the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson\'s expanded vision of j . r . r . tolkien\'s middle-earth .',
+ 'Review: effective but too-tepid biopic']
+
+ # process a batch of examples
+ >>> ds = ds.map(lambda example: tokenizer(example["text"]), batched=True)
+ # set number of processors
+ >>> ds = ds.map(add_prefix, num_proc=4)
+ ```
+ """
+ self._check_values_type()
+ if cache_file_names is None:
+ cache_file_names = dict.fromkeys(self)
+
+ dataset_dict = {}
+ for split, dataset in self.items():
+ if with_split:
+ function = bind(function, split)
+
+ dataset_dict[split] = dataset.map(
+ function=function,
+ with_indices=with_indices,
+ with_rank=with_rank,
+ input_columns=input_columns,
+ batched=batched,
+ batch_size=batch_size,
+ drop_last_batch=drop_last_batch,
+ remove_columns=remove_columns,
+ keep_in_memory=keep_in_memory,
+ load_from_cache_file=load_from_cache_file,
+ cache_file_name=cache_file_names[split],
+ writer_batch_size=writer_batch_size,
+ features=features,
+ disable_nullable=disable_nullable,
+ fn_kwargs=fn_kwargs,
+ num_proc=num_proc,
+ desc=desc,
+ try_original_type=try_original_type,
+ )
+
+ if with_split:
+ function = function.func
+
+ return DatasetDict(dataset_dict)
+
+ def filter(
+ self,
+ function: Optional[Callable] = None,
+ with_indices: bool = False,
+ with_rank: bool = False,
+ input_columns: Optional[Union[str, list[str]]] = None,
+ batched: bool = False,
+ batch_size: Optional[int] = 1000,
+ keep_in_memory: bool = False,
+ load_from_cache_file: Optional[bool] = None,
+ cache_file_names: Optional[dict[str, Optional[str]]] = None,
+ writer_batch_size: Optional[int] = 1000,
+ fn_kwargs: Optional[dict] = None,
+ num_proc: Optional[int] = None,
+ desc: Optional[str] = None,
+ ) -> "DatasetDict":
+ """Apply a filter function to all the elements in the table in batches
+ and update the table so that the dataset only includes examples according to the filter function.
+ The transformation is applied to all the datasets of the dataset dictionary.
+
+ Args:
+ function (`Callable`): Callable with one of the following signatures:
+
+ - `function(example: Dict[str, Any]) -> bool` if `batched=False` and `with_indices=False` and `with_rank=False`
+ - `function(example: Dict[str, Any], *extra_args) -> bool` if `batched=False` and `with_indices=True` and/or `with_rank=True` (one extra arg for each)
+ - `function(batch: Dict[str, list]) -> list[bool]` if `batched=True` and `with_indices=False` and `with_rank=False`
+ - `function(batch: Dict[str, list], *extra_args) -> list[bool]` if `batched=True` and `with_indices=True` and/or `with_rank=True` (one extra arg for each)
+
+ If no function is provided, defaults to an always `True` function: `lambda x: True`.
+ with_indices (`bool`, defaults to `False`):
+ Provide example indices to `function`. Note that in this case the
+ signature of `function` should be `def function(example, idx[, rank]): ...`.
+ with_rank (`bool`, defaults to `False`):
+ Provide process rank to `function`. Note that in this case the
+ signature of `function` should be `def function(example[, idx], rank): ...`.
+ input_columns (`[Union[str, list[str]]]`, *optional*, defaults to `None`):
+ The columns to be passed into `function` as
+ positional arguments. If `None`, a dict mapping to all formatted columns is passed as one argument.
+ batched (`bool`, defaults to `False`):
+ Provide batch of examples to `function`.
+ batch_size (`int`, *optional*, defaults to `1000`):
+ Number of examples per batch provided to `function` if `batched=True`
+ `batch_size <= 0` or `batch_size == None` then provide the full dataset as a single batch to `function`.
+ keep_in_memory (`bool`, defaults to `False`):
+ Keep the dataset in memory instead of writing it to a cache file.
+ load_from_cache_file (`Optional[bool]`, defaults to `True` if caching is enabled):
+ If a cache file storing the current computation from `function`
+ can be identified, use it instead of recomputing.
+ cache_file_names (`[Dict[str, str]]`, *optional*, defaults to `None`):
+ Provide the name of a path for the cache file. It is used to store the
+ results of the computation instead of the automatically generated cache file name.
+ You have to provide one `cache_file_name` per dataset in the dataset dictionary.
+ writer_batch_size (`int`, defaults to `1000`):
+ Number of rows per write operation for the cache file writer.
+ This value is a good trade-off between memory usage during the processing, and processing speed.
+ Higher value makes the processing do fewer lookups, lower value consume less temporary memory while running `map`.
+ fn_kwargs (`Dict`, *optional*, defaults to `None`):
+ Keyword arguments to be passed to `function`
+ num_proc (`int`, *optional*, defaults to `None`):
+ The number of processes to use for multiprocessing.
+ - If `None` or `0`, no multiprocessing is used and the operation runs in the main process.
+ - If greater than `1`, one or multiple worker processes are used to process data in parallel.
+ Note: The function passed to `map()` must be picklable for multiprocessing to work correctly
+ (i.e., prefer functions defined at the top level of a module, not inside another function or class).
+ desc (`str`, *optional*, defaults to `None`):
+ Meaningful description to be displayed alongside with the progress bar while filtering examples.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes")
+ >>> ds.filter(lambda x: x["label"] == 1)
+ DatasetDict({
+ train: Dataset({
+ features: ['text', 'label'],
+ num_rows: 4265
+ })
+ validation: Dataset({
+ features: ['text', 'label'],
+ num_rows: 533
+ })
+ test: Dataset({
+ features: ['text', 'label'],
+ num_rows: 533
+ })
+ })
+ ```
+ """
+ self._check_values_type()
+ if cache_file_names is None:
+ cache_file_names = dict.fromkeys(self)
+ return DatasetDict(
+ {
+ k: dataset.filter(
+ function=function,
+ with_indices=with_indices,
+ with_rank=with_rank,
+ input_columns=input_columns,
+ batched=batched,
+ batch_size=batch_size,
+ keep_in_memory=keep_in_memory,
+ load_from_cache_file=load_from_cache_file,
+ cache_file_name=cache_file_names[k],
+ writer_batch_size=writer_batch_size,
+ fn_kwargs=fn_kwargs,
+ num_proc=num_proc,
+ desc=desc,
+ )
+ for k, dataset in self.items()
+ }
+ )
+
+ def flatten_indices(
+ self,
+ keep_in_memory: bool = False,
+ cache_file_names: Optional[dict[str, Optional[str]]] = None,
+ writer_batch_size: Optional[int] = 1000,
+ features: Optional[Features] = None,
+ disable_nullable: bool = False,
+ num_proc: Optional[int] = None,
+ new_fingerprint: Optional[str] = None,
+ ) -> "DatasetDict":
+ """Create and cache a new Dataset by flattening the indices mapping.
+
+ Args:
+ keep_in_memory (`bool`, defaults to `False`):
+ Keep the dataset in memory instead of writing it to a cache file.
+ cache_file_names (`Dict[str, str]`, *optional*, default `None`):
+ Provide the name of a path for the cache file. It is used to store the
+ results of the computation instead of the automatically generated cache file name.
+ You have to provide one `cache_file_name` per dataset in the dataset dictionary.
+ writer_batch_size (`int`, defaults to `1000`):
+ Number of rows per write operation for the cache file writer.
+ This value is a good trade-off between memory usage during the processing, and processing speed.
+ Higher value makes the processing do fewer lookups, lower value consume less temporary memory while running `map`.
+ features (`Optional[datasets.Features]`, defaults to `None`):
+ Use a specific [`Features`] to store the cache file
+ instead of the automatically generated one.
+ disable_nullable (`bool`, defaults to `False`):
+ Allow null values in the table.
+ num_proc (`int`, optional, default `None`):
+ Max number of processes when generating cache. Already cached shards are loaded sequentially
+ new_fingerprint (`str`, *optional*, defaults to `None`):
+ The new fingerprint of the dataset after transform.
+ If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments
+ """
+ self._check_values_type()
+ if cache_file_names is None:
+ cache_file_names = dict.fromkeys(self)
+ return DatasetDict(
+ {
+ k: dataset.flatten_indices(
+ keep_in_memory=keep_in_memory,
+ cache_file_name=cache_file_names[k],
+ writer_batch_size=writer_batch_size,
+ features=features,
+ disable_nullable=disable_nullable,
+ num_proc=num_proc,
+ new_fingerprint=new_fingerprint,
+ )
+ for k, dataset in self.items()
+ }
+ )
+
+ def sort(
+ self,
+ column_names: Union[str, Sequence[str]],
+ reverse: Union[bool, Sequence[bool]] = False,
+ null_placement: str = "at_end",
+ keep_in_memory: bool = False,
+ load_from_cache_file: Optional[bool] = None,
+ indices_cache_file_names: Optional[dict[str, Optional[str]]] = None,
+ writer_batch_size: Optional[int] = 1000,
+ ) -> "DatasetDict":
+ """Create a new dataset sorted according to a single or multiple columns.
+
+ Args:
+ column_names (`Union[str, Sequence[str]]`):
+ Column name(s) to sort by.
+ reverse (`Union[bool, Sequence[bool]]`, defaults to `False`):
+ If `True`, sort by descending order rather than ascending. If a single bool is provided,
+ the value is applied to the sorting of all column names. Otherwise a list of bools with the
+ same length and order as column_names must be provided.
+ null_placement (`str`, defaults to `at_end`):
+ Put `None` values at the beginning if `at_start` or `first` or at the end if `at_end` or `last`
+ keep_in_memory (`bool`, defaults to `False`):
+ Keep the sorted indices in memory instead of writing it to a cache file.
+ load_from_cache_file (`Optional[bool]`, defaults to `True` if caching is enabled):
+ If a cache file storing the sorted indices
+ can be identified, use it instead of recomputing.
+ indices_cache_file_names (`[Dict[str, str]]`, *optional*, defaults to `None`):
+ Provide the name of a path for the cache file. It is used to store the
+ indices mapping instead of the automatically generated cache file name.
+ You have to provide one `cache_file_name` per dataset in the dataset dictionary.
+ writer_batch_size (`int`, defaults to `1000`):
+ Number of rows per write operation for the cache file writer.
+ Higher value gives smaller cache files, lower value consume less temporary memory.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset('cornell-movie-review-data/rotten_tomatoes')
+ >>> ds['train']['label'][:10]
+ [1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
+ >>> sorted_ds = ds.sort('label')
+ >>> sorted_ds['train']['label'][:10]
+ [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
+ >>> another_sorted_ds = ds.sort(['label', 'text'], reverse=[True, False])
+ >>> another_sorted_ds['train']['label'][:10]
+ [1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
+ ```
+ """
+ self._check_values_type()
+ if indices_cache_file_names is None:
+ indices_cache_file_names = dict.fromkeys(self)
+ return DatasetDict(
+ {
+ k: dataset.sort(
+ column_names=column_names,
+ reverse=reverse,
+ null_placement=null_placement,
+ keep_in_memory=keep_in_memory,
+ load_from_cache_file=load_from_cache_file,
+ indices_cache_file_name=indices_cache_file_names[k],
+ writer_batch_size=writer_batch_size,
+ )
+ for k, dataset in self.items()
+ }
+ )
+
+ def shuffle(
+ self,
+ seeds: Optional[Union[int, dict[str, Optional[int]]]] = None,
+ seed: Optional[int] = None,
+ generators: Optional[dict[str, np.random.Generator]] = None,
+ keep_in_memory: bool = False,
+ load_from_cache_file: Optional[bool] = None,
+ indices_cache_file_names: Optional[dict[str, Optional[str]]] = None,
+ writer_batch_size: Optional[int] = 1000,
+ ) -> "DatasetDict":
+ """Create a new Dataset where the rows are shuffled.
+
+ The transformation is applied to all the datasets of the dataset dictionary.
+
+ Currently shuffling uses numpy random generators.
+ You can either supply a NumPy BitGenerator to use, or a seed to initiate NumPy's default random generator (PCG64).
+
+ Args:
+ seeds (`Dict[str, int]` or `int`, *optional*):
+ A seed to initialize the default BitGenerator if `generator=None`.
+ If `None`, then fresh, unpredictable entropy will be pulled from the OS.
+ If an `int` or `array_like[ints]` is passed, then it will be passed to SeedSequence to derive the initial BitGenerator state.
+ You can provide one `seed` per dataset in the dataset dictionary.
+ seed (`int`, *optional*):
+ A seed to initialize the default BitGenerator if `generator=None`. Alias for seeds (a `ValueError` is raised if both are provided).
+ generators (`Dict[str, *optional*, np.random.Generator]`):
+ Numpy random Generator to use to compute the permutation of the dataset rows.
+ If `generator=None` (default), uses `np.random.default_rng` (the default BitGenerator (PCG64) of NumPy).
+ You have to provide one `generator` per dataset in the dataset dictionary.
+ keep_in_memory (`bool`, defaults to `False`):
+ Keep the dataset in memory instead of writing it to a cache file.
+ load_from_cache_file (`Optional[bool]`, defaults to `True` if caching is enabled):
+ If a cache file storing the current computation from `function`
+ can be identified, use it instead of recomputing.
+ indices_cache_file_names (`Dict[str, str]`, *optional*):
+ Provide the name of a path for the cache file. It is used to store the
+ indices mappings instead of the automatically generated cache file name.
+ You have to provide one `cache_file_name` per dataset in the dataset dictionary.
+ writer_batch_size (`int`, defaults to `1000`):
+ Number of rows per write operation for the cache file writer.
+ This value is a good trade-off between memory usage during the processing, and processing speed.
+ Higher value makes the processing do fewer lookups, lower value consume less temporary memory while running `map`.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes")
+ >>> ds["train"]["label"][:10]
+ [1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
+
+ # set a seed
+ >>> shuffled_ds = ds.shuffle(seed=42)
+ >>> shuffled_ds["train"]["label"][:10]
+ [0, 1, 0, 1, 0, 0, 0, 0, 0, 0]
+ ```
+ """
+ self._check_values_type()
+ if seed is not None and seeds is not None:
+ raise ValueError("Please specify seed or seeds, but not both")
+ seeds = seed if seed is not None else seeds
+ if seeds is None:
+ seeds = dict.fromkeys(self)
+ elif not isinstance(seeds, dict):
+ seeds = dict.fromkeys(self, seeds)
+ if generators is None:
+ generators = dict.fromkeys(self)
+ if indices_cache_file_names is None:
+ indices_cache_file_names = dict.fromkeys(self)
+ return DatasetDict(
+ {
+ k: dataset.shuffle(
+ seed=seeds[k],
+ generator=generators[k],
+ keep_in_memory=keep_in_memory,
+ load_from_cache_file=load_from_cache_file,
+ indices_cache_file_name=indices_cache_file_names[k],
+ writer_batch_size=writer_batch_size,
+ )
+ for k, dataset in self.items()
+ }
+ )
+
+ def save_to_disk(
+ self,
+ dataset_dict_path: PathLike,
+ max_shard_size: Optional[Union[str, int]] = None,
+ num_shards: Optional[dict[str, int]] = None,
+ num_proc: Optional[int] = None,
+ storage_options: Optional[dict] = None,
+ ):
+ """
+ Saves a dataset dict to a filesystem using `fsspec.spec.AbstractFileSystem`.
+
+ For [`Image`], [`Audio`] and [`Video`] data:
+
+ All the Image(), Audio() and Video() data are stored in the arrow files.
+ If you want to store paths or urls, please use the Value("string") type.
+
+ Args:
+ dataset_dict_path (`path-like`):
+ Path (e.g. `dataset/train`) or remote URI (e.g. `s3://my-bucket/dataset/train`)
+ of the dataset dict directory where the dataset dict will be saved to.
+ max_shard_size (`int` or `str`, *optional*, defaults to `"500MB"`):
+ The maximum size of the dataset shards to be saved to the filesystem. If expressed as a string, needs to be digits followed by a unit
+ (like `"50MB"`).
+ num_shards (`Dict[str, int]`, *optional*):
+ Number of shards to write. By default the number of shards depends on `max_shard_size` and `num_proc`.
+ You need to provide the number of shards for each dataset in the dataset dictionary.
+ Use a dictionary to define a different num_shards for each split.
+
+
+ num_proc (`int`, *optional*, default `None`):
+ Number of processes when downloading and generating the dataset locally.
+ Multiprocessing is disabled by default.
+
+
+ storage_options (`dict`, *optional*):
+ Key/value pairs to be passed on to the file-system backend, if any.
+
+
+
+ Example:
+
+ ```python
+ >>> dataset_dict.save_to_disk("path/to/dataset/directory")
+ >>> dataset_dict.save_to_disk("path/to/dataset/directory", max_shard_size="1GB")
+ >>> dataset_dict.save_to_disk("path/to/dataset/directory", num_shards={"train": 1024, "test": 8})
+ ```
+ """
+ fs: fsspec.AbstractFileSystem
+ fs, _ = url_to_fs(dataset_dict_path, **(storage_options or {}))
+
+ if num_shards is None:
+ num_shards = dict.fromkeys(self)
+ elif not isinstance(num_shards, dict):
+ raise ValueError(
+ "Please provide one `num_shards` per dataset in the dataset dictionary, e.g. {{'train': 128, 'test': 4}}"
+ )
+
+ fs.makedirs(dataset_dict_path, exist_ok=True)
+
+ with fs.open(
+ posixpath.join(dataset_dict_path, config.DATASETDICT_JSON_FILENAME),
+ "w",
+ encoding="utf-8",
+ ) as f:
+ json.dump({"splits": list(self)}, f)
+ for k, dataset in self.items():
+ dataset.save_to_disk(
+ posixpath.join(dataset_dict_path, k),
+ num_shards=num_shards.get(k),
+ max_shard_size=max_shard_size,
+ num_proc=num_proc,
+ storage_options=storage_options,
+ )
+
+ @staticmethod
+ def load_from_disk(
+ dataset_dict_path: PathLike,
+ keep_in_memory: Optional[bool] = None,
+ storage_options: Optional[dict] = None,
+ ) -> "DatasetDict":
+ """
+ Load a dataset that was previously saved using [`save_to_disk`] from a filesystem using `fsspec.spec.AbstractFileSystem`.
+
+ Args:
+ dataset_dict_path (`path-like`):
+ Path (e.g. `"dataset/train"`) or remote URI (e.g. `"s3//my-bucket/dataset/train"`)
+ of the dataset dict directory where the dataset dict will be loaded from.
+ keep_in_memory (`bool`, defaults to `None`):
+ Whether to copy the dataset in-memory. If `None`, the
+ dataset will not be copied in-memory unless explicitly enabled by setting
+ `datasets.config.IN_MEMORY_MAX_SIZE` to nonzero. See more details in the
+ [improve performance](../cache#improve-performance) section.
+ storage_options (`dict`, *optional*):
+ Key/value pairs to be passed on to the file-system backend, if any.
+
+
+
+ Returns:
+ [`DatasetDict`]
+
+ Example:
+
+ ```py
+ >>> ds = load_from_disk('path/to/dataset/directory')
+ ```
+ """
+ fs: fsspec.AbstractFileSystem
+ fs, dataset_dict_path = url_to_fs(dataset_dict_path, **(storage_options or {}))
+
+ dataset_dict_json_path = posixpath.join(dataset_dict_path, config.DATASETDICT_JSON_FILENAME)
+ dataset_state_json_path = posixpath.join(dataset_dict_path, config.DATASET_STATE_JSON_FILENAME)
+ dataset_info_path = posixpath.join(dataset_dict_path, config.DATASET_INFO_FILENAME)
+ if not fs.isfile(dataset_dict_json_path):
+ if fs.isfile(dataset_info_path) and fs.isfile(dataset_state_json_path):
+ raise FileNotFoundError(
+ f"No such file: '{dataset_dict_json_path}'. Expected to load a `DatasetDict` object, but got a `Dataset`. Please use either `datasets.load_from_disk` or `Dataset.load_from_disk` instead."
+ )
+ raise FileNotFoundError(
+ f"No such file: '{dataset_dict_json_path}'. Expected to load a `DatasetDict` object, but provided path is not a `DatasetDict`."
+ )
+
+ with fs.open(dataset_dict_json_path, "r", encoding="utf-8") as f:
+ splits = json.load(f)["splits"]
+
+ dataset_dict = DatasetDict()
+ for k in splits:
+ dataset_dict_split_path = posixpath.join(fs.unstrip_protocol(dataset_dict_path), k)
+ dataset_dict[k] = Dataset.load_from_disk(
+ dataset_dict_split_path,
+ keep_in_memory=keep_in_memory,
+ storage_options=storage_options,
+ )
+ return dataset_dict
+
+ @staticmethod
+ def from_csv(
+ path_or_paths: dict[str, PathLike],
+ features: Optional[Features] = None,
+ cache_dir: str = None,
+ keep_in_memory: bool = False,
+ **kwargs,
+ ) -> "DatasetDict":
+ """Create [`DatasetDict`] from CSV file(s).
+
+ Args:
+ path_or_paths (`dict` of path-like):
+ Path(s) of the CSV file(s).
+ features ([`Features`], *optional*):
+ Dataset features.
+ cache_dir (str, *optional*, defaults to `"~/.cache/huggingface/datasets"`):
+ Directory to cache data.
+ keep_in_memory (`bool`, defaults to `False`):
+ Whether to copy the data in-memory.
+ **kwargs (additional keyword arguments):
+ Keyword arguments to be passed to [`pandas.read_csv`].
+
+ Returns:
+ [`DatasetDict`]
+
+ Example:
+
+ ```py
+ >>> from datasets import DatasetDict
+ >>> ds = DatasetDict.from_csv({'train': 'path/to/dataset.csv'})
+ ```
+ """
+ # Dynamic import to avoid circular dependency
+ from .io.csv import CsvDatasetReader
+
+ return CsvDatasetReader(
+ path_or_paths,
+ features=features,
+ cache_dir=cache_dir,
+ keep_in_memory=keep_in_memory,
+ **kwargs,
+ ).read()
+
+ @staticmethod
+ def from_json(
+ path_or_paths: dict[str, PathLike],
+ features: Optional[Features] = None,
+ cache_dir: str = None,
+ keep_in_memory: bool = False,
+ **kwargs,
+ ) -> "DatasetDict":
+ """Create [`DatasetDict`] from JSON Lines file(s).
+
+ Args:
+ path_or_paths (`path-like` or list of `path-like`):
+ Path(s) of the JSON Lines file(s).
+ features ([`Features`], *optional*):
+ Dataset features.
+ cache_dir (str, *optional*, defaults to `"~/.cache/huggingface/datasets"`):
+ Directory to cache data.
+ keep_in_memory (`bool`, defaults to `False`):
+ Whether to copy the data in-memory.
+ **kwargs (additional keyword arguments):
+ Keyword arguments to be passed to [`JsonConfig`].
+
+ Returns:
+ [`DatasetDict`]
+
+ Example:
+
+ ```py
+ >>> from datasets import DatasetDict
+ >>> ds = DatasetDict.from_json({'train': 'path/to/dataset.json'})
+ ```
+ """
+ # Dynamic import to avoid circular dependency
+ from .io.json import JsonDatasetReader
+
+ return JsonDatasetReader(
+ path_or_paths,
+ features=features,
+ cache_dir=cache_dir,
+ keep_in_memory=keep_in_memory,
+ **kwargs,
+ ).read()
+
+ @staticmethod
+ def from_parquet(
+ path_or_paths: dict[str, PathLike],
+ features: Optional[Features] = None,
+ cache_dir: str = None,
+ keep_in_memory: bool = False,
+ columns: Optional[list[str]] = None,
+ **kwargs,
+ ) -> "DatasetDict":
+ """Create [`DatasetDict`] from Parquet file(s).
+
+ Args:
+ path_or_paths (`dict` of path-like):
+ Path(s) of the CSV file(s).
+ features ([`Features`], *optional*):
+ Dataset features.
+ cache_dir (`str`, *optional*, defaults to `"~/.cache/huggingface/datasets"`):
+ Directory to cache data.
+ keep_in_memory (`bool`, defaults to `False`):
+ Whether to copy the data in-memory.
+ columns (`list[str]`, *optional*):
+ If not `None`, only these columns will be read from the file.
+ A column name may be a prefix of a nested field, e.g. 'a' will select
+ 'a.b', 'a.c', and 'a.d.e'.
+ **kwargs (additional keyword arguments):
+ Keyword arguments to be passed to [`ParquetConfig`].
+
+ Returns:
+ [`DatasetDict`]
+
+ Example:
+
+ ```py
+ >>> from datasets import DatasetDict
+ >>> ds = DatasetDict.from_parquet({'train': 'path/to/dataset/parquet'})
+ ```
+ """
+ # Dynamic import to avoid circular dependency
+ from .io.parquet import ParquetDatasetReader
+
+ return ParquetDatasetReader(
+ path_or_paths,
+ features=features,
+ cache_dir=cache_dir,
+ keep_in_memory=keep_in_memory,
+ columns=columns,
+ **kwargs,
+ ).read()
+
+ @staticmethod
+ def from_text(
+ path_or_paths: dict[str, PathLike],
+ features: Optional[Features] = None,
+ cache_dir: str = None,
+ keep_in_memory: bool = False,
+ **kwargs,
+ ) -> "DatasetDict":
+ """Create [`DatasetDict`] from text file(s).
+
+ Args:
+ path_or_paths (`dict` of path-like):
+ Path(s) of the text file(s).
+ features ([`Features`], *optional*):
+ Dataset features.
+ cache_dir (`str`, *optional*, defaults to `"~/.cache/huggingface/datasets"`):
+ Directory to cache data.
+ keep_in_memory (`bool`, defaults to `False`):
+ Whether to copy the data in-memory.
+ **kwargs (additional keyword arguments):
+ Keyword arguments to be passed to [`TextConfig`].
+
+ Returns:
+ [`DatasetDict`]
+
+ Example:
+
+ ```py
+ >>> from datasets import DatasetDict
+ >>> ds = DatasetDict.from_text({'train': 'path/to/dataset.txt'})
+ ```
+ """
+ # Dynamic import to avoid circular dependency
+ from .io.text import TextDatasetReader
+
+ return TextDatasetReader(
+ path_or_paths,
+ features=features,
+ cache_dir=cache_dir,
+ keep_in_memory=keep_in_memory,
+ **kwargs,
+ ).read()
+
+ @is_documented_by(Dataset.align_labels_with_mapping)
+ def align_labels_with_mapping(self, label2id: dict, label_column: str) -> "DatasetDict":
+ self._check_values_type()
+ return DatasetDict(
+ {
+ k: dataset.align_labels_with_mapping(label2id=label2id, label_column=label_column)
+ for k, dataset in self.items()
+ }
+ )
+
+ def push_to_hub(
+ self,
+ repo_id,
+ config_name: str = "default",
+ set_default: Optional[bool] = None,
+ data_dir: Optional[str] = None,
+ commit_message: Optional[str] = None,
+ commit_description: Optional[str] = None,
+ private: Optional[bool] = None,
+ token: Optional[str] = None,
+ revision: Optional[str] = None,
+ create_pr: Optional[bool] = False,
+ max_shard_size: Optional[Union[int, str]] = None,
+ num_shards: Optional[dict[str, int]] = None,
+ embed_external_files: bool = True,
+ num_proc: Optional[int] = None,
+ ) -> CommitInfo:
+ """Pushes the [`DatasetDict`] to the hub as a Parquet dataset.
+ The [`DatasetDict`] is pushed using HTTP requests and does not need to have neither git or git-lfs installed.
+
+ Each dataset split will be pushed independently. The pushed dataset will keep the original split names.
+
+ The resulting Parquet files are self-contained by default: if your dataset contains [`Image`] or [`Audio`]
+ data, the Parquet files will store the bytes of your images or audio files.
+ You can disable this by setting `embed_external_files` to False.
+
+ Args:
+ repo_id (`str`):
+ The ID of the repository to push to in the following format: `/` or
+ `/`. Also accepts ``, which will default to the namespace
+ of the logged-in user.
+ config_name (`str`):
+ Configuration name of a dataset. Defaults to "default".
+ set_default (`bool`, *optional*):
+ Whether to set this configuration as the default one. Otherwise, the default configuration is the one
+ named "default".
+ data_dir (`str`, *optional*):
+ Directory name that will contain the uploaded data files. Defaults to the `config_name` if different
+ from "default", else "data".
+
+
+ commit_message (`str`, *optional*):
+ Message to commit while pushing. Will default to `"Upload dataset"`.
+ commit_description (`str`, *optional*):
+ Description of the commit that will be created.
+ Additionally, description of the PR if a PR is created (`create_pr` is True).
+
+
+ private (`bool`, *optional*):
+ Whether to make the repo private. If `None` (default), the repo will be public unless the
+ organization's default is private. This value is ignored if the repo already exists.
+ token (`str`, *optional*):
+ An optional authentication token for the Hugging Face Hub. If no token is passed, will default
+ to the token saved locally when logging in with `huggingface-cli login`. Will raise an error
+ if no token is passed and the user is not logged-in.
+ revision (`str`, *optional*):
+ Branch to push the uploaded files to. Defaults to the `"main"` branch.
+
+
+ create_pr (`bool`, *optional*, defaults to `False`):
+ Whether to create a PR with the uploaded files or directly commit.
+
+
+ max_shard_size (`int` or `str`, *optional*, defaults to `"500MB"`):
+ The maximum size of the dataset shards to be uploaded to the hub. If expressed as a string, needs to be digits followed by a unit
+ (like `"500MB"` or `"1GB"`).
+ num_shards (`Dict[str, int]`, *optional*):
+ Number of shards to write. By default, the number of shards depends on `max_shard_size`.
+ Use a dictionary to define a different num_shards for each split.
+
+
+ embed_external_files (`bool`, defaults to `True`):
+ Whether to embed file bytes in the shards.
+ In particular, this will do the following before the push for the fields of type:
+
+ - [`Audio`] and [`Image`] removes local path information and embed file content in the Parquet files.
+ num_proc (`int`, *optional*, defaults to `None`):
+ Number of processes when preparing and uploading the dataset.
+ This is helpful if the dataset is made of many samples or media files to embed.
+ Multiprocessing is disabled by default.
+
+
+
+ Return:
+ huggingface_hub.CommitInfo
+
+ Example:
+
+ ```python
+ >>> dataset_dict.push_to_hub("/")
+ >>> dataset_dict.push_to_hub("/", private=True)
+ >>> dataset_dict.push_to_hub("/", max_shard_size="1GB")
+ >>> dataset_dict.push_to_hub("/", num_shards={"train": 1024, "test": 8})
+ ```
+
+ If you want to add a new configuration (or subset) to a dataset (e.g. if the dataset has multiple tasks/versions/languages):
+
+ ```python
+ >>> english_dataset.push_to_hub("/", "en")
+ >>> french_dataset.push_to_hub("/", "fr")
+ >>> # later
+ >>> english_dataset = load_dataset("/", "en")
+ >>> french_dataset = load_dataset("/", "fr")
+ ```
+ """
+ if num_shards is None:
+ num_shards = dict.fromkeys(self)
+ elif not isinstance(num_shards, dict):
+ raise ValueError(
+ "Please provide one `num_shards` per dataset in the dataset dictionary, e.g. {{'train': 128, 'test': 4}}"
+ )
+
+ self._check_values_type()
+ self._check_values_features()
+ total_uploaded_size = 0
+ total_dataset_nbytes = 0
+ info_to_dump: DatasetInfo = next(iter(self.values())).info.copy()
+ info_to_dump.config_name = config_name
+ info_to_dump.splits = SplitDict()
+
+ for split in self.keys():
+ if not re.match(_split_re, split):
+ raise ValueError(f"Split name should match '{_split_re}' but got '{split}'.")
+
+ api = HfApi(endpoint=config.HF_ENDPOINT, token=token)
+
+ try:
+ repo_id = api.repo_info(repo_id, repo_type="dataset").id
+ except RepositoryNotFoundError:
+ repo_url = api.create_repo(
+ repo_id,
+ repo_type="dataset",
+ private=private,
+ exist_ok=True,
+ )
+ repo_id = repo_url.repo_id
+
+ if revision is not None and not revision.startswith("refs/pr/"):
+ # We do not call create_branch for a PR reference: 400 Bad Request
+ api.create_branch(
+ repo_id,
+ branch=revision,
+ token=token,
+ repo_type="dataset",
+ exist_ok=True,
+ )
+
+ if not data_dir:
+ data_dir = config_name if config_name != "default" else "data" # for backward compatibility
+
+ additions = []
+ for split in self.keys():
+ logger.info(f"Pushing split {split} to the Hub.")
+ # The split=key needs to be removed before merging
+ split_additions, uploaded_size, dataset_nbytes = self[split]._push_parquet_shards_to_hub(
+ repo_id,
+ data_dir=data_dir,
+ split=split,
+ token=token,
+ revision=revision,
+ create_pr=create_pr,
+ max_shard_size=max_shard_size,
+ num_shards=num_shards.get(split),
+ embed_external_files=embed_external_files,
+ num_proc=num_proc,
+ )
+ additions += split_additions
+ total_uploaded_size += uploaded_size
+ total_dataset_nbytes += dataset_nbytes
+ info_to_dump.splits[split] = SplitInfo(str(split), num_bytes=dataset_nbytes, num_examples=len(self[split]))
+ info_to_dump.download_checksums = None
+ info_to_dump.download_size = total_uploaded_size
+ info_to_dump.dataset_size = total_dataset_nbytes
+ info_to_dump.size_in_bytes = total_uploaded_size + total_dataset_nbytes
+
+ def get_deletions_and_dataset_card() -> tuple[str, list[CommitOperationDelete], str, Optional[str]]:
+ parent_commit = api.repo_info(repo_id, repo_type="dataset", revision=revision).sha
+
+ # Check if the repo already has a README.md and/or a dataset_infos.json to update them with the new split info (size and pattern)
+ # and delete old split shards (if they exist)
+ repo_with_dataset_card, repo_with_dataset_infos = False, False
+ repo_splits: list[str] = [] # use a list to keep the order of the splits
+ deletions: list[CommitOperationDelete] = []
+ repo_files_to_add = [addition.path_in_repo for addition in additions]
+ for repo_file in api.list_repo_tree(
+ repo_id=repo_id,
+ revision=parent_commit,
+ repo_type="dataset",
+ token=token,
+ recursive=True,
+ ):
+ if not isinstance(repo_file, RepoFile):
+ continue
+ if repo_file.rfilename == config.REPOCARD_FILENAME:
+ repo_with_dataset_card = True
+ elif repo_file.rfilename == config.DATASETDICT_INFOS_FILENAME:
+ repo_with_dataset_infos = True
+ elif (
+ repo_file.rfilename.startswith(tuple(f"{data_dir}/{split}-" for split in self.keys()))
+ and repo_file.rfilename not in repo_files_to_add
+ ):
+ deletions.append(CommitOperationDelete(path_in_repo=repo_file.rfilename))
+ elif fnmatch.fnmatch(
+ repo_file.rfilename,
+ PUSH_TO_HUB_WITHOUT_METADATA_CONFIGS_SPLIT_PATTERN_SHARDED.replace("{split}", "*"),
+ ):
+ pattern = glob_pattern_to_regex(PUSH_TO_HUB_WITHOUT_METADATA_CONFIGS_SPLIT_PATTERN_SHARDED)
+ split_pattern_fields = string_to_dict(repo_file.rfilename, pattern)
+ assert split_pattern_fields is not None
+ repo_split = split_pattern_fields["split"]
+ if repo_split not in repo_splits:
+ repo_splits.append(repo_split)
+
+ # get the info from the README to update them
+ if repo_with_dataset_card:
+ dataset_card_path = api.hf_hub_download(
+ repo_id,
+ config.REPOCARD_FILENAME,
+ repo_type="dataset",
+ revision=parent_commit,
+ )
+ dataset_card = DatasetCard.load(Path(dataset_card_path))
+ dataset_card_data = dataset_card.data
+ metadata_configs = MetadataConfigs.from_dataset_card_data(dataset_card_data)
+ # get the deprecated dataset_infos.json to update them
+ elif repo_with_dataset_infos:
+ dataset_card = None
+ dataset_card_data = DatasetCardData()
+ metadata_configs = MetadataConfigs()
+ else:
+ dataset_card = None
+ dataset_card_data = DatasetCardData()
+ metadata_configs = MetadataConfigs()
+ # create the metadata configs if it was uploaded with push_to_hub before metadata configs existed
+ if not metadata_configs and repo_splits:
+ default_metadata_configs_to_dump = {
+ "data_files": [{"split": split, "path": f"data/{split}-*"} for split in repo_splits]
+ }
+ MetadataConfigs({"default": default_metadata_configs_to_dump}).to_dataset_card_data(dataset_card_data)
+ metadata_config_to_dump = {
+ "data_files": [{"split": split, "path": f"{data_dir}/{split}-*"} for split in self.keys()],
+ }
+ configs_to_dump = {config_name: metadata_config_to_dump}
+ if set_default and config_name != "default":
+ if metadata_configs:
+ current_default_config_name = metadata_configs.get_default_config_name()
+ if current_default_config_name == "default":
+ raise ValueError(
+ "There exists a configuration named 'default'. To set a different configuration as default, "
+ "rename the 'default' one first."
+ )
+ if current_default_config_name:
+ _ = metadata_configs[current_default_config_name].pop("default")
+ configs_to_dump[current_default_config_name] = metadata_configs[current_default_config_name]
+ metadata_config_to_dump["default"] = True
+ # push to the deprecated dataset_infos.json
+ if repo_with_dataset_infos:
+ dataset_infos_path = api.hf_hub_download(
+ repo_id,
+ config.DATASETDICT_INFOS_FILENAME,
+ repo_type="dataset",
+ revision=parent_commit,
+ )
+ with open(dataset_infos_path, encoding="utf-8") as f:
+ dataset_infos: dict = json.load(f)
+ dataset_infos[config_name] = asdict(info_to_dump)
+ new_dataset_infos = json.dumps(dataset_infos, indent=4)
+ else:
+ new_dataset_infos = None
+ # push to README
+ DatasetInfosDict({config_name: info_to_dump}).to_dataset_card_data(dataset_card_data)
+ MetadataConfigs(configs_to_dump).to_dataset_card_data(dataset_card_data)
+ new_dataset_card = (
+ DatasetCard(f"---\n{dataset_card_data}\n---\n") if dataset_card is None else dataset_card
+ )
+ return parent_commit, deletions, new_dataset_card, new_dataset_infos
+
+ commit_message = commit_message if commit_message is not None else "Upload dataset"
+ if len(additions) > config.UPLOADS_MAX_NUMBER_PER_COMMIT:
+ logger.info(
+ f"Number of files to upload is larger than {config.UPLOADS_MAX_NUMBER_PER_COMMIT}. Splitting the push into multiple commits."
+ )
+ num_commits = math.ceil(len(additions) / config.UPLOADS_MAX_NUMBER_PER_COMMIT)
+ for i in range(0, num_commits):
+ operations = additions[
+ i * config.UPLOADS_MAX_NUMBER_PER_COMMIT : (i + 1) * config.UPLOADS_MAX_NUMBER_PER_COMMIT
+ ]
+ for retry, sleep_time in enumerate(itertools.chain(range(10), itertools.repeat(30)), start=1):
+ # We need to retry if another commit happens at the same time
+ sleep_time *= 1 + random.random()
+ try:
+ commit_info = api.create_commit(
+ repo_id,
+ operations=operations,
+ commit_message=commit_message + f" (part {i:05d}-of-{num_commits:05d})",
+ commit_description=commit_description,
+ repo_type="dataset",
+ revision=revision,
+ create_pr=create_pr,
+ )
+ except HfHubHTTPError as err:
+ if (
+ err.__context__
+ and isinstance(err.__context__, HfHubHTTPError)
+ and err.__context__.response.status_code == 409
+ ):
+ # 409 is Conflict (another commit is in progress)
+ time.sleep(sleep_time)
+ logger.info(
+ f"Retrying intermediate commit for {repo_id}, {config_name} ({retry}/n with status_code {err.__context__.response.status_code})"
+ )
+ continue
+ else:
+ raise
+ break
+ logger.info(
+ f"Commit #{i + 1} completed"
+ + (f" (still {num_commits - i - 1} to go)" if num_commits - i - 1 else "")
+ + "."
+ )
+ last_commit_additions = []
+ else:
+ last_commit_additions = additions
+
+ for retry, sleep_time in enumerate(itertools.chain(range(10), itertools.repeat(30)), start=1):
+ # We need to retry if there was a commit in between in case it touched the dataset card data
+ sleep_time *= 1 + random.random()
+ parent_commit, deletions, dataset_card, dataset_infos = get_deletions_and_dataset_card()
+ dataset_card_additions = []
+ if dataset_infos:
+ dataset_card_additions.append(
+ CommitOperationAdd(
+ path_in_repo=config.DATASETDICT_INFOS_FILENAME,
+ path_or_fileobj=dataset_infos.encode("utf-8"),
+ )
+ )
+ dataset_card_additions.append(
+ CommitOperationAdd(path_in_repo=config.REPOCARD_FILENAME, path_or_fileobj=str(dataset_card).encode())
+ )
+ try:
+ commit_info = api.create_commit(
+ repo_id,
+ operations=last_commit_additions + dataset_card_additions + deletions,
+ commit_message=commit_message,
+ commit_description=commit_description,
+ repo_type="dataset",
+ revision=revision,
+ create_pr=create_pr,
+ parent_commit=parent_commit,
+ )
+ except HfHubHTTPError as err:
+ if (
+ err.__context__
+ and isinstance(err.__context__, HfHubHTTPError)
+ and err.__context__.response.status_code in (412, 409)
+ ):
+ # 412 is Precondition failed (parent_commit isn't satisfied)
+ # 409 is Conflict (another commit is in progress)
+ time.sleep(sleep_time)
+ logger.info(
+ f"Retrying commit for {repo_id}, {config_name} ({retry}/n with status_code {err.__context__.response.status_code})"
+ )
+ continue
+ else:
+ raise
+ break
+
+ return commit_info
+
+
+class IterableDatasetDict(dict[Union[str, NamedSplit], IterableDataset]):
+ def _check_values_type(self):
+ for dataset in self.values():
+ if not isinstance(dataset, IterableDataset):
+ raise TypeError(f"Values in `DatasetDict` should be of type `Dataset` but got type '{type(dataset)}'")
+
+ def _check_values_features(self):
+ items = [(key, dataset._resolve_features()) for key, dataset in self.items()]
+ for item_a, item_b in zip(items[:-1], items[1:]):
+ if item_a[1].features != item_b[1].features:
+ raise ValueError(
+ f"All datasets in `DatasetDict` should have the same features but features for '{item_a[0]}' and '{item_b[0]}' don't match: {item_a[1].features} != {item_b[1].features}"
+ )
+
+ def __repr__(self):
+ repr = "\n".join([f"{k}: {v}" for k, v in self.items()])
+ repr = re.sub(r"^", " " * 4, repr, count=0, flags=re.M)
+ return f"IterableDatasetDict({{\n{repr}\n}})"
+
+ @property
+ def num_columns(self) -> dict[str, Optional[int]]:
+ """Number of columns in each split of the dataset.
+ This can contain None valies if some splits have unknown features (e.g. after a map() operation).
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes")
+ >>> ds.num_columns
+ {'test': 2, 'train': 2, 'validation': 2}
+ ```
+ """
+ self._check_values_type()
+ return {k: dataset.num_columns for k, dataset in self.items()}
+
+ @property
+ def column_names(self) -> dict[str, Optional[list[str]]]:
+ """Names of the columns in each split of the dataset.
+ This can contain None valies if some splits have unknown features (e.g. after a map() operation).
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes")
+ >>> ds.column_names
+ {'test': ['text', 'label'],
+ 'train': ['text', 'label'],
+ 'validation': ['text', 'label']}
+ ```
+ """
+ self._check_values_type()
+ return {k: dataset.column_names for k, dataset in self.items()}
+
+ def with_format(
+ self,
+ type: Optional[str] = None,
+ ) -> "IterableDatasetDict":
+ """
+ Return a dataset with the specified format.
+
+ Args:
+
+ type (`str`, *optional*):
+ Either output type selected in `[None, 'numpy', 'torch', 'tensorflow', 'jax', 'arrow', 'pandas', 'polars']`.
+ `None` means it returns python objects (default).
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> from transformers import AutoTokenizer
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation", streaming=True)
+ >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
+ >>> ds = ds.map(lambda x: tokenizer(x['text'], truncation=True, padding=True), batched=True)
+ >>> ds = ds.with_format("torch")
+ >>> next(iter(ds))
+ {'text': 'compassionately explores the seemingly irreconcilable situation between conservative christian parents and their estranged gay and lesbian children .',
+ 'label': tensor(1),
+ 'input_ids': tensor([ 101, 18027, 16310, 16001, 1103, 9321, 178, 11604, 7235, 6617,
+ 1742, 2165, 2820, 1206, 6588, 22572, 12937, 1811, 2153, 1105,
+ 1147, 12890, 19587, 6463, 1105, 15026, 1482, 119, 102, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0]),
+ 'token_type_ids': tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]),
+ 'attention_mask': tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])}
+ ```
+ """
+ return IterableDatasetDict({k: dataset.with_format(type=type) for k, dataset in self.items()})
+
+ def map(
+ self,
+ function: Optional[Callable] = None,
+ with_indices: bool = False,
+ with_split: bool = False,
+ input_columns: Optional[Union[str, list[str]]] = None,
+ batched: bool = False,
+ batch_size: int = 1000,
+ drop_last_batch: bool = False,
+ remove_columns: Optional[Union[str, list[str]]] = None,
+ fn_kwargs: Optional[dict] = None,
+ ) -> "IterableDatasetDict":
+ """
+ Apply a function to all the examples in the iterable dataset (individually or in batches) and update them.
+ If your function returns a column that already exists, then it overwrites it.
+ The function is applied on-the-fly on the examples when iterating over the dataset.
+ The transformation is applied to all the datasets of the dataset dictionary.
+
+ You can specify whether the function should be batched or not with the `batched` parameter:
+
+ - If batched is `False`, then the function takes 1 example in and should return 1 example.
+ An example is a dictionary, e.g. `{"text": "Hello there !"}`.
+ - If batched is `True` and `batch_size` is 1, then the function takes a batch of 1 example as input and can return a batch with 1 or more examples.
+ A batch is a dictionary, e.g. a batch of 1 example is `{"text": ["Hello there !"]}`.
+ - If batched is `True` and `batch_size` is `n` > 1, then the function takes a batch of `n` examples as input and can return a batch with `n` examples, or with an arbitrary number of examples.
+ Note that the last batch may have less than `n` examples.
+ A batch is a dictionary, e.g. a batch of `n` examples is `{"text": ["Hello there !"] * n}`.
+
+ If the function is asynchronous, then `map` will run your function in parallel, with up to one thousand simultaneous calls.
+ It is recommended to use a `asyncio.Semaphore` in your function if you want to set a maximum number of operations that can run at the same time.
+
+ Args:
+ function (`Callable`, *optional*, defaults to `None`):
+ Function applied on-the-fly on the examples when you iterate on the dataset.
+ It must have one of the following signatures:
+
+ - `function(example: Dict[str, Any]) -> Dict[str, Any]` if `batched=False` and `with_indices=False`
+ - `function(example: Dict[str, Any], idx: int) -> Dict[str, Any]` if `batched=False` and `with_indices=True`
+ - `function(batch: Dict[str, list]) -> Dict[str, list]` if `batched=True` and `with_indices=False`
+ - `function(batch: Dict[str, list], indices: list[int]) -> Dict[str, list]` if `batched=True` and `with_indices=True`
+
+ For advanced usage, the function can also return a `pyarrow.Table`.
+ If the function is asynchronous, then `map` will run your function in parallel.
+ Moreover if your function returns nothing (`None`), then `map` will run your function and return the dataset unchanged.
+ If no function is provided, default to identity function: `lambda x: x`.
+ with_indices (`bool`, defaults to `False`):
+ Provide example indices to `function`. Note that in this case the signature of `function` should be `def function(example, idx[, rank]): ...`.
+ input_columns (`[Union[str, list[str]]]`, *optional*, defaults to `None`):
+ The columns to be passed into `function`
+ as positional arguments. If `None`, a dict mapping to all formatted columns is passed as one argument.
+ batched (`bool`, defaults to `False`):
+ Provide batch of examples to `function`.
+ batch_size (`int`, *optional*, defaults to `1000`):
+ Number of examples per batch provided to `function` if `batched=True`.
+ drop_last_batch (`bool`, defaults to `False`):
+ Whether a last batch smaller than the `batch_size` should be
+ dropped instead of being processed by the function.
+ remove_columns (`[list[str]]`, *optional*, defaults to `None`):
+ Remove a selection of columns while doing the mapping.
+ Columns will be removed before updating the examples with the output of `function`, i.e. if `function` is adding
+ columns with names in `remove_columns`, these columns will be kept.
+ fn_kwargs (`Dict`, *optional*, defaults to `None`):
+ Keyword arguments to be passed to `function`
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", streaming=True)
+ >>> def add_prefix(example):
+ ... example["text"] = "Review: " + example["text"]
+ ... return example
+ >>> ds = ds.map(add_prefix)
+ >>> next(iter(ds["train"]))
+ {'label': 1,
+ 'text': 'Review: the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'}
+ ```
+ """
+
+ dataset_dict = {}
+ for split, dataset in self.items():
+ if with_split:
+ function = bind(function, split)
+
+ dataset_dict[split] = dataset.map(
+ function=function,
+ with_indices=with_indices,
+ input_columns=input_columns,
+ batched=batched,
+ batch_size=batch_size,
+ drop_last_batch=drop_last_batch,
+ remove_columns=remove_columns,
+ fn_kwargs=fn_kwargs,
+ )
+
+ if with_split:
+ function = function.func
+
+ return IterableDatasetDict(dataset_dict)
+
+ def filter(
+ self,
+ function: Optional[Callable] = None,
+ with_indices=False,
+ input_columns: Optional[Union[str, list[str]]] = None,
+ batched: bool = False,
+ batch_size: Optional[int] = 1000,
+ fn_kwargs: Optional[dict] = None,
+ ) -> "IterableDatasetDict":
+ """Apply a filter function to all the elements so that the dataset only includes examples according to the filter function.
+ The filtering is done on-the-fly when iterating over the dataset.
+ The filtering is applied to all the datasets of the dataset dictionary.
+
+ Args:
+ function (`Callable`):
+ Callable with one of the following signatures:
+
+ - `function(example: Dict[str, Any]) -> bool` if `with_indices=False, batched=False`
+ - `function(example: Dict[str, Any], indices: int) -> bool` if `with_indices=True, batched=False`
+ - `function(example: Dict[str, list]) -> list[bool]` if `with_indices=False, batched=True`
+ - `function(example: Dict[str, list], indices: list[int]) -> list[bool]` if `with_indices=True, batched=True`
+
+ If no function is provided, defaults to an always True function: `lambda x: True`.
+ with_indices (`bool`, defaults to `False`):
+ Provide example indices to `function`. Note that in this case the signature of `function` should be `def function(example, idx): ...`.
+ input_columns (`str` or `list[str]`, *optional*):
+ The columns to be passed into `function` as
+ positional arguments. If `None`, a dict mapping to all formatted columns is passed as one argument.
+ batched (`bool`, defaults to `False`):
+ Provide batch of examples to `function`
+ batch_size (`int`, *optional*, defaults to `1000`):
+ Number of examples per batch provided to `function` if `batched=True`.
+ fn_kwargs (`Dict`, *optional*, defaults to `None`):
+ Keyword arguments to be passed to `function`
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", streaming=True)
+ >>> ds = ds.filter(lambda x: x["label"] == 0)
+ >>> list(ds["train"].take(3))
+ [{'label': 0, 'text': 'Review: simplistic , silly and tedious .'},
+ {'label': 0,
+ 'text': "Review: it's so laddish and juvenile , only teenage boys could possibly find it funny ."},
+ {'label': 0,
+ 'text': 'Review: exploitative and largely devoid of the depth or sophistication that would make watching such a graphic treatment of the crimes bearable .'}]
+ ```
+ """
+ return IterableDatasetDict(
+ {
+ k: dataset.filter(
+ function=function,
+ with_indices=with_indices,
+ input_columns=input_columns,
+ batched=batched,
+ batch_size=batch_size,
+ fn_kwargs=fn_kwargs,
+ )
+ for k, dataset in self.items()
+ }
+ )
+
+ def shuffle(
+ self,
+ seed=None,
+ generator: Optional[np.random.Generator] = None,
+ buffer_size: int = 1000,
+ ) -> "IterableDatasetDict":
+ """
+ Randomly shuffles the elements of this dataset.
+ The shuffling is applied to all the datasets of the dataset dictionary.
+
+ This dataset fills a buffer with buffer_size elements, then randomly samples elements from this buffer,
+ replacing the selected elements with new elements. For perfect shuffling, a buffer size greater than or
+ equal to the full size of the dataset is required.
+
+ For instance, if your dataset contains 10,000 elements but `buffer_size` is set to 1000, then `shuffle` will
+ initially select a random element from only the first 1000 elements in the buffer. Once an element is
+ selected, its space in the buffer is replaced by the next (i.e. 1,001-st) element,
+ maintaining the 1000 element buffer.
+
+ If the dataset is made of several shards, it also does `shuffle` the order of the shards.
+ However if the order has been fixed by using [`~datasets.IterableDataset.skip`] or [`~datasets.IterableDataset.take`]
+ then the order of the shards is kept unchanged.
+
+ Args:
+ seed (`int`, *optional*, defaults to `None`):
+ Random seed that will be used to shuffle the dataset.
+ It is used to sample from the shuffle buffer and also to shuffle the data shards.
+ generator (`numpy.random.Generator`, *optional*):
+ Numpy random Generator to use to compute the permutation of the dataset rows.
+ If `generator=None` (default), uses `np.random.default_rng` (the default BitGenerator (PCG64) of NumPy).
+ buffer_size (`int`, defaults to `1000`):
+ Size of the buffer.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", streaming=True)
+ >>> list(ds["train"].take(3))
+ [{'label': 1,
+ 'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'},
+ {'label': 1,
+ 'text': 'the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson\'s expanded vision of j . r . r . tolkien\'s middle-earth .'},
+ {'label': 1, 'text': 'effective but too-tepid biopic'}]
+ >>> ds = ds.shuffle(seed=42)
+ >>> list(ds["train"].take(3))
+ [{'label': 1,
+ 'text': "a sports movie with action that's exciting on the field and a story you care about off it ."},
+ {'label': 1,
+ 'text': 'at its best , the good girl is a refreshingly adult take on adultery . . .'},
+ {'label': 1,
+ 'text': "sam jones became a very lucky filmmaker the day wilco got dropped from their record label , proving that one man's ruin may be another's fortune ."}]
+ ```
+ """
+ return IterableDatasetDict(
+ {
+ k: dataset.shuffle(seed=seed, generator=generator, buffer_size=buffer_size)
+ for k, dataset in self.items()
+ }
+ )
+
+ def rename_column(self, original_column_name: str, new_column_name: str) -> "IterableDatasetDict":
+ """
+ Rename a column in the dataset, and move the features associated to the original column under the new column
+ name.
+ The renaming is applied to all the datasets of the dataset dictionary.
+
+ Args:
+ original_column_name (`str`):
+ Name of the column to rename.
+ new_column_name (`str`):
+ New name for the column.
+
+ Returns:
+ [`IterableDatasetDict`]: A copy of the dataset with a renamed column.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", streaming=True)
+ >>> ds = ds.rename_column("text", "movie_review")
+ >>> next(iter(ds["train"]))
+ {'label': 1,
+ 'movie_review': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'}
+ ```
+ """
+ return IterableDatasetDict(
+ {
+ k: dataset.rename_column(
+ original_column_name=original_column_name,
+ new_column_name=new_column_name,
+ )
+ for k, dataset in self.items()
+ }
+ )
+
+ def rename_columns(self, column_mapping: dict[str, str]) -> "IterableDatasetDict":
+ """
+ Rename several columns in the dataset, and move the features associated to the original columns under
+ the new column names.
+ The renaming is applied to all the datasets of the dataset dictionary.
+
+ Args:
+ column_mapping (`Dict[str, str]`):
+ A mapping of columns to rename to their new names.
+
+ Returns:
+ [`IterableDatasetDict`]: A copy of the dataset with renamed columns
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", streaming=True)
+ >>> ds = ds.rename_columns({"text": "movie_review", "label": "rating"})
+ >>> next(iter(ds["train"]))
+ {'movie_review': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .',
+ 'rating': 1}
+ ```
+ """
+ return IterableDatasetDict(
+ {k: dataset.rename_columns(column_mapping=column_mapping) for k, dataset in self.items()}
+ )
+
+ def remove_columns(self, column_names: Union[str, list[str]]) -> "IterableDatasetDict":
+ """
+ Remove one or several column(s) in the dataset and the features associated to them.
+ The removal is done on-the-fly on the examples when iterating over the dataset.
+ The removal is applied to all the datasets of the dataset dictionary.
+
+
+ Args:
+ column_names (`Union[str, list[str]]`):
+ Name of the column(s) to remove.
+
+ Returns:
+ [`IterableDatasetDict`]: A copy of the dataset object without the columns to remove.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", streaming=True)
+ >>> ds = ds.remove_columns("label")
+ >>> next(iter(ds["train"]))
+ {'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'}
+ ```
+ """
+ return IterableDatasetDict({k: dataset.remove_columns(column_names) for k, dataset in self.items()})
+
+ def select_columns(self, column_names: Union[str, list[str]]) -> "IterableDatasetDict":
+ """Select one or several column(s) in the dataset and the features
+ associated to them. The selection is done on-the-fly on the examples
+ when iterating over the dataset. The selection is applied to all the
+ datasets of the dataset dictionary.
+
+
+ Args:
+ column_names (`Union[str, list[str]]`):
+ Name of the column(s) to keep.
+
+ Returns:
+ [`IterableDatasetDict`]: A copy of the dataset object with only selected columns.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", streaming=True)
+ >>> ds = ds.select("text")
+ >>> next(iter(ds["train"]))
+ {'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'}
+ ```
+ """
+ return IterableDatasetDict({k: dataset.select_columns(column_names) for k, dataset in self.items()})
+
+ def cast_column(self, column: str, feature: FeatureType) -> "IterableDatasetDict":
+ """Cast column to feature for decoding.
+ The type casting is applied to all the datasets of the dataset dictionary.
+
+ Args:
+ column (`str`):
+ Column name.
+ feature ([`Feature`]):
+ Target feature.
+
+ Returns:
+ [`IterableDatasetDict`]
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset, ClassLabel
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", streaming=True)
+ >>> ds["train"].features
+ {'label': ClassLabel(names=['neg', 'pos']),
+ 'text': Value('string')}
+ >>> ds = ds.cast_column('label', ClassLabel(names=['bad', 'good']))
+ >>> ds["train"].features
+ {'label': ClassLabel(names=['bad', 'good']),
+ 'text': Value('string')}
+ ```
+ """
+ return IterableDatasetDict(
+ {k: dataset.cast_column(column=column, feature=feature) for k, dataset in self.items()}
+ )
+
+ def cast(
+ self,
+ features: Features,
+ ) -> "IterableDatasetDict":
+ """
+ Cast the dataset to a new set of features.
+ The type casting is applied to all the datasets of the dataset dictionary.
+
+ Args:
+ features (`Features`):
+ New features to cast the dataset to.
+ The name of the fields in the features must match the current column names.
+ The type of the data must also be convertible from one type to the other.
+ For non-trivial conversion, e.g. `string` <-> `ClassLabel` you should use [`map`] to update the Dataset.
+
+ Returns:
+ [`IterableDatasetDict`]: A copy of the dataset with casted features.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", streaming=True)
+ >>> ds["train"].features
+ {'label': ClassLabel(names=['neg', 'pos']),
+ 'text': Value('string')}
+ >>> new_features = ds["train"].features.copy()
+ >>> new_features['label'] = ClassLabel(names=['bad', 'good'])
+ >>> new_features['text'] = Value('large_string')
+ >>> ds = ds.cast(new_features)
+ >>> ds["train"].features
+ {'label': ClassLabel(names=['bad', 'good']),
+ 'text': Value('large_string')}
+ ```
+ """
+ return IterableDatasetDict({k: dataset.cast(features=features) for k, dataset in self.items()})
+
+ def push_to_hub(
+ self,
+ repo_id,
+ config_name: str = "default",
+ set_default: Optional[bool] = None,
+ data_dir: Optional[str] = None,
+ commit_message: Optional[str] = None,
+ commit_description: Optional[str] = None,
+ private: Optional[bool] = None,
+ token: Optional[str] = None,
+ revision: Optional[str] = None,
+ create_pr: Optional[bool] = False,
+ # max_shard_size: Optional[Union[int, str]] = None, # TODO(QL): add arg
+ num_shards: Optional[dict[str, int]] = None,
+ embed_external_files: bool = True,
+ num_proc: Optional[int] = None,
+ ) -> CommitInfo:
+ """Pushes the [`DatasetDict`] to the hub as a Parquet dataset.
+ The [`DatasetDict`] is pushed using HTTP requests and does not need to have neither git or git-lfs installed.
+
+ Each dataset split will be pushed independently. The pushed dataset will keep the original split names.
+
+ The resulting Parquet files are self-contained by default: if your dataset contains [`Image`] or [`Audio`]
+ data, the Parquet files will store the bytes of your images or audio files.
+ You can disable this by setting `embed_external_files` to False.
+
+ Args:
+ repo_id (`str`):
+ The ID of the repository to push to in the following format: `/` or
+ `/`. Also accepts ``, which will default to the namespace
+ of the logged-in user.
+ config_name (`str`):
+ Configuration name of a dataset. Defaults to "default".
+ set_default (`bool`, *optional*):
+ Whether to set this configuration as the default one. Otherwise, the default configuration is the one
+ named "default".
+ data_dir (`str`, *optional*):
+ Directory name that will contain the uploaded data files. Defaults to the `config_name` if different
+ from "default", else "data".
+
+
+ commit_message (`str`, *optional*):
+ Message to commit while pushing. Will default to `"Upload dataset"`.
+ commit_description (`str`, *optional*):
+ Description of the commit that will be created.
+ Additionally, description of the PR if a PR is created (`create_pr` is True).
+
+
+ private (`bool`, *optional*):
+ Whether to make the repo private. If `None` (default), the repo will be public unless the
+ organization's default is private. This value is ignored if the repo already exists.
+ token (`str`, *optional*):
+ An optional authentication token for the Hugging Face Hub. If no token is passed, will default
+ to the token saved locally when logging in with `huggingface-cli login`. Will raise an error
+ if no token is passed and the user is not logged-in.
+ revision (`str`, *optional*):
+ Branch to push the uploaded files to. Defaults to the `"main"` branch.
+ create_pr (`bool`, *optional*, defaults to `False`):
+ Whether to create a PR with the uploaded files or directly commit.
+ num_shards (`Dict[str, int]`, *optional*):
+ Number of shards to write. Equals to this dataset's `.num_shards` by default.
+ Use a dictionary to define a different num_shards for each split.
+ embed_external_files (`bool`, defaults to `True`):
+ Whether to embed file bytes in the shards.
+ In particular, this will do the following before the push for the fields of type:
+
+ - [`Audio`] and [`Image`] removes local path information and embed file content in the Parquet files.
+ num_proc (`int`, *optional*, defaults to `None`):
+ Number of processes when preparing and uploading the dataset.
+ This is helpful if the dataset is made of many samples or media files to embed.
+ Multiprocessing is disabled by default.
+
+
+
+ Return:
+ huggingface_hub.CommitInfo
+
+ Example:
+
+ ```python
+ >>> dataset_dict.push_to_hub("/")
+ >>> dataset_dict.push_to_hub("/", private=True)
+ >>> dataset_dict.push_to_hub("/", num_shards={"train": 1024, "test": 8})
+ ```
+
+ If you want to add a new configuration (or subset) to a dataset (e.g. if the dataset has multiple tasks/versions/languages):
+
+ ```python
+ >>> english_dataset.push_to_hub("/", "en")
+ >>> french_dataset.push_to_hub("/", "fr")
+ >>> # later
+ >>> english_dataset = load_dataset("/", "en")
+ >>> french_dataset = load_dataset("/", "fr")
+ ```
+ """
+ if num_shards is None:
+ num_shards = dict.fromkeys(self)
+ elif not isinstance(num_shards, dict):
+ raise ValueError(
+ "Please provide one `num_shards` per dataset in the dataset dictionary, e.g. {{'train': 128, 'test': 4}}"
+ )
+
+ self._check_values_type()
+ self._check_values_features()
+ total_uploaded_size = 0
+ total_dataset_nbytes = 0
+ info_to_dump: DatasetInfo = next(iter(self.values())).info.copy()
+ info_to_dump.config_name = config_name
+ info_to_dump.splits = SplitDict()
+
+ for split in self.keys():
+ if not re.match(_split_re, split):
+ raise ValueError(f"Split name should match '{_split_re}' but got '{split}'.")
+
+ api = HfApi(endpoint=config.HF_ENDPOINT, token=token)
+
+ try:
+ repo_id = api.repo_info(repo_id, repo_type="dataset").id
+ except RepositoryNotFoundError:
+ repo_url = api.create_repo(
+ repo_id,
+ repo_type="dataset",
+ private=private,
+ exist_ok=True,
+ )
+ repo_id = repo_url.repo_id
+
+ if revision is not None and not revision.startswith("refs/pr/"):
+ # We do not call create_branch for a PR reference: 400 Bad Request
+ api.create_branch(
+ repo_id,
+ branch=revision,
+ token=token,
+ repo_type="dataset",
+ exist_ok=True,
+ )
+
+ if not data_dir:
+ data_dir = config_name if config_name != "default" else "data" # for backward compatibility
+
+ additions = []
+ for split in self.keys():
+ logger.info(f"Pushing split {split} to the Hub.")
+ # The split=key needs to be removed before merging
+ split_additions, uploaded_size, dataset_nbytes, num_examples = self[split]._push_parquet_shards_to_hub(
+ repo_id,
+ data_dir=data_dir,
+ split=split,
+ token=token,
+ revision=revision,
+ create_pr=create_pr,
+ # max_shard_size=max_shard_size, # TODO(QL): add arg
+ num_shards=num_shards.get(split),
+ embed_external_files=embed_external_files,
+ num_proc=num_proc,
+ )
+ additions += split_additions
+ total_uploaded_size += uploaded_size
+ total_dataset_nbytes += dataset_nbytes
+ info_to_dump.splits[split] = SplitInfo(str(split), num_bytes=dataset_nbytes, num_examples=num_examples)
+ info_to_dump.download_checksums = None
+ info_to_dump.download_size = total_uploaded_size
+ info_to_dump.dataset_size = total_dataset_nbytes
+ info_to_dump.size_in_bytes = total_uploaded_size + total_dataset_nbytes
+
+ def get_deletions_and_dataset_card() -> tuple[str, list[CommitOperationDelete], str, Optional[str]]:
+ parent_commit = api.repo_info(repo_id, repo_type="dataset", revision=revision).sha
+
+ # Check if the repo already has a README.md and/or a dataset_infos.json to update them with the new split info (size and pattern)
+ # and delete old split shards (if they exist)
+ repo_with_dataset_card, repo_with_dataset_infos = False, False
+ repo_splits: list[str] = [] # use a list to keep the order of the splits
+ deletions: list[CommitOperationDelete] = []
+ repo_files_to_add = [addition.path_in_repo for addition in additions]
+ for repo_file in api.list_repo_tree(
+ repo_id=repo_id,
+ revision=parent_commit,
+ repo_type="dataset",
+ token=token,
+ recursive=True,
+ ):
+ if not isinstance(repo_file, RepoFile):
+ continue
+ if repo_file.rfilename == config.REPOCARD_FILENAME:
+ repo_with_dataset_card = True
+ elif repo_file.rfilename == config.DATASETDICT_INFOS_FILENAME:
+ repo_with_dataset_infos = True
+ elif (
+ repo_file.rfilename.startswith(tuple(f"{data_dir}/{split}-" for split in self.keys()))
+ and repo_file.rfilename not in repo_files_to_add
+ ):
+ deletions.append(CommitOperationDelete(path_in_repo=repo_file.rfilename))
+ elif fnmatch.fnmatch(
+ repo_file.rfilename,
+ PUSH_TO_HUB_WITHOUT_METADATA_CONFIGS_SPLIT_PATTERN_SHARDED.replace("{split}", "*"),
+ ):
+ pattern = glob_pattern_to_regex(PUSH_TO_HUB_WITHOUT_METADATA_CONFIGS_SPLIT_PATTERN_SHARDED)
+ split_pattern_fields = string_to_dict(repo_file.rfilename, pattern)
+ assert split_pattern_fields is not None
+ repo_split = split_pattern_fields["split"]
+ if repo_split not in repo_splits:
+ repo_splits.append(repo_split)
+
+ # get the info from the README to update them
+ if repo_with_dataset_card:
+ dataset_card_path = api.hf_hub_download(
+ repo_id,
+ config.REPOCARD_FILENAME,
+ repo_type="dataset",
+ revision=parent_commit,
+ )
+ dataset_card = DatasetCard.load(Path(dataset_card_path))
+ dataset_card_data = dataset_card.data
+ metadata_configs = MetadataConfigs.from_dataset_card_data(dataset_card_data)
+ # get the deprecated dataset_infos.json to update them
+ elif repo_with_dataset_infos:
+ dataset_card = None
+ dataset_card_data = DatasetCardData()
+ metadata_configs = MetadataConfigs()
+ else:
+ dataset_card = None
+ dataset_card_data = DatasetCardData()
+ metadata_configs = MetadataConfigs()
+ # create the metadata configs if it was uploaded with push_to_hub before metadata configs existed
+ if not metadata_configs and repo_splits:
+ default_metadata_configs_to_dump = {
+ "data_files": [{"split": split, "path": f"data/{split}-*"} for split in repo_splits]
+ }
+ MetadataConfigs({"default": default_metadata_configs_to_dump}).to_dataset_card_data(dataset_card_data)
+ metadata_config_to_dump = {
+ "data_files": [{"split": split, "path": f"{data_dir}/{split}-*"} for split in self.keys()],
+ }
+ configs_to_dump = {config_name: metadata_config_to_dump}
+ if set_default and config_name != "default":
+ if metadata_configs:
+ current_default_config_name = metadata_configs.get_default_config_name()
+ if current_default_config_name == "default":
+ raise ValueError(
+ "There exists a configuration named 'default'. To set a different configuration as default, "
+ "rename the 'default' one first."
+ )
+ if current_default_config_name:
+ _ = metadata_configs[current_default_config_name].pop("default")
+ configs_to_dump[current_default_config_name] = metadata_configs[current_default_config_name]
+ metadata_config_to_dump["default"] = True
+ # push to the deprecated dataset_infos.json
+ if repo_with_dataset_infos:
+ dataset_infos_path = api.hf_hub_download(
+ repo_id,
+ config.DATASETDICT_INFOS_FILENAME,
+ repo_type="dataset",
+ revision=parent_commit,
+ )
+ with open(dataset_infos_path, encoding="utf-8") as f:
+ dataset_infos: dict = json.load(f)
+ dataset_infos[config_name] = asdict(info_to_dump)
+ new_dataset_infos = json.dumps(dataset_infos, indent=4)
+ else:
+ new_dataset_infos = None
+ # push to README
+ DatasetInfosDict({config_name: info_to_dump}).to_dataset_card_data(dataset_card_data)
+ MetadataConfigs(configs_to_dump).to_dataset_card_data(dataset_card_data)
+ new_dataset_card = (
+ DatasetCard(f"---\n{dataset_card_data}\n---\n") if dataset_card is None else dataset_card
+ )
+ return parent_commit, deletions, new_dataset_card, new_dataset_infos
+
+ commit_message = commit_message if commit_message is not None else "Upload dataset"
+ if len(additions) > config.UPLOADS_MAX_NUMBER_PER_COMMIT:
+ logger.info(
+ f"Number of files to upload is larger than {config.UPLOADS_MAX_NUMBER_PER_COMMIT}. Splitting the push into multiple commits."
+ )
+ num_commits = math.ceil(len(additions) / config.UPLOADS_MAX_NUMBER_PER_COMMIT)
+ for i in range(0, num_commits):
+ operations = additions[
+ i * config.UPLOADS_MAX_NUMBER_PER_COMMIT : (i + 1) * config.UPLOADS_MAX_NUMBER_PER_COMMIT
+ ]
+ for retry, sleep_time in enumerate(itertools.chain(range(10), itertools.repeat(30)), start=1):
+ # We need to retry if another commit happens at the same time
+ sleep_time *= 1 + random.random()
+ try:
+ commit_info = api.create_commit(
+ repo_id,
+ operations=operations,
+ commit_message=commit_message + f" (part {i:05d}-of-{num_commits:05d})",
+ commit_description=commit_description,
+ repo_type="dataset",
+ revision=revision,
+ create_pr=create_pr,
+ )
+ except HfHubHTTPError as err:
+ if (
+ err.__context__
+ and isinstance(err.__context__, HfHubHTTPError)
+ and err.__context__.response.status_code == 409
+ ):
+ # 409 is Conflict (another commit is in progress)
+ time.sleep(sleep_time)
+ logger.info(
+ f"Retrying intermediate commit for {repo_id}, {config_name} ({retry}/n with status_code {err.__context__.response.status_code})"
+ )
+ continue
+ else:
+ raise
+ break
+ logger.info(
+ f"Commit #{i + 1} completed"
+ + (f" (still {num_commits - i - 1} to go)" if num_commits - i - 1 else "")
+ + "."
+ )
+ last_commit_additions = []
+ else:
+ last_commit_additions = additions
+
+ for retry, sleep_time in enumerate(itertools.chain(range(10), itertools.repeat(30)), start=1):
+ # We need to retry if there was a commit in between in case it touched the dataset card data
+ sleep_time *= 1 + random.random()
+ parent_commit, deletions, dataset_card, dataset_infos = get_deletions_and_dataset_card()
+ dataset_card_additions = []
+ if dataset_infos:
+ dataset_card_additions.append(
+ CommitOperationAdd(
+ path_in_repo=config.DATASETDICT_INFOS_FILENAME,
+ path_or_fileobj=dataset_infos.encode("utf-8"),
+ )
+ )
+ dataset_card_additions.append(
+ CommitOperationAdd(path_in_repo=config.REPOCARD_FILENAME, path_or_fileobj=str(dataset_card).encode())
+ )
+ try:
+ commit_info = api.create_commit(
+ repo_id,
+ operations=last_commit_additions + dataset_card_additions + deletions,
+ commit_message=commit_message,
+ commit_description=commit_description,
+ repo_type="dataset",
+ revision=revision,
+ create_pr=create_pr,
+ parent_commit=parent_commit,
+ )
+ except HfHubHTTPError as err:
+ if (
+ err.__context__
+ and isinstance(err.__context__, HfHubHTTPError)
+ and err.__context__.response.status_code in (412, 409)
+ ):
+ # 412 is Precondition failed (parent_commit isn't satisfied)
+ # 409 is Conflict (another commit is in progress)
+ time.sleep(sleep_time)
+ logger.info(
+ f"Retrying commit for {repo_id}, {config_name} ({retry}/n with status_code {err.__context__.response.status_code})"
+ )
+ continue
+ else:
+ raise
+ break
+
+ return commit_info
diff --git a/venv/lib/python3.10/site-packages/datasets/distributed.py b/venv/lib/python3.10/site-packages/datasets/distributed.py
new file mode 100644
index 0000000000000000000000000000000000000000..4697948f342a81d489084b068758327156a89b8b
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/distributed.py
@@ -0,0 +1,39 @@
+from typing import TypeVar
+
+from .arrow_dataset import Dataset, _split_by_node_map_style_dataset
+from .iterable_dataset import IterableDataset, _split_by_node_iterable_dataset
+
+
+DatasetType = TypeVar("DatasetType", Dataset, IterableDataset)
+
+
+def split_dataset_by_node(dataset: DatasetType, rank: int, world_size: int) -> DatasetType:
+ """
+ Split a dataset for the node at rank `rank` in a pool of nodes of size `world_size`.
+
+ For map-style datasets:
+
+ Each node is assigned a chunk of data, e.g. rank 0 is given the first chunk of the dataset.
+ To maximize data loading throughput, chunks are made of contiguous data on disk if possible.
+
+ For iterable datasets:
+
+ If the dataset has a number of shards that is a factor of `world_size` (i.e. if `dataset.num_shards % world_size == 0`),
+ then the shards are evenly assigned across the nodes, which is the most optimized.
+ Otherwise, each node keeps 1 example out of `world_size`, skipping the other examples.
+
+ Args:
+ dataset ([`Dataset`] or [`IterableDataset`]):
+ The dataset to split by node.
+ rank (`int`):
+ Rank of the current node.
+ world_size (`int`):
+ Total number of nodes.
+
+ Returns:
+ [`Dataset`] or [`IterableDataset`]: The dataset to be used on the node at rank `rank`.
+ """
+ if isinstance(dataset, Dataset):
+ return _split_by_node_map_style_dataset(dataset, rank=rank, world_size=world_size)
+ else:
+ return _split_by_node_iterable_dataset(dataset, rank=rank, world_size=world_size)
diff --git a/venv/lib/python3.10/site-packages/datasets/download/__init__.py b/venv/lib/python3.10/site-packages/datasets/download/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..8ae0d436504dc2e609b0ca8851509c72a161dbde
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/download/__init__.py
@@ -0,0 +1,10 @@
+__all__ = [
+ "DownloadConfig",
+ "DownloadManager",
+ "DownloadMode",
+ "StreamingDownloadManager",
+]
+
+from .download_config import DownloadConfig
+from .download_manager import DownloadManager, DownloadMode
+from .streaming_download_manager import StreamingDownloadManager
diff --git a/venv/lib/python3.10/site-packages/datasets/download/__pycache__/__init__.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/download/__pycache__/__init__.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..b48e615da3c4c307cf4d1d15e919eb1a0270c571
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/download/__pycache__/__init__.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/download/__pycache__/download_config.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/download/__pycache__/download_config.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..40fc09c53694cabedce0e42d200501bd9371ae98
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/download/__pycache__/download_config.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/download/__pycache__/download_manager.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/download/__pycache__/download_manager.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..1859f5fb295c23f1c024d2c38ca4bd468cc5cc0a
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/download/__pycache__/download_manager.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/download/__pycache__/streaming_download_manager.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/download/__pycache__/streaming_download_manager.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..335fa37de546d313a9c73f14101ad779fe373363
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/download/__pycache__/streaming_download_manager.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/download/download_config.py b/venv/lib/python3.10/site-packages/datasets/download/download_config.py
new file mode 100644
index 0000000000000000000000000000000000000000..6efae72b671bd0977c52453582dc70561766334b
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/download/download_config.py
@@ -0,0 +1,81 @@
+import copy
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Optional, Union
+
+from .. import config
+
+
+@dataclass
+class DownloadConfig:
+ """Configuration for our cached path manager.
+
+ Attributes:
+ cache_dir (`str` or `Path`, *optional*):
+ Specify a cache directory to save the file to (overwrite the
+ default cache dir).
+ force_download (`bool`, defaults to `False`):
+ If `True`, re-download the file even if it's already cached in
+ the cache dir.
+ resume_download (`bool`, defaults to `False`):
+ If `True`, resume the download if an incompletely received file is
+ found.
+ proxies (`dict`, *optional*):
+ user_agent (`str`, *optional*):
+ Optional string or dict that will be appended to the user-agent on remote
+ requests.
+ extract_compressed_file (`bool`, defaults to `False`):
+ If `True` and the path point to a zip or tar file,
+ extract the compressed file in a folder along the archive.
+ force_extract (`bool`, defaults to `False`):
+ If `True` when `extract_compressed_file` is `True` and the archive
+ was already extracted, re-extract the archive and override the folder where it was extracted.
+ delete_extracted (`bool`, defaults to `False`):
+ Whether to delete (or keep) the extracted files.
+ extract_on_the_fly (`bool`, defaults to `False`):
+ If `True`, extract compressed files while they are being read.
+ use_etag (`bool`, defaults to `True`):
+ Whether to use the ETag HTTP response header to validate the cached files.
+ num_proc (`int`, *optional*):
+ The number of processes to launch to download the files in parallel.
+ max_retries (`int`, default to `1`):
+ The number of times to retry an HTTP request if it fails.
+ token (`str` or `bool`, *optional*):
+ Optional string or boolean to use as Bearer token
+ for remote files on the Datasets Hub. If `True`, or not specified, will get token from `~/.huggingface`.
+ storage_options (`dict`, *optional*):
+ Key/value pairs to be passed on to the dataset file-system backend, if any.
+ download_desc (`str`, *optional*):
+ A description to be displayed alongside with the progress bar while downloading the files.
+ disable_tqdm (`bool`, defaults to `False`):
+ Whether to disable the individual files download progress bar
+ """
+
+ cache_dir: Optional[Union[str, Path]] = None
+ force_download: bool = False
+ resume_download: bool = False
+ local_files_only: bool = False
+ proxies: Optional[dict] = None
+ user_agent: Optional[str] = None
+ extract_compressed_file: bool = False
+ force_extract: bool = False
+ delete_extracted: bool = False
+ extract_on_the_fly: bool = False
+ use_etag: bool = True
+ num_proc: Optional[int] = None
+ max_retries: int = 1
+ token: Optional[Union[str, bool]] = None
+ storage_options: dict[str, Any] = field(default_factory=dict)
+ download_desc: Optional[str] = None
+ disable_tqdm: bool = False
+
+ def copy(self) -> "DownloadConfig":
+ return self.__class__(**{k: copy.deepcopy(v) for k, v in self.__dict__.items()})
+
+ def __setattr__(self, name, value):
+ if name == "token" and getattr(self, "storage_options", None) is not None:
+ if "hf" not in self.storage_options:
+ self.storage_options["hf"] = {"endpoint": config.HF_ENDPOINT, "token": value}
+ elif getattr(self.storage_options["hf"], "token", None) is None:
+ self.storage_options["hf"]["token"] = value
+ super().__setattr__(name, value)
diff --git a/venv/lib/python3.10/site-packages/datasets/download/download_manager.py b/venv/lib/python3.10/site-packages/datasets/download/download_manager.py
new file mode 100644
index 0000000000000000000000000000000000000000..b6ee1d28e2b3dd203d62d27137861b33651324db
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/download/download_manager.py
@@ -0,0 +1,340 @@
+# Copyright 2020 The TensorFlow Datasets Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""Download manager interface."""
+
+import enum
+import io
+import multiprocessing
+import os
+from datetime import datetime
+from functools import partial
+from typing import Optional, Union
+
+import fsspec
+from fsspec.core import url_to_fs
+from tqdm.contrib.concurrent import thread_map
+
+from .. import config
+from ..utils import tqdm as hf_tqdm
+from ..utils.file_utils import (
+ ArchiveIterable,
+ FilesIterable,
+ cached_path,
+ is_relative_path,
+ stack_multiprocessing_download_progress_bars,
+ url_or_path_join,
+)
+from ..utils.info_utils import get_size_checksum_dict
+from ..utils.logging import get_logger, tqdm
+from ..utils.py_utils import NestedDataStructure, map_nested
+from ..utils.track import tracked_str
+from .download_config import DownloadConfig
+
+
+logger = get_logger(__name__)
+
+
+class DownloadMode(enum.Enum):
+ """`Enum` for how to treat pre-existing downloads and data.
+
+ The default mode is `REUSE_DATASET_IF_EXISTS`, which will reuse both
+ raw downloads and the prepared dataset if they exist.
+
+ The generations modes:
+
+ | | Downloads | Dataset |
+ |-------------------------------------|-----------|---------|
+ | `REUSE_DATASET_IF_EXISTS` (default) | Reuse | Reuse |
+ | `REUSE_CACHE_IF_EXISTS` | Reuse | Fresh |
+ | `FORCE_REDOWNLOAD` | Fresh | Fresh |
+
+ """
+
+ REUSE_DATASET_IF_EXISTS = "reuse_dataset_if_exists"
+ REUSE_CACHE_IF_EXISTS = "reuse_cache_if_exists"
+ FORCE_REDOWNLOAD = "force_redownload"
+
+
+class DownloadManager:
+ is_streaming = False
+
+ def __init__(
+ self,
+ dataset_name: Optional[str] = None,
+ data_dir: Optional[str] = None,
+ download_config: Optional[DownloadConfig] = None,
+ base_path: Optional[str] = None,
+ record_checksums=True,
+ ):
+ """Download manager constructor.
+
+ Args:
+ data_dir:
+ can be used to specify a manual directory to get the files from.
+ dataset_name (`str`):
+ name of dataset this instance will be used for. If
+ provided, downloads will contain which datasets they were used for.
+ download_config (`DownloadConfig`):
+ to specify the cache directory and other
+ download options
+ base_path (`str`):
+ base path that is used when relative paths are used to
+ download files. This can be a remote url.
+ record_checksums (`bool`, defaults to `True`):
+ Whether to record the checksums of the downloaded files. If None, the value is inferred from the builder.
+ """
+ self._dataset_name = dataset_name
+ self._data_dir = data_dir
+ self._base_path = base_path or os.path.abspath(".")
+ # To record what is being used: {url: {num_bytes: int, checksum: str}}
+ self._recorded_sizes_checksums: dict[str, dict[str, Optional[Union[int, str]]]] = {}
+ self.record_checksums = record_checksums
+ self.download_config = download_config or DownloadConfig()
+ self.downloaded_paths = {}
+ self.extracted_paths = {}
+
+ @property
+ def manual_dir(self):
+ return self._data_dir
+
+ @property
+ def downloaded_size(self):
+ """Returns the total size of downloaded files."""
+ return sum(checksums_dict["num_bytes"] for checksums_dict in self._recorded_sizes_checksums.values())
+
+ def _record_sizes_checksums(self, url_or_urls: NestedDataStructure, downloaded_path_or_paths: NestedDataStructure):
+ """Record size/checksum of downloaded files."""
+ delay = 5
+ for url, path in hf_tqdm(
+ list(zip(url_or_urls.flatten(), downloaded_path_or_paths.flatten())),
+ delay=delay,
+ desc="Computing checksums",
+ ):
+ # call str to support PathLike objects
+ self._recorded_sizes_checksums[str(url)] = get_size_checksum_dict(
+ path, record_checksum=self.record_checksums
+ )
+
+ def download(self, url_or_urls):
+ """Download given URL(s).
+
+ By default, only one process is used for download. Pass customized `download_config.num_proc` to change this behavior.
+
+ Args:
+ url_or_urls (`str` or `list` or `dict`):
+ URL or `list` or `dict` of URLs to download. Each URL is a `str`.
+
+ Returns:
+ `str` or `list` or `dict`:
+ The downloaded paths matching the given input `url_or_urls`.
+
+ Example:
+
+ ```py
+ >>> downloaded_files = dl_manager.download('https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz')
+ ```
+ """
+ download_config = self.download_config.copy()
+ download_config.extract_compressed_file = False
+ if download_config.download_desc is None:
+ download_config.download_desc = "Downloading data"
+
+ download_func = partial(self._download_batched, download_config=download_config)
+
+ start_time = datetime.now()
+ with stack_multiprocessing_download_progress_bars():
+ downloaded_path_or_paths = map_nested(
+ download_func,
+ url_or_urls,
+ map_tuple=True,
+ num_proc=download_config.num_proc,
+ desc="Downloading data files",
+ batched=True,
+ batch_size=-1,
+ )
+ duration = datetime.now() - start_time
+ logger.info(f"Downloading took {duration.total_seconds() // 60} min")
+ url_or_urls = NestedDataStructure(url_or_urls)
+ downloaded_path_or_paths = NestedDataStructure(downloaded_path_or_paths)
+ self.downloaded_paths.update(dict(zip(url_or_urls.flatten(), downloaded_path_or_paths.flatten())))
+
+ start_time = datetime.now()
+ self._record_sizes_checksums(url_or_urls, downloaded_path_or_paths)
+ duration = datetime.now() - start_time
+ logger.info(f"Checksum Computation took {duration.total_seconds() // 60} min")
+
+ return downloaded_path_or_paths.data
+
+ def _download_batched(
+ self,
+ url_or_filenames: list[str],
+ download_config: DownloadConfig,
+ ) -> list[str]:
+ if len(url_or_filenames) >= 16:
+ download_config = download_config.copy()
+ download_config.disable_tqdm = True
+ download_func = partial(self._download_single, download_config=download_config)
+
+ fs: fsspec.AbstractFileSystem
+ path = str(url_or_filenames[0])
+ if is_relative_path(path):
+ # append the relative path to the base_path
+ path = url_or_path_join(self._base_path, path)
+ fs, path = url_to_fs(path, **download_config.storage_options)
+ size = 0
+ try:
+ size = fs.info(path).get("size", 0)
+ except Exception:
+ pass
+ max_workers = (
+ config.HF_DATASETS_MULTITHREADING_MAX_WORKERS if size < (20 << 20) else 1
+ ) # enable multithreading if files are small
+
+ return thread_map(
+ download_func,
+ url_or_filenames,
+ desc=download_config.download_desc or "Downloading",
+ unit="files",
+ position=multiprocessing.current_process()._identity[-1] # contains the ranks of subprocesses
+ if os.environ.get("HF_DATASETS_STACK_MULTIPROCESSING_DOWNLOAD_PROGRESS_BARS") == "1"
+ and multiprocessing.current_process()._identity
+ else None,
+ max_workers=max_workers,
+ tqdm_class=tqdm,
+ )
+ else:
+ return [
+ self._download_single(url_or_filename, download_config=download_config)
+ for url_or_filename in url_or_filenames
+ ]
+
+ def _download_single(self, url_or_filename: str, download_config: DownloadConfig) -> str:
+ url_or_filename = str(url_or_filename)
+ if is_relative_path(url_or_filename):
+ # append the relative path to the base_path
+ url_or_filename = url_or_path_join(self._base_path, url_or_filename)
+ out = cached_path(url_or_filename, download_config=download_config)
+ out = tracked_str(out)
+ out.set_origin(url_or_filename)
+ return out
+
+ def iter_archive(self, path_or_buf: Union[str, io.BufferedReader]):
+ """Iterate over files within an archive.
+
+ Args:
+ path_or_buf (`str` or `io.BufferedReader`):
+ Archive path or archive binary file object.
+
+ Yields:
+ `tuple[str, io.BufferedReader]`:
+ 2-tuple (path_within_archive, file_object).
+ File object is opened in binary mode.
+
+ Example:
+
+ ```py
+ >>> archive = dl_manager.download('https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz')
+ >>> files = dl_manager.iter_archive(archive)
+ ```
+ """
+
+ if hasattr(path_or_buf, "read"):
+ return ArchiveIterable.from_buf(path_or_buf)
+ else:
+ return ArchiveIterable.from_urlpath(path_or_buf)
+
+ def iter_files(self, paths: Union[str, list[str]]):
+ """Iterate over file paths.
+
+ Args:
+ paths (`str` or `list` of `str`):
+ Root paths.
+
+ Yields:
+ `str`: File path.
+
+ Example:
+
+ ```py
+ >>> files = dl_manager.download_and_extract('https://huggingface.co/datasets/beans/resolve/main/data/train.zip')
+ >>> files = dl_manager.iter_files(files)
+ ```
+ """
+ return FilesIterable.from_urlpaths(paths)
+
+ def extract(self, path_or_paths):
+ """Extract given path(s).
+
+ Args:
+ path_or_paths (path or `list` or `dict`):
+ Path of file to extract. Each path is a `str`.
+
+ Returns:
+ extracted_path(s): `str`, The extracted paths matching the given input
+ path_or_paths.
+
+ Example:
+
+ ```py
+ >>> downloaded_files = dl_manager.download('https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz')
+ >>> extracted_files = dl_manager.extract(downloaded_files)
+ ```
+ """
+ download_config = self.download_config.copy()
+ download_config.extract_compressed_file = True
+ extract_func = partial(self._download_single, download_config=download_config)
+ extracted_paths = map_nested(
+ extract_func,
+ path_or_paths,
+ num_proc=download_config.num_proc,
+ desc="Extracting data files",
+ )
+ path_or_paths = NestedDataStructure(path_or_paths)
+ extracted_paths = NestedDataStructure(extracted_paths)
+ self.extracted_paths.update(dict(zip(path_or_paths.flatten(), extracted_paths.flatten())))
+ return extracted_paths.data
+
+ def download_and_extract(self, url_or_urls):
+ """Download and extract given `url_or_urls`.
+
+ Is roughly equivalent to:
+
+ ```
+ extracted_paths = dl_manager.extract(dl_manager.download(url_or_urls))
+ ```
+
+ Args:
+ url_or_urls (`str` or `list` or `dict`):
+ URL or `list` or `dict` of URLs to download and extract. Each URL is a `str`.
+
+ Returns:
+ extracted_path(s): `str`, extracted paths of given URL(s).
+ """
+ return self.extract(self.download(url_or_urls))
+
+ def get_recorded_sizes_checksums(self):
+ return self._recorded_sizes_checksums.copy()
+
+ def delete_extracted_files(self):
+ paths_to_delete = set(self.extracted_paths.values()) - set(self.downloaded_paths.values())
+ for key, path in list(self.extracted_paths.items()):
+ if path in paths_to_delete and os.path.isfile(path):
+ os.remove(path)
+ del self.extracted_paths[key]
+
+ def manage_extracted_files(self):
+ if self.download_config.delete_extracted:
+ self.delete_extracted_files()
diff --git a/venv/lib/python3.10/site-packages/datasets/download/streaming_download_manager.py b/venv/lib/python3.10/site-packages/datasets/download/streaming_download_manager.py
new file mode 100644
index 0000000000000000000000000000000000000000..ff2dc1a64bd95616ca78f515e0ee2f14becfc5f6
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/download/streaming_download_manager.py
@@ -0,0 +1,219 @@
+import io
+import os
+from collections.abc import Iterable
+from typing import Optional, Union
+
+from ..utils.file_utils import ( # noqa: F401 # backward compatibility
+ SINGLE_FILE_COMPRESSION_PROTOCOLS,
+ ArchiveIterable,
+ FilesIterable,
+ _get_extraction_protocol,
+ _get_path_extension,
+ _prepare_path_and_storage_options,
+ is_relative_path,
+ url_or_path_join,
+ xbasename,
+ xdirname,
+ xet_parse,
+ xexists,
+ xgetsize,
+ xglob,
+ xgzip_open,
+ xisdir,
+ xisfile,
+ xjoin,
+ xlistdir,
+ xnumpy_load,
+ xopen,
+ xpandas_read_csv,
+ xpandas_read_excel,
+ xPath,
+ xpyarrow_parquet_read_table,
+ xrelpath,
+ xsio_loadmat,
+ xsplit,
+ xsplitext,
+ xwalk,
+ xxml_dom_minidom_parse,
+)
+from ..utils.logging import get_logger
+from ..utils.py_utils import map_nested
+from .download_config import DownloadConfig
+
+
+logger = get_logger(__name__)
+
+
+class StreamingDownloadManager:
+ """
+ Download manager that uses the "::" separator to navigate through (possibly remote) compressed archives.
+ Contrary to the regular `DownloadManager`, the `download` and `extract` methods don't actually download nor extract
+ data, but they rather return the path or url that could be opened using the `xopen` function which extends the
+ built-in `open` function to stream data from remote files.
+ """
+
+ is_streaming = True
+
+ def __init__(
+ self,
+ dataset_name: Optional[str] = None,
+ data_dir: Optional[str] = None,
+ download_config: Optional[DownloadConfig] = None,
+ base_path: Optional[str] = None,
+ ):
+ self._dataset_name = dataset_name
+ self._data_dir = data_dir
+ self._base_path = base_path or os.path.abspath(".")
+ self.download_config = download_config or DownloadConfig()
+ self.downloaded_size = None
+ self.record_checksums = False
+
+ @property
+ def manual_dir(self):
+ return self._data_dir
+
+ def download(self, url_or_urls):
+ """Normalize URL(s) of files to stream data from.
+ This is the lazy version of `DownloadManager.download` for streaming.
+
+ Args:
+ url_or_urls (`str` or `list` or `dict`):
+ URL(s) of files to stream data from. Each url is a `str`.
+
+ Returns:
+ url(s): (`str` or `list` or `dict`), URL(s) to stream data from matching the given input url_or_urls.
+
+ Example:
+
+ ```py
+ >>> downloaded_files = dl_manager.download('https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz')
+ ```
+ """
+ url_or_urls = map_nested(self._download_single, url_or_urls, map_tuple=True)
+ return url_or_urls
+
+ def _download_single(self, urlpath: str) -> str:
+ urlpath = str(urlpath)
+ if is_relative_path(urlpath):
+ # append the relative path to the base_path
+ urlpath = url_or_path_join(self._base_path, urlpath)
+ return urlpath
+
+ def extract(self, url_or_urls):
+ """Add extraction protocol for given url(s) for streaming.
+
+ This is the lazy version of `DownloadManager.extract` for streaming.
+
+ Args:
+ url_or_urls (`str` or `list` or `dict`):
+ URL(s) of files to stream data from. Each url is a `str`.
+
+ Returns:
+ url(s): (`str` or `list` or `dict`), URL(s) to stream data from matching the given input `url_or_urls`.
+
+ Example:
+
+ ```py
+ >>> downloaded_files = dl_manager.download('https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz')
+ >>> extracted_files = dl_manager.extract(downloaded_files)
+ ```
+ """
+ urlpaths = map_nested(self._extract, url_or_urls, map_tuple=True)
+ return urlpaths
+
+ def _extract(self, urlpath: str) -> str:
+ urlpath = str(urlpath)
+ protocol = _get_extraction_protocol(urlpath, download_config=self.download_config)
+ # get inner file: zip://train-00000.json.gz::https://foo.bar/data.zip -> zip://train-00000.json.gz
+ path = urlpath.split("::")[0]
+ extension = _get_path_extension(path)
+ if extension in ["tgz", "tar"] or path.endswith((".tar.gz", ".tar.bz2", ".tar.xz")):
+ raise NotImplementedError(
+ f"Extraction protocol for TAR archives like '{urlpath}' is not implemented in streaming mode. "
+ f"Please use `dl_manager.iter_archive` instead.\n\n"
+ f"Example usage:\n\n"
+ f"\turl = dl_manager.download(url)\n"
+ f"\ttar_archive_iterator = dl_manager.iter_archive(url)\n\n"
+ f"\tfor filename, file in tar_archive_iterator:\n"
+ f"\t\t..."
+ )
+ if protocol is None:
+ # no extraction
+ return urlpath
+ elif protocol in SINGLE_FILE_COMPRESSION_PROTOCOLS:
+ # there is one single file which is the uncompressed file
+ inner_file = os.path.basename(urlpath.split("::")[0])
+ inner_file = inner_file[: inner_file.rindex(".")] if "." in inner_file else inner_file
+ return f"{protocol}://{inner_file}::{urlpath}"
+ else:
+ return f"{protocol}://::{urlpath}"
+
+ def download_and_extract(self, url_or_urls):
+ """Prepare given `url_or_urls` for streaming (add extraction protocol).
+
+ This is the lazy version of `DownloadManager.download_and_extract` for streaming.
+
+ Is equivalent to:
+
+ ```
+ urls = dl_manager.extract(dl_manager.download(url_or_urls))
+ ```
+
+ Args:
+ url_or_urls (`str` or `list` or `dict`):
+ URL(s) to stream from data from. Each url is a `str`.
+
+ Returns:
+ url(s): (`str` or `list` or `dict`), URL(s) to stream data from matching the given input `url_or_urls`.
+ """
+ return self.extract(self.download(url_or_urls))
+
+ def iter_archive(self, urlpath_or_buf: Union[str, io.BufferedReader]) -> Iterable[tuple]:
+ """Iterate over files within an archive.
+
+ Args:
+ urlpath_or_buf (`str` or `io.BufferedReader`):
+ Archive path or archive binary file object.
+
+ Yields:
+ `tuple[str, io.BufferedReader]`:
+ 2-tuple (path_within_archive, file_object).
+ File object is opened in binary mode.
+
+ Example:
+
+ ```py
+ >>> archive = dl_manager.download('https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz')
+ >>> files = dl_manager.iter_archive(archive)
+ ```
+ """
+
+ if hasattr(urlpath_or_buf, "read"):
+ return ArchiveIterable.from_buf(urlpath_or_buf)
+ else:
+ return ArchiveIterable.from_urlpath(urlpath_or_buf, download_config=self.download_config)
+
+ def iter_files(self, urlpaths: Union[str, list[str]]) -> Iterable[str]:
+ """Iterate over files.
+
+ Args:
+ urlpaths (`str` or `list` of `str`):
+ Root paths.
+
+ Yields:
+ str: File URL path.
+
+ Example:
+
+ ```py
+ >>> files = dl_manager.download_and_extract('https://huggingface.co/datasets/beans/resolve/main/data/train.zip')
+ >>> files = dl_manager.iter_files(files)
+ ```
+ """
+ return FilesIterable.from_urlpaths(urlpaths, download_config=self.download_config)
+
+ def manage_extracted_files(self):
+ pass
+
+ def get_recorded_sizes_checksums(self):
+ pass
diff --git a/venv/lib/python3.10/site-packages/datasets/exceptions.py b/venv/lib/python3.10/site-packages/datasets/exceptions.py
new file mode 100644
index 0000000000000000000000000000000000000000..7c1724fb572dc98a2b65ad07f0a1a08c5d2a32e5
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/exceptions.py
@@ -0,0 +1,119 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2023 The HuggingFace Authors.
+from typing import Any, Optional, Union
+
+from huggingface_hub import HfFileSystem
+
+from . import config
+from .table import CastError
+from .utils.track import TrackedIterableFromGenerator, tracked_list, tracked_str
+
+
+class DatasetsError(Exception):
+ """Base class for exceptions in this library."""
+
+
+class DefunctDatasetError(DatasetsError):
+ """The dataset has been defunct."""
+
+
+class FileNotFoundDatasetsError(DatasetsError, FileNotFoundError):
+ """FileNotFoundError raised by this library."""
+
+
+class DataFilesNotFoundError(FileNotFoundDatasetsError):
+ """No (supported) data files found."""
+
+
+class DatasetNotFoundError(FileNotFoundDatasetsError):
+ """Dataset not found.
+
+ Raised when trying to access:
+ - a missing dataset, or
+ - a private/gated dataset and the user is not authenticated.
+ """
+
+
+class DatasetBuildError(DatasetsError):
+ pass
+
+
+class ManualDownloadError(DatasetBuildError):
+ pass
+
+
+class FileFormatError(DatasetBuildError):
+ pass
+
+
+class DatasetGenerationError(DatasetBuildError):
+ pass
+
+
+class DatasetGenerationCastError(DatasetGenerationError):
+ @classmethod
+ def from_cast_error(
+ cls,
+ cast_error: CastError,
+ builder_name: str,
+ gen_kwargs: dict[str, Any],
+ token: Optional[Union[bool, str]],
+ ) -> "DatasetGenerationCastError":
+ explanation_message = (
+ f"\n\nAll the data files must have the same columns, but at some point {cast_error.details()}"
+ )
+ formatted_tracked_gen_kwargs: list[str] = []
+ for gen_kwarg in gen_kwargs.values():
+ if not isinstance(gen_kwarg, (tracked_str, tracked_list, TrackedIterableFromGenerator)):
+ continue
+ while (
+ isinstance(gen_kwarg, (tracked_list, TrackedIterableFromGenerator)) and gen_kwarg.last_item is not None
+ ):
+ gen_kwarg = gen_kwarg.last_item
+ if isinstance(gen_kwarg, tracked_str):
+ gen_kwarg = gen_kwarg.get_origin()
+ if isinstance(gen_kwarg, str) and gen_kwarg.startswith("hf://"):
+ resolved_path = HfFileSystem(endpoint=config.HF_ENDPOINT, token=token).resolve_path(gen_kwarg)
+ gen_kwarg = "hf://" + resolved_path.unresolve()
+ if "@" + resolved_path.revision in gen_kwarg:
+ gen_kwarg = (
+ gen_kwarg.replace("@" + resolved_path.revision, "", 1)
+ + f" (at revision {resolved_path.revision})"
+ )
+ formatted_tracked_gen_kwargs.append(str(gen_kwarg))
+ if formatted_tracked_gen_kwargs:
+ explanation_message += f"\n\nThis happened while the {builder_name} dataset builder was generating data using\n\n{', '.join(formatted_tracked_gen_kwargs)}"
+ help_message = "\n\nPlease either edit the data files to have matching columns, or separate them into different configurations (see docs at https://hf.co/docs/hub/datasets-manual-configuration#multiple-configurations)"
+ return cls("An error occurred while generating the dataset" + explanation_message + help_message)
+
+
+class ChecksumVerificationError(DatasetsError):
+ """Error raised during checksums verifications of downloaded files."""
+
+
+class UnexpectedDownloadedFileError(ChecksumVerificationError):
+ """Some downloaded files were not expected."""
+
+
+class ExpectedMoreDownloadedFilesError(ChecksumVerificationError):
+ """Some files were supposed to be downloaded but were not."""
+
+
+class NonMatchingChecksumError(ChecksumVerificationError):
+ """The downloaded file checksum don't match the expected checksum."""
+
+
+class SplitsVerificationError(DatasetsError):
+ """Error raised during splits verifications."""
+
+
+class UnexpectedSplitsError(SplitsVerificationError):
+ """The expected splits of the downloaded file is missing."""
+
+
+class ExpectedMoreSplitsError(SplitsVerificationError):
+ """Some recorded splits are missing."""
+
+
+class NonMatchingSplitsSizesError(SplitsVerificationError):
+ """The splits sizes don't match the expected splits sizes."""
diff --git a/venv/lib/python3.10/site-packages/datasets/features/__init__.py b/venv/lib/python3.10/site-packages/datasets/features/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..40a3568039ae3c65ef50b764c21123c81952f0ff
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/features/__init__.py
@@ -0,0 +1,26 @@
+__all__ = [
+ "Audio",
+ "Array2D",
+ "Array3D",
+ "Array4D",
+ "Array5D",
+ "ClassLabel",
+ "Features",
+ "LargeList",
+ "List",
+ "Sequence",
+ "Value",
+ "Image",
+ "Translation",
+ "TranslationVariableLanguages",
+ "Video",
+ "Pdf",
+ "Nifti",
+]
+from .audio import Audio
+from .features import Array2D, Array3D, Array4D, Array5D, ClassLabel, Features, LargeList, List, Sequence, Value
+from .image import Image
+from .nifti import Nifti
+from .pdf import Pdf
+from .translation import Translation, TranslationVariableLanguages
+from .video import Video
diff --git a/venv/lib/python3.10/site-packages/datasets/features/__pycache__/__init__.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/features/__pycache__/__init__.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..8d01cf9c5b008aee7f4db121dd69e141fdaf7040
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/features/__pycache__/__init__.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/features/__pycache__/_torchcodec.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/features/__pycache__/_torchcodec.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..8a598f406fa7508fd53eb1e57653fe119390d97b
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/features/__pycache__/_torchcodec.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/features/__pycache__/audio.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/features/__pycache__/audio.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..d9e52aa58c26faadc1403c06eeb0e7be0325bf57
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/features/__pycache__/audio.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/features/__pycache__/features.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/features/__pycache__/features.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..b2787e7b3650f7e8f3e683d4dfba8466a88abdc9
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/features/__pycache__/features.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/features/__pycache__/image.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/features/__pycache__/image.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..1de6a8248b2b0c0771ae8559c4d298e973f6aafa
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/features/__pycache__/image.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/features/__pycache__/nifti.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/features/__pycache__/nifti.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..9f530afa091ba2a56554ee66dfd1959909e65e80
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/features/__pycache__/nifti.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/features/__pycache__/pdf.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/features/__pycache__/pdf.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..94a6c733c60737b83b18e85ba377639883e48a1f
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/features/__pycache__/pdf.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/features/__pycache__/translation.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/features/__pycache__/translation.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..3589a5f50514b7a0428e1a7fd7d1c1b6ee7a47a8
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/features/__pycache__/translation.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/features/__pycache__/video.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/features/__pycache__/video.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..f110d1dd2b874f27c79a37dfed485722dd9ac4ca
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/features/__pycache__/video.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/features/_torchcodec.py b/venv/lib/python3.10/site-packages/datasets/features/_torchcodec.py
new file mode 100644
index 0000000000000000000000000000000000000000..d71b4156a5124be4a6433ce76a3794d55393499c
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/features/_torchcodec.py
@@ -0,0 +1,15 @@
+import numpy as np
+from torchcodec.decoders import AudioDecoder as _AudioDecoder
+
+
+class AudioDecoder(_AudioDecoder):
+ def __getitem__(self, key: str):
+ if key == "array":
+ y = self.get_all_samples().data.cpu().numpy()
+ return np.mean(y, axis=tuple(range(y.ndim - 1))) if y.ndim > 1 else y
+ elif key == "sampling_rate":
+ return self.get_samples_played_in_range(0, 0).sample_rate
+ elif hasattr(super(), "__getitem__"):
+ return super().__getitem__(key)
+ else:
+ raise TypeError("'torchcodec.decoders.AudioDecoder' object is not subscriptable")
diff --git a/venv/lib/python3.10/site-packages/datasets/features/audio.py b/venv/lib/python3.10/site-packages/datasets/features/audio.py
new file mode 100644
index 0000000000000000000000000000000000000000..d9513f289f5397a132775a1fc631d470370ee7fc
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/features/audio.py
@@ -0,0 +1,330 @@
+import os
+from dataclasses import dataclass, field
+from io import BytesIO
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union
+
+import numpy as np
+import pyarrow as pa
+
+from .. import config
+from ..download.download_config import DownloadConfig
+from ..table import array_cast
+from ..utils.file_utils import is_local_path, xopen
+from ..utils.py_utils import no_op_if_value_is_null, string_to_dict
+
+
+if TYPE_CHECKING:
+ from torchcodec.decoders import AudioDecoder
+
+ from .features import FeatureType
+
+
+@dataclass
+class Audio:
+ """Audio [`Feature`] to extract audio data from an audio file.
+
+ Input: The Audio feature accepts as input:
+ - A `str`: Absolute path to the audio file (i.e. random access is allowed).
+ - A `pathlib.Path`: path to the audio file (i.e. random access is allowed).
+ - A `dict` with the keys:
+
+ - `path`: String with relative path of the audio file to the archive file.
+ - `bytes`: Bytes content of the audio file.
+
+ This is useful for parquet or webdataset files which embed audio files.
+
+ - A `dict` with the keys:
+
+ - `array`: Array containing the audio sample
+ - `sampling_rate`: Integer corresponding to the sampling rate of the audio sample.
+
+ - A `torchcodec.decoders.AudioDecoder`: torchcodec audio decoder object.
+
+ Output: The Audio features output data as `torchcodec.decoders.AudioDecoder` objects, with additional keys:
+
+ - `array`: Array containing the audio sample
+ - `sampling_rate`: Integer corresponding to the sampling rate of the audio sample.
+
+ Args:
+ sampling_rate (`int`, *optional*):
+ Target sampling rate. If `None`, the native sampling rate is used.
+ num_channels (`int`, *optional*):
+ The desired number of channels of the samples. By default, the number of channels of the source is used.
+ Audio decoding will return samples with shape (num_channels, num_samples)
+ Currently `None` (number of channels of the source, default), `1` (mono) or `2` (stereo) channels are supported.
+ The `num_channels` argument is passed to `torchcodec.decoders.AudioDecoder`.
+
+
+ decode (`bool`, defaults to `True`):
+ Whether to decode the audio data. If `False`,
+ returns the underlying dictionary in the format `{"path": audio_path, "bytes": audio_bytes}`.
+ stream_index (`int`, *optional*):
+ The streaming index to use from the file. If `None` defaults to the "best" index.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset, Audio
+ >>> ds = load_dataset("PolyAI/minds14", name="en-US", split="train")
+ >>> ds = ds.cast_column("audio", Audio(sampling_rate=44100, num_channels=2))
+ >>> ds[0]["audio"]
+
+ >>> audio = ds[0]["audio"]
+ >>> audio.get_samples_played_in_range(0, 10)
+ AudioSamples:
+ data (shape): torch.Size([2, 110592])
+ pts_seconds: 0.0
+ duration_seconds: 2.507755102040816
+ sample_rate: 44100
+ ```
+ """
+
+ sampling_rate: Optional[int] = None
+ decode: bool = True
+ num_channels: Optional[int] = None
+ stream_index: Optional[int] = None
+ id: Optional[str] = field(default=None, repr=False)
+ # Automatically constructed
+ dtype: ClassVar[str] = "dict"
+ pa_type: ClassVar[Any] = pa.struct({"bytes": pa.binary(), "path": pa.string()})
+ _type: str = field(default="Audio", init=False, repr=False)
+
+ def __call__(self):
+ return self.pa_type
+
+ def encode_example(self, value: Union[str, bytes, bytearray, dict, "AudioDecoder"]) -> dict:
+ """Encode example into a format for Arrow.
+
+ Args:
+ value (`str`, `bytes`,`bytearray`,`dict`, `AudioDecoder`):
+ Data passed as input to Audio feature.
+
+ Returns:
+ `dict`
+ """
+ try:
+ import torch
+ from torchcodec.encoders import AudioEncoder # needed to write audio files
+ except ImportError as err:
+ raise ImportError("To support encoding audio data, please install 'torchcodec'.") from err
+
+ if value is None:
+ raise ValueError("value must be provided")
+
+ if config.TORCHCODEC_AVAILABLE:
+ from torchcodec.decoders import AudioDecoder
+
+ else:
+ AudioDecoder = None
+
+ if isinstance(value, str):
+ return {"bytes": None, "path": value}
+ elif isinstance(value, Path):
+ return {"bytes": None, "path": str(value.absolute())}
+ elif isinstance(value, (bytes, bytearray)):
+ return {"bytes": value, "path": None}
+ elif AudioDecoder is not None and isinstance(value, AudioDecoder):
+ return encode_torchcodec_audio(value)
+ elif "array" in value:
+ # convert the audio array to wav bytes
+ buffer = BytesIO()
+ AudioEncoder(
+ torch.from_numpy(value["array"].astype(np.float32)), sample_rate=value["sampling_rate"]
+ ).to_file_like(buffer, format="wav", num_channels=self.num_channels)
+ return {"bytes": buffer.getvalue(), "path": None}
+ elif value.get("path") is not None and os.path.isfile(value["path"]):
+ # we set "bytes": None to not duplicate the data if they're already available locally
+ if value["path"].endswith("pcm"):
+ # "PCM" only has raw audio bytes
+ if value.get("sampling_rate") is None:
+ # At least, If you want to convert "PCM-byte" to "WAV-byte", you have to know sampling rate
+ raise KeyError("To use PCM files, please specify a 'sampling_rate' in Audio object")
+ if value.get("bytes"):
+ # If we already had PCM-byte, we don`t have to make "read file, make bytes" (just use it!)
+ bytes_value = np.frombuffer(value["bytes"], dtype=np.int16).astype(np.float32) / 32767
+ else:
+ bytes_value = np.memmap(value["path"], dtype="h", mode="r").astype(np.float32) / 32767
+
+ buffer = BytesIO()
+ AudioEncoder(torch.from_numpy(bytes_value), sample_rate=value["sampling_rate"]).to_file_like(
+ buffer, format="wav", num_channels=self.num_channels
+ )
+ return {"bytes": buffer.getvalue(), "path": None}
+ else:
+ return {"bytes": None, "path": value.get("path")}
+ elif value.get("bytes") is not None or value.get("path") is not None:
+ # store the audio bytes, and path is used to infer the audio format using the file extension
+ return {"bytes": value.get("bytes"), "path": value.get("path")}
+ else:
+ raise ValueError(
+ f"An audio sample should have one of 'path' or 'bytes' but they are missing or None in {value}."
+ )
+
+ def decode_example(
+ self, value: dict, token_per_repo_id: Optional[dict[str, Union[str, bool, None]]] = None
+ ) -> "AudioDecoder":
+ """Decode example audio file into audio data.
+
+ Args:
+ value (`dict`):
+ A dictionary with keys:
+
+ - `path`: String with relative audio file path.
+ - `bytes`: Bytes of the audio file.
+ token_per_repo_id (`dict`, *optional*):
+ To access and decode
+ audio files from private repositories on the Hub, you can pass
+ a dictionary repo_id (`str`) -> token (`bool` or `str`)
+
+ Returns:
+ `torchcodec.decoders.AudioDecoder`
+ """
+ if config.TORCHCODEC_AVAILABLE:
+ from ._torchcodec import AudioDecoder
+ else:
+ raise ImportError("To support decoding audio data, please install 'torchcodec'.")
+
+ if not self.decode:
+ raise RuntimeError("Decoding is disabled for this feature. Please use Audio(decode=True) instead.")
+
+ path, bytes = (value["path"], value["bytes"]) if value["bytes"] is not None else (value["path"], None)
+ if path is None and bytes is None:
+ raise ValueError(f"An audio sample should have one of 'path' or 'bytes' but both are None in {value}.")
+
+ if bytes is None and is_local_path(path):
+ audio = AudioDecoder(
+ path, stream_index=self.stream_index, sample_rate=self.sampling_rate, num_channels=self.num_channels
+ )
+
+ elif bytes is None:
+ token_per_repo_id = token_per_repo_id or {}
+ source_url = path.split("::")[-1]
+ pattern = (
+ config.HUB_DATASETS_URL if source_url.startswith(config.HF_ENDPOINT) else config.HUB_DATASETS_HFFS_URL
+ )
+ source_url_fields = string_to_dict(source_url, pattern)
+ token = token_per_repo_id.get(source_url_fields["repo_id"]) if source_url_fields is not None else None
+
+ download_config = DownloadConfig(token=token)
+ f = xopen(path, "rb", download_config=download_config)
+ audio = AudioDecoder(
+ f, stream_index=self.stream_index, sample_rate=self.sampling_rate, num_channels=self.num_channels
+ )
+
+ else:
+ audio = AudioDecoder(
+ bytes, stream_index=self.stream_index, sample_rate=self.sampling_rate, num_channels=self.num_channels
+ )
+ audio._hf_encoded = {"path": path, "bytes": bytes}
+ audio.metadata.path = path
+ return audio
+
+ def flatten(self) -> Union["FeatureType", dict[str, "FeatureType"]]:
+ """If in the decodable state, raise an error, otherwise flatten the feature into a dictionary."""
+ from .features import Value
+
+ if self.decode:
+ raise ValueError("Cannot flatten a decoded Audio feature.")
+ return {
+ "bytes": Value("binary"),
+ "path": Value("string"),
+ }
+
+ def cast_storage(self, storage: Union[pa.StringArray, pa.StructArray]) -> pa.StructArray:
+ """Cast an Arrow array to the Audio arrow storage type.
+ The Arrow types that can be converted to the Audio pyarrow storage type are:
+
+ - `pa.string()` - it must contain the "path" data
+ - `pa.binary()` - it must contain the audio bytes
+ - `pa.struct({"bytes": pa.binary()})`
+ - `pa.struct({"path": pa.string()})`
+ - `pa.struct({"bytes": pa.binary(), "path": pa.string()})` - order doesn't matter
+
+ Args:
+ storage (`Union[pa.StringArray, pa.StructArray]`):
+ PyArrow array to cast.
+
+ Returns:
+ `pa.StructArray`: Array in the Audio arrow storage type, that is
+ `pa.struct({"bytes": pa.binary(), "path": pa.string()})`
+ """
+ if pa.types.is_string(storage.type):
+ bytes_array = pa.array([None] * len(storage), type=pa.binary())
+ storage = pa.StructArray.from_arrays([bytes_array, storage], ["bytes", "path"], mask=storage.is_null())
+ elif pa.types.is_binary(storage.type):
+ path_array = pa.array([None] * len(storage), type=pa.string())
+ storage = pa.StructArray.from_arrays([storage, path_array], ["bytes", "path"], mask=storage.is_null())
+ elif pa.types.is_struct(storage.type) and storage.type.get_all_field_indices("array"):
+ storage = pa.array(
+ [Audio().encode_example(x) if x is not None else None for x in storage.to_numpy(zero_copy_only=False)]
+ )
+ elif pa.types.is_struct(storage.type):
+ if storage.type.get_field_index("bytes") >= 0:
+ bytes_array = storage.field("bytes")
+ else:
+ bytes_array = pa.array([None] * len(storage), type=pa.binary())
+ if storage.type.get_field_index("path") >= 0:
+ path_array = storage.field("path")
+ else:
+ path_array = pa.array([None] * len(storage), type=pa.string())
+ storage = pa.StructArray.from_arrays([bytes_array, path_array], ["bytes", "path"], mask=storage.is_null())
+ return array_cast(storage, self.pa_type)
+
+ def embed_storage(self, storage: pa.StructArray, token_per_repo_id=None) -> pa.StructArray:
+ """Embed audio files into the Arrow array.
+
+ Args:
+ storage (`pa.StructArray`):
+ PyArrow array to embed.
+
+ Returns:
+ `pa.StructArray`: Array in the Audio arrow storage type, that is
+ `pa.struct({"bytes": pa.binary(), "path": pa.string()})`.
+ """
+ if token_per_repo_id is None:
+ token_per_repo_id = {}
+
+ @no_op_if_value_is_null
+ def path_to_bytes(path):
+ source_url = path.split("::")[-1]
+ pattern = (
+ config.HUB_DATASETS_URL if source_url.startswith(config.HF_ENDPOINT) else config.HUB_DATASETS_HFFS_URL
+ )
+ source_url_fields = string_to_dict(source_url, pattern)
+ token = token_per_repo_id.get(source_url_fields["repo_id"]) if source_url_fields is not None else None
+ download_config = DownloadConfig(token=token)
+ with xopen(path, "rb", download_config=download_config) as f:
+ return f.read()
+
+ bytes_array = pa.array(
+ [
+ (path_to_bytes(x["path"]) if x["bytes"] is None else x["bytes"]) if x is not None else None
+ for x in storage.to_pylist()
+ ],
+ type=pa.binary(),
+ )
+ path_array = pa.array(
+ [os.path.basename(path) if path is not None else None for path in storage.field("path").to_pylist()],
+ type=pa.string(),
+ )
+ storage = pa.StructArray.from_arrays([bytes_array, path_array], ["bytes", "path"], mask=bytes_array.is_null())
+ return array_cast(storage, self.pa_type)
+
+
+def encode_torchcodec_audio(audio: "AudioDecoder") -> dict:
+ if hasattr(audio, "_hf_encoded"):
+ return audio._hf_encoded
+ else:
+ try:
+ from torchcodec.encoders import AudioEncoder # needed to write audio files
+ except ImportError as err:
+ raise ImportError("To support encoding audio data, please install 'torchcodec'.") from err
+
+ samples = audio.get_all_samples()
+ buffer = BytesIO()
+ num_channels = samples.data.shape[0]
+ AudioEncoder(samples.data.cpu(), sample_rate=samples.sample_rate).to_file_like(
+ buffer, format="wav", num_channels=num_channels
+ )
+ return {"bytes": buffer.getvalue(), "path": None}
diff --git a/venv/lib/python3.10/site-packages/datasets/features/features.py b/venv/lib/python3.10/site-packages/datasets/features/features.py
new file mode 100644
index 0000000000000000000000000000000000000000..88259767ae08819f6d0f394ed2db402a6dcbe677
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/features/features.py
@@ -0,0 +1,2325 @@
+# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""This class handle features definition in datasets and some utilities to display table type."""
+
+import copy
+import json
+import re
+import sys
+from collections.abc import Iterable, Mapping
+from collections.abc import Sequence as SequenceABC
+from collections.abc import Sequence as Sequence_
+from dataclasses import InitVar, dataclass, field, fields
+from functools import reduce, wraps
+from operator import mul
+from typing import Any, Callable, ClassVar, Literal, Optional, Union
+
+import numpy as np
+import pandas as pd
+import pyarrow as pa
+import pyarrow.compute as pc
+import pyarrow.types
+from pandas.api.extensions import ExtensionArray as PandasExtensionArray
+from pandas.api.extensions import ExtensionDtype as PandasExtensionDtype
+
+from .. import config
+from ..naming import camelcase_to_snakecase, snakecase_to_camelcase
+from ..table import array_cast
+from ..utils import experimental, logging
+from ..utils.py_utils import asdict, first_non_null_value, zip_dict
+from .audio import Audio
+from .image import Image, encode_pil_image
+from .nifti import Nifti
+from .pdf import Pdf, encode_pdfplumber_pdf
+from .translation import Translation, TranslationVariableLanguages
+from .video import Video
+
+
+logger = logging.get_logger(__name__)
+
+
+def _arrow_to_datasets_dtype(arrow_type: pa.DataType) -> str:
+ """
+ _arrow_to_datasets_dtype takes a pyarrow.DataType and converts it to a datasets string dtype.
+ In effect, `dt == string_to_arrow(_arrow_to_datasets_dtype(dt))`
+ """
+ if pyarrow.types.is_null(arrow_type):
+ return "null"
+ elif pyarrow.types.is_boolean(arrow_type):
+ return "bool"
+ elif pyarrow.types.is_int8(arrow_type):
+ return "int8"
+ elif pyarrow.types.is_int16(arrow_type):
+ return "int16"
+ elif pyarrow.types.is_int32(arrow_type):
+ return "int32"
+ elif pyarrow.types.is_int64(arrow_type):
+ return "int64"
+ elif pyarrow.types.is_uint8(arrow_type):
+ return "uint8"
+ elif pyarrow.types.is_uint16(arrow_type):
+ return "uint16"
+ elif pyarrow.types.is_uint32(arrow_type):
+ return "uint32"
+ elif pyarrow.types.is_uint64(arrow_type):
+ return "uint64"
+ elif pyarrow.types.is_float16(arrow_type):
+ return "float16" # pyarrow dtype is "halffloat"
+ elif pyarrow.types.is_float32(arrow_type):
+ return "float32" # pyarrow dtype is "float"
+ elif pyarrow.types.is_float64(arrow_type):
+ return "float64" # pyarrow dtype is "double"
+ elif pyarrow.types.is_time32(arrow_type):
+ return f"time32[{pa.type_for_alias(str(arrow_type)).unit}]"
+ elif pyarrow.types.is_time64(arrow_type):
+ return f"time64[{pa.type_for_alias(str(arrow_type)).unit}]"
+ elif pyarrow.types.is_timestamp(arrow_type):
+ if arrow_type.tz is None:
+ return f"timestamp[{arrow_type.unit}]"
+ elif arrow_type.tz:
+ return f"timestamp[{arrow_type.unit}, tz={arrow_type.tz}]"
+ else:
+ raise ValueError(f"Unexpected timestamp object {arrow_type}.")
+ elif pyarrow.types.is_date32(arrow_type):
+ return "date32" # pyarrow dtype is "date32[day]"
+ elif pyarrow.types.is_date64(arrow_type):
+ return "date64" # pyarrow dtype is "date64[ms]"
+ elif pyarrow.types.is_duration(arrow_type):
+ return f"duration[{arrow_type.unit}]"
+ elif pyarrow.types.is_decimal128(arrow_type):
+ return f"decimal128({arrow_type.precision}, {arrow_type.scale})"
+ elif pyarrow.types.is_decimal256(arrow_type):
+ return f"decimal256({arrow_type.precision}, {arrow_type.scale})"
+ elif pyarrow.types.is_binary(arrow_type):
+ return "binary"
+ elif pyarrow.types.is_large_binary(arrow_type):
+ return "large_binary"
+ elif pyarrow.types.is_binary_view(arrow_type):
+ return "binary_view"
+ elif pyarrow.types.is_string(arrow_type):
+ return "string"
+ elif pyarrow.types.is_large_string(arrow_type):
+ return "large_string"
+ elif pyarrow.types.is_string_view(arrow_type):
+ return "string_view"
+ elif pyarrow.types.is_dictionary(arrow_type):
+ return _arrow_to_datasets_dtype(arrow_type.value_type)
+ else:
+ raise ValueError(f"Arrow type {arrow_type} does not have a datasets dtype equivalent.")
+
+
+def string_to_arrow(datasets_dtype: str) -> pa.DataType:
+ """
+ string_to_arrow takes a datasets string dtype and converts it to a pyarrow.DataType.
+
+ In effect, `dt == string_to_arrow(_arrow_to_datasets_dtype(dt))`
+
+ This is necessary because the datasets.Value() primitive type is constructed using a string dtype
+
+ Value(dtype=str)
+
+ But Features.type (via `get_nested_type()` expects to resolve Features into a pyarrow Schema,
+ which means that each Value() must be able to resolve into a corresponding pyarrow.DataType, which is the
+ purpose of this function.
+ """
+
+ def _dtype_error_msg(dtype, pa_dtype, examples=None, urls=None):
+ msg = f"{dtype} is not a validly formatted string representation of the pyarrow {pa_dtype} type."
+ if examples:
+ examples = ", ".join(examples[:-1]) + " or " + examples[-1] if len(examples) > 1 else examples[0]
+ msg += f"\nValid examples include: {examples}."
+ if urls:
+ urls = ", ".join(urls[:-1]) + " and " + urls[-1] if len(urls) > 1 else urls[0]
+ msg += f"\nFor more insformation, see: {urls}."
+ return msg
+
+ if datasets_dtype in pa.__dict__:
+ return pa.__dict__[datasets_dtype]()
+
+ if (datasets_dtype + "_") in pa.__dict__:
+ return pa.__dict__[datasets_dtype + "_"]()
+
+ timestamp_matches = re.search(r"^timestamp\[(.*)\]$", datasets_dtype)
+ if timestamp_matches:
+ timestamp_internals = timestamp_matches.group(1)
+ internals_matches = re.search(r"^(s|ms|us|ns),\s*tz=([a-zA-Z0-9/_+\-:]*)$", timestamp_internals)
+ if timestamp_internals in ["s", "ms", "us", "ns"]:
+ return pa.timestamp(timestamp_internals)
+ elif internals_matches:
+ return pa.timestamp(internals_matches.group(1), internals_matches.group(2))
+ else:
+ raise ValueError(
+ _dtype_error_msg(
+ datasets_dtype,
+ "timestamp",
+ examples=["timestamp[us]", "timestamp[us, tz=America/New_York"],
+ urls=["https://arrow.apache.org/docs/python/generated/pyarrow.timestamp.html"],
+ )
+ )
+
+ duration_matches = re.search(r"^duration\[(.*)\]$", datasets_dtype)
+ if duration_matches:
+ duration_internals = duration_matches.group(1)
+ if duration_internals in ["s", "ms", "us", "ns"]:
+ return pa.duration(duration_internals)
+ else:
+ raise ValueError(
+ _dtype_error_msg(
+ datasets_dtype,
+ "duration",
+ examples=["duration[s]", "duration[us]"],
+ urls=["https://arrow.apache.org/docs/python/generated/pyarrow.duration.html"],
+ )
+ )
+
+ time_matches = re.search(r"^time(.*)\[(.*)\]$", datasets_dtype)
+ if time_matches:
+ time_internals_bits = time_matches.group(1)
+ if time_internals_bits == "32":
+ time_internals_unit = time_matches.group(2)
+ if time_internals_unit in ["s", "ms"]:
+ return pa.time32(time_internals_unit)
+ else:
+ raise ValueError(
+ f"{time_internals_unit} is not a valid unit for the pyarrow time32 type. Supported units: s (second) and ms (millisecond)."
+ )
+ elif time_internals_bits == "64":
+ time_internals_unit = time_matches.group(2)
+ if time_internals_unit in ["us", "ns"]:
+ return pa.time64(time_internals_unit)
+ else:
+ raise ValueError(
+ f"{time_internals_unit} is not a valid unit for the pyarrow time64 type. Supported units: us (microsecond) and ns (nanosecond)."
+ )
+ else:
+ raise ValueError(
+ _dtype_error_msg(
+ datasets_dtype,
+ "time",
+ examples=["time32[s]", "time64[us]"],
+ urls=[
+ "https://arrow.apache.org/docs/python/generated/pyarrow.time32.html",
+ "https://arrow.apache.org/docs/python/generated/pyarrow.time64.html",
+ ],
+ )
+ )
+
+ decimal_matches = re.search(r"^decimal(.*)\((.*)\)$", datasets_dtype)
+ if decimal_matches:
+ decimal_internals_bits = decimal_matches.group(1)
+ if decimal_internals_bits == "128":
+ decimal_internals_precision_and_scale = re.search(r"^(\d+),\s*(-?\d+)$", decimal_matches.group(2))
+ if decimal_internals_precision_and_scale:
+ precision = decimal_internals_precision_and_scale.group(1)
+ scale = decimal_internals_precision_and_scale.group(2)
+ return pa.decimal128(int(precision), int(scale))
+ else:
+ raise ValueError(
+ _dtype_error_msg(
+ datasets_dtype,
+ "decimal128",
+ examples=["decimal128(10, 2)", "decimal128(4, -2)"],
+ urls=["https://arrow.apache.org/docs/python/generated/pyarrow.decimal128.html"],
+ )
+ )
+ elif decimal_internals_bits == "256":
+ decimal_internals_precision_and_scale = re.search(r"^(\d+),\s*(-?\d+)$", decimal_matches.group(2))
+ if decimal_internals_precision_and_scale:
+ precision = decimal_internals_precision_and_scale.group(1)
+ scale = decimal_internals_precision_and_scale.group(2)
+ return pa.decimal256(int(precision), int(scale))
+ else:
+ raise ValueError(
+ _dtype_error_msg(
+ datasets_dtype,
+ "decimal256",
+ examples=["decimal256(30, 2)", "decimal256(38, -4)"],
+ urls=["https://arrow.apache.org/docs/python/generated/pyarrow.decimal256.html"],
+ )
+ )
+ else:
+ raise ValueError(
+ _dtype_error_msg(
+ datasets_dtype,
+ "decimal",
+ examples=["decimal128(12, 3)", "decimal256(40, 6)"],
+ urls=[
+ "https://arrow.apache.org/docs/python/generated/pyarrow.decimal128.html",
+ "https://arrow.apache.org/docs/python/generated/pyarrow.decimal256.html",
+ ],
+ )
+ )
+
+ raise ValueError(
+ f"Neither {datasets_dtype} nor {datasets_dtype + '_'} seems to be a pyarrow data type. "
+ f"Please make sure to use a correct data type, see: "
+ f"https://arrow.apache.org/docs/python/api/datatypes.html#factory-functions"
+ )
+
+
+def _cast_to_python_objects(obj: Any, only_1d_for_numpy: bool, optimize_list_casting: bool) -> tuple[Any, bool]:
+ """
+ Cast pytorch/tensorflow/pandas objects to python numpy array/lists.
+ It works recursively.
+
+ If `optimize_list_casting` is True, to avoid iterating over possibly long lists, it first checks (recursively) if the first element that is not None or empty (if it is a sequence) has to be casted.
+ If the first element needs to be casted, then all the elements of the list will be casted, otherwise they'll stay the same.
+ This trick allows to cast objects that contain tokenizers outputs without iterating over every single token for example.
+
+ Args:
+ obj: the object (nested struct) to cast.
+ only_1d_for_numpy (bool): whether to keep the full multi-dim tensors as multi-dim numpy arrays, or convert them to
+ nested lists of 1-dimensional numpy arrays. This can be useful to keep only 1-d arrays to instantiate Arrow arrays.
+ Indeed Arrow only support converting 1-dimensional array values.
+ optimize_list_casting (bool): whether to optimize list casting by checking the first non-null element to see if it needs to be casted
+ and if it doesn't, not checking the rest of the list elements.
+
+ Returns:
+ casted_obj: the casted object
+ has_changed (bool): True if the object has been changed, False if it is identical
+ """
+
+ if config.TF_AVAILABLE and "tensorflow" in sys.modules:
+ import tensorflow as tf
+
+ if config.TORCH_AVAILABLE and "torch" in sys.modules:
+ import torch
+
+ if config.JAX_AVAILABLE and "jax" in sys.modules:
+ import jax.numpy as jnp
+
+ if config.PIL_AVAILABLE and "PIL" in sys.modules:
+ import PIL.Image
+
+ if config.PDFPLUMBER_AVAILABLE and "pdfplumber" in sys.modules:
+ import pdfplumber
+
+ if config.TORCHCODEC_AVAILABLE and "torchcodec" in sys.modules:
+ from torchcodec.decoders import AudioDecoder, VideoDecoder
+
+ if isinstance(obj, np.ndarray):
+ if obj.ndim == 0:
+ return obj[()], True
+ elif not only_1d_for_numpy or obj.ndim == 1:
+ return obj, False
+ else:
+ return (
+ [
+ _cast_to_python_objects(
+ x, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting
+ )[0]
+ for x in obj
+ ],
+ True,
+ )
+ elif config.TORCH_AVAILABLE and "torch" in sys.modules and isinstance(obj, torch.Tensor):
+ if obj.dtype == torch.bfloat16:
+ return _cast_to_python_objects(
+ obj.detach().to(torch.float).cpu().numpy(),
+ only_1d_for_numpy=only_1d_for_numpy,
+ optimize_list_casting=optimize_list_casting,
+ )[0], True
+ if obj.ndim == 0:
+ return obj.detach().cpu().numpy()[()], True
+ elif not only_1d_for_numpy or obj.ndim == 1:
+ return obj.detach().cpu().numpy(), True
+ else:
+ return (
+ [
+ _cast_to_python_objects(
+ x, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting
+ )[0]
+ for x in obj.detach().cpu().numpy()
+ ],
+ True,
+ )
+ elif config.TF_AVAILABLE and "tensorflow" in sys.modules and isinstance(obj, tf.Tensor):
+ if obj.ndim == 0:
+ return obj.numpy()[()], True
+ elif not only_1d_for_numpy or obj.ndim == 1:
+ return obj.numpy(), True
+ else:
+ return (
+ [
+ _cast_to_python_objects(
+ x, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting
+ )[0]
+ for x in obj.numpy()
+ ],
+ True,
+ )
+ elif config.JAX_AVAILABLE and "jax" in sys.modules and isinstance(obj, jnp.ndarray):
+ if obj.ndim == 0:
+ return np.asarray(obj)[()], True
+ elif not only_1d_for_numpy or obj.ndim == 1:
+ return np.asarray(obj), True
+ else:
+ return (
+ [
+ _cast_to_python_objects(
+ x, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting
+ )[0]
+ for x in np.asarray(obj)
+ ],
+ True,
+ )
+ elif config.PIL_AVAILABLE and "PIL" in sys.modules and isinstance(obj, PIL.Image.Image):
+ return encode_pil_image(obj), True
+ elif config.PDFPLUMBER_AVAILABLE and "pdfplumber" in sys.modules and isinstance(obj, pdfplumber.pdf.PDF):
+ return encode_pdfplumber_pdf(obj), True
+ elif isinstance(obj, pd.Series):
+ return (
+ _cast_to_python_objects(
+ obj.tolist(), only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting
+ )[0],
+ True,
+ )
+ elif isinstance(obj, pd.DataFrame):
+ return (
+ {
+ key: _cast_to_python_objects(
+ value, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting
+ )[0]
+ for key, value in obj.to_dict("series").items()
+ },
+ True,
+ )
+ elif isinstance(obj, pd.Timestamp):
+ return obj.to_pydatetime(), True
+ elif isinstance(obj, pd.Timedelta):
+ return obj.to_pytimedelta(), True
+ elif isinstance(obj, Mapping):
+ has_changed = not isinstance(obj, dict)
+ output = {}
+ for k, v in obj.items():
+ casted_v, has_changed_v = _cast_to_python_objects(
+ v, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting
+ )
+ has_changed |= has_changed_v
+ output[k] = casted_v
+ return output if has_changed else obj, has_changed
+ elif hasattr(obj, "__array__"):
+ if np.isscalar(obj):
+ return obj, False
+ else:
+ return (
+ _cast_to_python_objects(
+ obj.__array__(), only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting
+ )[0],
+ True,
+ )
+ elif isinstance(obj, (list, tuple)):
+ if len(obj) > 0:
+ for first_elmt in obj:
+ if _check_non_null_non_empty_recursive(first_elmt):
+ break
+ casted_first_elmt, has_changed_first_elmt = _cast_to_python_objects(
+ first_elmt, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting
+ )
+ if has_changed_first_elmt or not optimize_list_casting:
+ return (
+ [
+ _cast_to_python_objects(
+ elmt, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting
+ )[0]
+ for elmt in obj
+ ],
+ True,
+ )
+ else:
+ if isinstance(obj, (list, tuple)):
+ return obj, False
+ else:
+ return list(obj), True
+ else:
+ return obj, False
+ elif config.TORCHCODEC_AVAILABLE and "torchcodec" in sys.modules and isinstance(obj, VideoDecoder):
+ v = Video()
+ return v.encode_example(obj), True
+ elif config.TORCHCODEC_AVAILABLE and "torchcodec" in sys.modules and isinstance(obj, AudioDecoder):
+ a = Audio()
+ return a.encode_example(obj), True
+ else:
+ return obj, False
+
+
+def cast_to_python_objects(obj: Any, only_1d_for_numpy=False, optimize_list_casting=True) -> Any:
+ """
+ Cast numpy/pytorch/tensorflow/pandas objects to python lists.
+ It works recursively.
+
+ If `optimize_list_casting` is True, To avoid iterating over possibly long lists, it first checks (recursively) if the first element that is not None or empty (if it is a sequence) has to be casted.
+ If the first element needs to be casted, then all the elements of the list will be casted, otherwise they'll stay the same.
+ This trick allows to cast objects that contain tokenizers outputs without iterating over every single token for example.
+
+ Args:
+ obj: the object (nested struct) to cast
+ only_1d_for_numpy (bool, default ``False``): whether to keep the full multi-dim tensors as multi-dim numpy arrays, or convert them to
+ nested lists of 1-dimensional numpy arrays. This can be useful to keep only 1-d arrays to instantiate Arrow arrays.
+ Indeed Arrow only support converting 1-dimensional array values.
+ optimize_list_casting (bool, default ``True``): whether to optimize list casting by checking the first non-null element to see if it needs to be casted
+ and if it doesn't, not checking the rest of the list elements.
+
+ Returns:
+ casted_obj: the casted object
+ """
+ return _cast_to_python_objects(
+ obj, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting
+ )[0]
+
+
+@dataclass(repr=False)
+class Value:
+ """
+ Scalar feature value of a particular data type.
+
+ The possible dtypes of `Value` are as follows:
+ - `null`
+ - `bool`
+ - `int8`
+ - `int16`
+ - `int32`
+ - `int64`
+ - `uint8`
+ - `uint16`
+ - `uint32`
+ - `uint64`
+ - `float16`
+ - `float32` (alias float)
+ - `float64` (alias double)
+ - `time32[(s|ms)]`
+ - `time64[(us|ns)]`
+ - `timestamp[(s|ms|us|ns)]`
+ - `timestamp[(s|ms|us|ns), tz=(tzstring)]`
+ - `date32`
+ - `date64`
+ - `duration[(s|ms|us|ns)]`
+ - `decimal128(precision, scale)`
+ - `decimal256(precision, scale)`
+ - `binary`
+ - `large_binary`
+ - `binary_view`
+ - `string`
+ - `large_string`
+ - `string_view`
+
+ Args:
+ dtype (`str`):
+ Name of the data type.
+
+ Example:
+
+ ```py
+ >>> from datasets import Features
+ >>> features = Features({'stars': Value('int32')})
+ >>> features
+ {'stars': Value('int32')}
+ ```
+ """
+
+ dtype: str
+ id: Optional[str] = field(default=None, repr=False)
+ # Automatically constructed
+ pa_type: ClassVar[Any] = None
+ _type: str = field(default="Value", init=False, repr=False)
+
+ def __post_init__(self):
+ if self.dtype == "double": # fix inferred type
+ self.dtype = "float64"
+ if self.dtype == "float": # fix inferred type
+ self.dtype = "float32"
+ self.pa_type = string_to_arrow(self.dtype)
+
+ def __call__(self):
+ return self.pa_type
+
+ def encode_example(self, value):
+ if pa.types.is_boolean(self.pa_type):
+ return bool(value)
+ elif pa.types.is_integer(self.pa_type):
+ return int(value)
+ elif pa.types.is_floating(self.pa_type):
+ return float(value)
+ elif pa.types.is_string(self.pa_type):
+ return str(value)
+ elif pa.types.is_large_string(self.pa_type):
+ return str(value)
+ elif pa.types.is_string_view(self.pa_type):
+ return str(value)
+ else:
+ return value
+
+ def __repr__(self):
+ return f"{type(self).__name__}('{self.dtype}')"
+
+
+class _ArrayXD:
+ def __post_init__(self):
+ self.shape = tuple(self.shape)
+
+ def __call__(self):
+ pa_type = globals()[self.__class__.__name__ + "ExtensionType"](self.shape, self.dtype)
+ return pa_type
+
+ def encode_example(self, value):
+ return value
+
+
+@dataclass
+class Array2D(_ArrayXD):
+ """Create a two-dimensional array.
+
+ Args:
+ shape (`tuple`):
+ Size of each dimension.
+ dtype (`str`):
+ Name of the data type.
+
+ Example:
+
+ ```py
+ >>> from datasets import Features
+ >>> features = Features({'x': Array2D(shape=(1, 3), dtype='int32')})
+ ```
+ """
+
+ shape: tuple
+ dtype: str
+ id: Optional[str] = field(default=None, repr=False)
+ # Automatically constructed
+ _type: str = field(default="Array2D", init=False, repr=False)
+
+
+@dataclass
+class Array3D(_ArrayXD):
+ """Create a three-dimensional array.
+
+ Args:
+ shape (`tuple`):
+ Size of each dimension.
+ dtype (`str`):
+ Name of the data type.
+
+ Example:
+
+ ```py
+ >>> from datasets import Features
+ >>> features = Features({'x': Array3D(shape=(1, 2, 3), dtype='int32')})
+ ```
+ """
+
+ shape: tuple
+ dtype: str
+ id: Optional[str] = field(default=None, repr=False)
+ # Automatically constructed
+ _type: str = field(default="Array3D", init=False, repr=False)
+
+
+@dataclass
+class Array4D(_ArrayXD):
+ """Create a four-dimensional array.
+
+ Args:
+ shape (`tuple`):
+ Size of each dimension.
+ dtype (`str`):
+ Name of the data type.
+
+ Example:
+
+ ```py
+ >>> from datasets import Features
+ >>> features = Features({'x': Array4D(shape=(1, 2, 2, 3), dtype='int32')})
+ ```
+ """
+
+ shape: tuple
+ dtype: str
+ id: Optional[str] = field(default=None, repr=False)
+ # Automatically constructed
+ _type: str = field(default="Array4D", init=False, repr=False)
+
+
+@dataclass
+class Array5D(_ArrayXD):
+ """Create a five-dimensional array.
+
+ Args:
+ shape (`tuple`):
+ Size of each dimension.
+ dtype (`str`):
+ Name of the data type.
+
+ Example:
+
+ ```py
+ >>> from datasets import Features
+ >>> features = Features({'x': Array5D(shape=(1, 2, 2, 3, 3), dtype='int32')})
+ ```
+ """
+
+ shape: tuple
+ dtype: str
+ id: Optional[str] = field(default=None, repr=False)
+ # Automatically constructed
+ _type: str = field(default="Array5D", init=False, repr=False)
+
+
+class _ArrayXDExtensionType(pa.ExtensionType):
+ ndims: Optional[int] = None
+
+ def __init__(self, shape: tuple, dtype: str):
+ if self.ndims is None or self.ndims <= 1:
+ raise ValueError("You must instantiate an array type with a value for dim that is > 1")
+ if len(shape) != self.ndims:
+ raise ValueError(f"shape={shape} and ndims={self.ndims} don't match")
+ for dim in range(1, self.ndims):
+ if shape[dim] is None:
+ raise ValueError(f"Support only dynamic size on first dimension. Got: {shape}")
+ self.shape = tuple(shape)
+ self.value_type = dtype
+ self.storage_dtype = self._generate_dtype(self.value_type)
+ pa.ExtensionType.__init__(self, self.storage_dtype, f"{self.__class__.__module__}.{self.__class__.__name__}")
+
+ def __arrow_ext_serialize__(self):
+ return json.dumps((self.shape, self.value_type)).encode()
+
+ @classmethod
+ def __arrow_ext_deserialize__(cls, storage_type, serialized):
+ args = json.loads(serialized)
+ return cls(*args)
+
+ # This was added to pa.ExtensionType in pyarrow >= 13.0.0
+ def __reduce__(self):
+ return self.__arrow_ext_deserialize__, (self.storage_type, self.__arrow_ext_serialize__())
+
+ def __hash__(self):
+ return hash((self.__class__, self.shape, self.value_type))
+
+ def __arrow_ext_class__(self):
+ return ArrayExtensionArray
+
+ def _generate_dtype(self, dtype):
+ dtype = string_to_arrow(dtype)
+ for d in reversed(self.shape):
+ dtype = pa.list_(dtype)
+ # Don't specify the size of the list, since fixed length list arrays have issues
+ # being validated after slicing in pyarrow 0.17.1
+ return dtype
+
+ def to_pandas_dtype(self):
+ return PandasArrayExtensionDtype(self.value_type)
+
+
+class Array2DExtensionType(_ArrayXDExtensionType):
+ ndims = 2
+
+
+class Array3DExtensionType(_ArrayXDExtensionType):
+ ndims = 3
+
+
+class Array4DExtensionType(_ArrayXDExtensionType):
+ ndims = 4
+
+
+class Array5DExtensionType(_ArrayXDExtensionType):
+ ndims = 5
+
+
+# Register the extension types for deserialization
+pa.register_extension_type(Array2DExtensionType((1, 2), "int64"))
+pa.register_extension_type(Array3DExtensionType((1, 2, 3), "int64"))
+pa.register_extension_type(Array4DExtensionType((1, 2, 3, 4), "int64"))
+pa.register_extension_type(Array5DExtensionType((1, 2, 3, 4, 5), "int64"))
+
+
+def _is_zero_copy_only(pa_type: pa.DataType, unnest: bool = False) -> bool:
+ """
+ When converting a pyarrow array to a numpy array, we must know whether this could be done in zero-copy or not.
+ This function returns the value of the ``zero_copy_only`` parameter to pass to ``.to_numpy()``, given the type of the pyarrow array.
+
+ # zero copy is available for all primitive types except booleans and temporal types (date, time, timestamp or duration)
+ # primitive types are types for which the physical representation in arrow and in numpy
+ # https://github.com/wesm/arrow/blob/c07b9b48cf3e0bbbab493992a492ae47e5b04cad/python/pyarrow/types.pxi#L821
+ # see https://arrow.apache.org/docs/python/generated/pyarrow.Array.html#pyarrow.Array.to_numpy
+ # and https://issues.apache.org/jira/browse/ARROW-2871?jql=text%20~%20%22boolean%20to_numpy%22
+ """
+
+ def _unnest_pa_type(pa_type: pa.DataType) -> pa.DataType:
+ if pa.types.is_list(pa_type):
+ return _unnest_pa_type(pa_type.value_type)
+ return pa_type
+
+ if unnest:
+ pa_type = _unnest_pa_type(pa_type)
+ return pa.types.is_primitive(pa_type) and not (pa.types.is_boolean(pa_type) or pa.types.is_temporal(pa_type))
+
+
+class ArrayExtensionArray(pa.ExtensionArray):
+ def __array__(self):
+ zero_copy_only = _is_zero_copy_only(self.storage.type, unnest=True)
+ return self.to_numpy(zero_copy_only=zero_copy_only)
+
+ def __getitem__(self, i):
+ return self.storage[i]
+
+ def to_numpy(self, zero_copy_only=True):
+ storage: pa.ListArray = self.storage
+ null_mask = storage.is_null().to_numpy(zero_copy_only=False)
+
+ if self.type.shape[0] is not None:
+ size = 1
+ null_indices = np.arange(len(storage))[null_mask] - np.arange(np.sum(null_mask))
+
+ for i in range(self.type.ndims):
+ size *= self.type.shape[i]
+ storage = storage.flatten()
+ numpy_arr = storage.to_numpy(zero_copy_only=zero_copy_only)
+ numpy_arr = numpy_arr.reshape(len(self) - len(null_indices), *self.type.shape)
+
+ if len(null_indices):
+ numpy_arr = np.insert(numpy_arr.astype(np.float64), null_indices, np.nan, axis=0)
+
+ else:
+ shape = self.type.shape
+ ndims = self.type.ndims
+ arrays = []
+ first_dim_offsets = np.array([off.as_py() for off in storage.offsets])
+ for i, is_null in enumerate(null_mask):
+ if is_null:
+ arrays.append(np.nan)
+ else:
+ storage_el = storage[i : i + 1]
+ first_dim = first_dim_offsets[i + 1] - first_dim_offsets[i]
+ # flatten storage
+ for _ in range(ndims):
+ storage_el = storage_el.flatten()
+
+ numpy_arr = storage_el.to_numpy(zero_copy_only=zero_copy_only)
+ arrays.append(numpy_arr.reshape(first_dim, *shape[1:]))
+
+ if len(np.unique(np.diff(first_dim_offsets))) > 1:
+ # ragged
+ numpy_arr = np.empty(len(arrays), dtype=object)
+ numpy_arr[:] = arrays
+ else:
+ numpy_arr = np.array(arrays)
+
+ return numpy_arr
+
+ def to_pylist(self, maps_as_pydicts: Optional[Literal["lossy", "strict"]] = None):
+ zero_copy_only = _is_zero_copy_only(self.storage.type, unnest=True)
+ numpy_arr = self.to_numpy(zero_copy_only=zero_copy_only)
+ if self.type.shape[0] is None and numpy_arr.dtype == object:
+ return [arr.tolist() for arr in numpy_arr.tolist()]
+ else:
+ return numpy_arr.tolist()
+
+
+class PandasArrayExtensionDtype(PandasExtensionDtype):
+ _metadata = "value_type"
+
+ def __init__(self, value_type: Union["PandasArrayExtensionDtype", np.dtype]):
+ self._value_type = value_type
+
+ def __from_arrow__(self, array: Union[pa.Array, pa.ChunkedArray]):
+ if isinstance(array, pa.ChunkedArray):
+ array = array.type.wrap_array(pa.concat_arrays([chunk.storage for chunk in array.chunks]))
+ zero_copy_only = _is_zero_copy_only(array.storage.type, unnest=True)
+ numpy_arr = array.to_numpy(zero_copy_only=zero_copy_only)
+ return PandasArrayExtensionArray(numpy_arr)
+
+ @classmethod
+ def construct_array_type(cls):
+ return PandasArrayExtensionArray
+
+ @property
+ def type(self) -> type:
+ return np.ndarray
+
+ @property
+ def kind(self) -> str:
+ return "O"
+
+ @property
+ def name(self) -> str:
+ return f"array[{self.value_type}]"
+
+ @property
+ def value_type(self) -> np.dtype:
+ return self._value_type
+
+
+class PandasArrayExtensionArray(PandasExtensionArray):
+ def __init__(self, data: np.ndarray, copy: bool = False):
+ self._data = data if not copy else np.array(data)
+ self._dtype = PandasArrayExtensionDtype(data.dtype)
+
+ def __array__(self, dtype=None):
+ """
+ Convert to NumPy Array.
+ Note that Pandas expects a 1D array when dtype is set to object.
+ But for other dtypes, the returned shape is the same as the one of ``data``.
+
+ More info about pandas 1D requirement for PandasExtensionArray here:
+ https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.api.extensions.ExtensionArray.html#pandas.api.extensions.ExtensionArray
+
+ """
+ if dtype == np.dtype(object):
+ out = np.empty(len(self._data), dtype=object)
+ for i in range(len(self._data)):
+ out[i] = self._data[i]
+ return out
+ if dtype is None:
+ return self._data
+ else:
+ return self._data.astype(dtype)
+
+ def copy(self, deep: bool = False) -> "PandasArrayExtensionArray":
+ return PandasArrayExtensionArray(self._data, copy=True)
+
+ @classmethod
+ def _from_sequence(
+ cls, scalars, dtype: Optional[PandasArrayExtensionDtype] = None, copy: bool = False
+ ) -> "PandasArrayExtensionArray":
+ if len(scalars) > 1 and all(
+ isinstance(x, np.ndarray) and x.shape == scalars[0].shape and x.dtype == scalars[0].dtype for x in scalars
+ ):
+ data = np.array(scalars, dtype=dtype if dtype is None else dtype.value_type, copy=copy)
+ else:
+ data = np.empty(len(scalars), dtype=object)
+ data[:] = scalars
+ return cls(data, copy=copy)
+
+ @classmethod
+ def _concat_same_type(cls, to_concat: Sequence_["PandasArrayExtensionArray"]) -> "PandasArrayExtensionArray":
+ if len(to_concat) > 1 and all(
+ va._data.shape == to_concat[0]._data.shape and va._data.dtype == to_concat[0]._data.dtype
+ for va in to_concat
+ ):
+ data = np.vstack([va._data for va in to_concat])
+ else:
+ data = np.empty(len(to_concat), dtype=object)
+ data[:] = [va._data for va in to_concat]
+ return cls(data, copy=False)
+
+ @property
+ def dtype(self) -> PandasArrayExtensionDtype:
+ return self._dtype
+
+ @property
+ def nbytes(self) -> int:
+ return self._data.nbytes
+
+ def isna(self) -> np.ndarray:
+ return np.array([pd.isna(arr).any() for arr in self._data])
+
+ def __setitem__(self, key: Union[int, slice, np.ndarray], value: Any) -> None:
+ raise NotImplementedError()
+
+ def __getitem__(self, item: Union[int, slice, np.ndarray]) -> Union[np.ndarray, "PandasArrayExtensionArray"]:
+ if isinstance(item, int):
+ return self._data[item]
+ return PandasArrayExtensionArray(self._data[item], copy=False)
+
+ def take(
+ self, indices: Sequence_[int], allow_fill: bool = False, fill_value: bool = None
+ ) -> "PandasArrayExtensionArray":
+ indices: np.ndarray = np.asarray(indices, dtype=int)
+ if allow_fill:
+ fill_value = (
+ self.dtype.na_value if fill_value is None else np.asarray(fill_value, dtype=self.dtype.value_type)
+ )
+ mask = indices == -1
+ if (indices < -1).any():
+ raise ValueError("Invalid value in `indices`, must be all >= -1 for `allow_fill` is True")
+ elif len(self) > 0:
+ pass
+ elif not np.all(mask):
+ raise IndexError("Invalid take for empty PandasArrayExtensionArray, must be all -1.")
+ else:
+ data = np.array([fill_value] * len(indices), dtype=self.dtype.value_type)
+ return PandasArrayExtensionArray(data, copy=False)
+ took = self._data.take(indices, axis=0)
+ if allow_fill and mask.any():
+ took[mask] = [fill_value] * np.sum(mask)
+ return PandasArrayExtensionArray(took, copy=False)
+
+ def __len__(self) -> int:
+ return len(self._data)
+
+ def __eq__(self, other) -> np.ndarray:
+ if not isinstance(other, PandasArrayExtensionArray):
+ raise NotImplementedError(f"Invalid type to compare to: {type(other)}")
+ return (self._data == other._data).all()
+
+
+def pandas_types_mapper(dtype):
+ if isinstance(dtype, _ArrayXDExtensionType):
+ return PandasArrayExtensionDtype(dtype.value_type)
+
+
+@dataclass
+class ClassLabel:
+ """Feature type for integer class labels.
+
+ There are 3 ways to define a `ClassLabel`, which correspond to the 3 arguments:
+
+ * `num_classes`: Create 0 to (num_classes-1) labels.
+ * `names`: List of label strings.
+ * `names_file`: File containing the list of labels.
+
+ Under the hood the labels are stored as integers.
+ You can use negative integers to represent unknown/missing labels.
+
+ Args:
+ num_classes (`int`, *optional*):
+ Number of classes. All labels must be < `num_classes`.
+ names (`list` of `str`, *optional*):
+ String names for the integer classes.
+ The order in which the names are provided is kept.
+ names_file (`str`, *optional*):
+ Path to a file with names for the integer classes, one per line.
+
+ Example:
+
+ ```py
+ >>> from datasets import Features, ClassLabel
+ >>> features = Features({'label': ClassLabel(num_classes=3, names=['bad', 'ok', 'good'])})
+ >>> features
+ {'label': ClassLabel(names=['bad', 'ok', 'good'])}
+ ```
+ """
+
+ num_classes: InitVar[Optional[int]] = None # Pseudo-field: ignored by asdict/fields when converting to/from dict
+ names: list[str] = None
+ names_file: InitVar[Optional[str]] = None # Pseudo-field: ignored by asdict/fields when converting to/from dict
+ id: Optional[str] = field(default=None, repr=False)
+ # Automatically constructed
+ dtype: ClassVar[str] = "int64"
+ pa_type: ClassVar[Any] = pa.int64()
+ _str2int: ClassVar[dict[str, int]] = None
+ _int2str: ClassVar[dict[int, int]] = None
+ _type: str = field(default="ClassLabel", init=False, repr=False)
+
+ def __post_init__(self, num_classes, names_file):
+ self.num_classes = num_classes
+ self.names_file = names_file
+ if self.names_file is not None and self.names is not None:
+ raise ValueError("Please provide either names or names_file but not both.")
+ # Set self.names
+ if self.names is None:
+ if self.names_file is not None:
+ self.names = self._load_names_from_file(self.names_file)
+ elif self.num_classes is not None:
+ self.names = [str(i) for i in range(self.num_classes)]
+ else:
+ raise ValueError("Please provide either num_classes, names or names_file.")
+ elif not isinstance(self.names, SequenceABC):
+ raise TypeError(f"Please provide names as a list, is {type(self.names)}")
+ # Set self.num_classes
+ if self.num_classes is None:
+ self.num_classes = len(self.names)
+ elif self.num_classes != len(self.names):
+ raise ValueError(
+ "ClassLabel number of names do not match the defined num_classes. "
+ f"Got {len(self.names)} names VS {self.num_classes} num_classes"
+ )
+ # Prepare mappings
+ self._int2str = [str(name) for name in self.names]
+ self._str2int = {name: i for i, name in enumerate(self._int2str)}
+ if len(self._int2str) != len(self._str2int):
+ raise ValueError("Some label names are duplicated. Each label name should be unique.")
+
+ def __call__(self):
+ return self.pa_type
+
+ def str2int(self, values: Union[str, Iterable]) -> Union[int, Iterable]:
+ """Conversion class name `string` => `integer`.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train")
+ >>> ds.features["label"].str2int('neg')
+ 0
+ ```
+ """
+ if not isinstance(values, str) and not isinstance(values, Iterable):
+ raise ValueError(
+ f"Values {values} should be a string or an Iterable (list, numpy array, pytorch, tensorflow tensors)"
+ )
+ return_list = True
+ if isinstance(values, str):
+ values = [values]
+ return_list = False
+
+ output = [self._strval2int(value) for value in values]
+ return output if return_list else output[0]
+
+ def _strval2int(self, value: str) -> int:
+ failed_parse = False
+ value = str(value)
+ # first attempt - raw string value
+ int_value = self._str2int.get(value)
+ if int_value is None:
+ # second attempt - strip whitespace
+ int_value = self._str2int.get(value.strip())
+ if int_value is None:
+ # third attempt - convert str to int
+ try:
+ int_value = int(value)
+ except ValueError:
+ failed_parse = True
+ else:
+ if int_value < -1 or int_value >= self.num_classes:
+ failed_parse = True
+ if failed_parse:
+ raise ValueError(f"Invalid string class label {value}")
+ return int_value
+
+ def int2str(self, values: Union[int, Iterable]) -> Union[str, Iterable]:
+ """Conversion `integer` => class name `string`.
+
+ Regarding unknown/missing labels: passing negative integers raises `ValueError`.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train")
+ >>> ds.features["label"].int2str(0)
+ 'neg'
+ ```
+ """
+ if not isinstance(values, int) and not isinstance(values, Iterable):
+ raise ValueError(
+ f"Values {values} should be an integer or an Iterable (list, numpy array, pytorch, tensorflow tensors)"
+ )
+ return_list = True
+ if isinstance(values, int):
+ values = [values]
+ return_list = False
+
+ for v in values:
+ if not 0 <= v < self.num_classes:
+ raise ValueError(f"Invalid integer class label {v:d}")
+
+ output = [self._int2str[int(v)] for v in values]
+ return output if return_list else output[0]
+
+ def encode_example(self, example_data):
+ if self.num_classes is None:
+ raise ValueError(
+ "Trying to use ClassLabel feature with undefined number of class. "
+ "Please set ClassLabel.names or num_classes."
+ )
+
+ # If a string is given, convert to associated integer
+ if isinstance(example_data, str):
+ example_data = self.str2int(example_data)
+
+ # Allowing -1 to mean no label.
+ if not -1 <= example_data < self.num_classes:
+ raise ValueError(f"Class label {example_data:d} greater than configured num_classes {self.num_classes}")
+ return example_data
+
+ def cast_storage(self, storage: Union[pa.StringArray, pa.IntegerArray]) -> pa.Int64Array:
+ """Cast an Arrow array to the `ClassLabel` arrow storage type.
+ The Arrow types that can be converted to the `ClassLabel` pyarrow storage type are:
+
+ - `pa.string()`
+ - `pa.int()`
+
+ Args:
+ storage (`Union[pa.StringArray, pa.IntegerArray]`):
+ PyArrow array to cast.
+
+ Returns:
+ `pa.Int64Array`: Array in the `ClassLabel` arrow storage type.
+ """
+ if isinstance(storage, pa.IntegerArray) and len(storage) > 0:
+ min_max = pc.min_max(storage).as_py()
+ if min_max["max"] is not None and min_max["max"] >= self.num_classes:
+ raise ValueError(
+ f"Class label {min_max['max']} greater than configured num_classes {self.num_classes}"
+ )
+ elif isinstance(storage, pa.StringArray):
+ storage = pa.array(
+ [self._strval2int(label) if label is not None else None for label in storage.to_pylist()]
+ )
+ return array_cast(storage, self.pa_type)
+
+ @staticmethod
+ def _load_names_from_file(names_filepath):
+ with open(names_filepath, encoding="utf-8") as f:
+ return [name.strip() for name in f.read().split("\n") if name.strip()] # Filter empty names
+
+
+class Sequence:
+ """
+ A `Sequence` is a utility that automatically converts internal dictionary feature into a dictionary of
+ lists. This behavior is implemented to have a compatibility layer with the TensorFlow Datasets library but may be
+ un-wanted in some cases. If you don't want this behavior, you can use a [`List`] or a [`LargeList`]
+ instead of the [`Sequence`].
+
+ Args:
+ feature ([`FeatureType`]):
+ Child feature data type of each item within the large list.
+ length (optional `int`, default to -1):
+ Length of the list if it is fixed.
+ Defaults to -1 which means an arbitrary length.
+
+ Returns:
+ [`List`] of the specified feature, except `dict` of sub-features
+ which are converted to `dict` of lists of sub-features for compatibility with TFDS.
+
+ """
+
+ def __new__(cls, feature=None, length=-1, **kwargs):
+ # useful to still get isinstance(Sequence(Value("int64")), Sequence)
+ if (
+ cls is Sequence
+ and isinstance(feature, dict)
+ and any(not isinstance(subfeature, List) for subfeature in feature.values())
+ ):
+ out = {key: List(value, length=length, **kwargs) for key, value in feature.items()}
+ else:
+ out = super().__new__(List)
+ return out
+
+
+@dataclass(repr=False)
+class List(Sequence):
+ """Feature type for large list data composed of child feature data type.
+
+ It is backed by `pyarrow.ListType`, which uses 32-bit offsets or a fixed length.
+
+ Args:
+ feature ([`FeatureType`]):
+ Child feature data type of each item within the large list.
+ length (optional `int`, default to -1):
+ Length of the list if it is fixed.
+ Defaults to -1 which means an arbitrary length.
+ """
+
+ feature: Any
+ length: int = -1
+ id: Optional[str] = field(default=None, repr=False)
+ # Automatically constructed
+ pa_type: ClassVar[Any] = None
+ _type: str = field(default="List", init=False, repr=False)
+
+ def __repr__(self):
+ if self.length != -1:
+ return f"{type(self).__name__}({self.feature}, length={self.length})"
+ else:
+ return f"{type(self).__name__}({self.feature})"
+
+
+@dataclass(repr=False)
+class LargeList:
+ """Feature type for large list data composed of child feature data type.
+
+ It is backed by `pyarrow.LargeListType`, which is like `pyarrow.ListType` but with 64-bit rather than 32-bit offsets.
+
+ Args:
+ feature ([`FeatureType`]):
+ Child feature data type of each item within the large list.
+ """
+
+ feature: Any
+ id: Optional[str] = field(default=None, repr=False)
+ # Automatically constructed
+ pa_type: ClassVar[Any] = None
+ _type: str = field(default="LargeList", init=False, repr=False)
+
+ def __repr__(self):
+ return f"{type(self).__name__}({self.feature})"
+
+
+FeatureType = Union[
+ dict,
+ list,
+ tuple,
+ Value,
+ ClassLabel,
+ Translation,
+ TranslationVariableLanguages,
+ LargeList,
+ List,
+ Array2D,
+ Array3D,
+ Array4D,
+ Array5D,
+ Audio,
+ Image,
+ Video,
+ Pdf,
+ Nifti,
+]
+
+
+def _check_non_null_non_empty_recursive(obj, schema: Optional[FeatureType] = None) -> bool:
+ """
+ Check if the object is not None.
+ If the object is a list or a tuple, recursively check the first element of the sequence and stop if at any point the first element is not a sequence or is an empty sequence.
+ """
+ if obj is None:
+ return False
+ elif isinstance(obj, (list, tuple)) and (schema is None or isinstance(schema, (list, tuple, LargeList, List))):
+ if len(obj) > 0:
+ if schema is None:
+ pass
+ elif isinstance(schema, (list, tuple)):
+ schema = schema[0]
+ else:
+ schema = schema.feature
+ return _check_non_null_non_empty_recursive(obj[0], schema)
+ else:
+ return False
+ else:
+ return True
+
+
+def get_nested_type(schema: FeatureType) -> pa.DataType:
+ """
+ get_nested_type() converts a datasets.FeatureType into a pyarrow.DataType, and acts as the inverse of
+ generate_from_arrow_type().
+
+ It performs double-duty as the implementation of Features.type and handles the conversion of
+ datasets.Feature->pa.struct
+ """
+ # Nested structures: we allow dict, list/tuples, sequences
+ if isinstance(schema, Features):
+ return pa.struct(
+ {key: get_nested_type(schema[key]) for key in schema}
+ ) # Features is subclass of dict, and dict order is deterministic since Python 3.6
+ elif isinstance(schema, dict):
+ return pa.struct(
+ {key: get_nested_type(schema[key]) for key in schema}
+ ) # however don't sort on struct types since the order matters
+ elif isinstance(schema, (list, tuple)):
+ if len(schema) != 1:
+ raise ValueError("When defining list feature, you should just provide one example of the inner type")
+ value_type = get_nested_type(schema[0])
+ return pa.list_(value_type)
+ elif isinstance(schema, LargeList):
+ value_type = get_nested_type(schema.feature)
+ return pa.large_list(value_type)
+ elif isinstance(schema, List):
+ value_type = get_nested_type(schema.feature)
+ return pa.list_(value_type, schema.length)
+
+ # Other objects are callable which returns their data type (ClassLabel, Array2D, Translation, Arrow datatype creation methods)
+ return schema()
+
+
+def encode_nested_example(schema, obj, level=0):
+ """Encode a nested example.
+ This is used since some features (in particular ClassLabel) have some logic during encoding.
+
+ To avoid iterating over possibly long lists, it first checks (recursively) if the first element that is not None or empty (if it is a sequence) has to be encoded.
+ If the first element needs to be encoded, then all the elements of the list will be encoded, otherwise they'll stay the same.
+ """
+ # Nested structures: we allow dict, list/tuples, sequences
+ if isinstance(schema, dict):
+ if level == 0 and obj is None:
+ raise ValueError("Got None but expected a dictionary instead")
+ return (
+ {k: encode_nested_example(schema[k], obj.get(k), level=level + 1) for k in schema}
+ if obj is not None
+ else None
+ )
+ elif isinstance(schema, (LargeList, List)):
+ if obj is None:
+ return None
+ else:
+ if len(obj) > 0:
+ sub_schema = schema.feature
+ for first_elmt in obj:
+ if _check_non_null_non_empty_recursive(first_elmt, sub_schema):
+ break
+ try:
+ changed = bool(encode_nested_example(sub_schema, first_elmt, level=level + 1) != first_elmt)
+ except ValueError: # can happen when comparing arrays
+ changed = False
+ if changed:
+ return [encode_nested_example(sub_schema, o, level=level + 1) for o in obj]
+ return list(obj)
+ # Object with special encoding:
+ # ClassLabel will convert from string to int, TranslationVariableLanguages does some checks
+ elif hasattr(schema, "encode_example"):
+ return schema.encode_example(obj) if obj is not None else None
+ # Other object should be directly convertible to a native Arrow type (like Translation and Translation)
+ return obj
+
+
+def decode_nested_example(schema, obj, token_per_repo_id: Optional[dict[str, Union[str, bool, None]]] = None):
+ """Decode a nested example.
+ This is used since some features (in particular Audio and Image) have some logic during decoding.
+
+ To avoid iterating over possibly long lists, it first checks (recursively) if the first element that is not None or empty (if it is a sequence) has to be decoded.
+ If the first element needs to be decoded, then all the elements of the list will be decoded, otherwise they'll stay the same.
+ """
+ # Nested structures: we allow dict, list/tuples, sequences
+ if isinstance(schema, dict):
+ return (
+ {k: decode_nested_example(sub_schema, sub_obj) for k, (sub_schema, sub_obj) in zip_dict(schema, obj)}
+ if obj is not None
+ else None
+ )
+ elif isinstance(schema, (list, tuple)):
+ sub_schema = schema[0]
+ if obj is None:
+ return None
+ else:
+ if len(obj) > 0:
+ for first_elmt in obj:
+ if _check_non_null_non_empty_recursive(first_elmt, sub_schema):
+ break
+ if decode_nested_example(sub_schema, first_elmt) != first_elmt:
+ return [decode_nested_example(sub_schema, o) for o in obj]
+ return list(obj)
+ elif isinstance(schema, (LargeList, List)):
+ if obj is None:
+ return None
+ else:
+ sub_schema = schema.feature
+ if len(obj) > 0:
+ for first_elmt in obj:
+ if _check_non_null_non_empty_recursive(first_elmt, sub_schema):
+ break
+ if decode_nested_example(sub_schema, first_elmt) != first_elmt:
+ return [decode_nested_example(sub_schema, o) for o in obj]
+ return list(obj)
+ # Object with special decoding:
+ elif hasattr(schema, "decode_example") and getattr(schema, "decode", True):
+ # we pass the token to read and decode files from private repositories in streaming mode
+ return schema.decode_example(obj, token_per_repo_id=token_per_repo_id) if obj is not None else None
+ return obj
+
+
+_FEATURE_TYPES: dict[str, FeatureType] = {
+ Value.__name__: Value,
+ ClassLabel.__name__: ClassLabel,
+ Translation.__name__: Translation,
+ TranslationVariableLanguages.__name__: TranslationVariableLanguages,
+ LargeList.__name__: LargeList,
+ List.__name__: List,
+ Array2D.__name__: Array2D,
+ Array3D.__name__: Array3D,
+ Array4D.__name__: Array4D,
+ Array5D.__name__: Array5D,
+ Audio.__name__: Audio,
+ Image.__name__: Image,
+ Video.__name__: Video,
+ Pdf.__name__: Pdf,
+ Nifti.__name__: Nifti,
+}
+
+
+@experimental
+def register_feature(
+ feature_cls: type,
+ feature_type: str,
+):
+ """
+ Register a Feature object using a name and class.
+ This function must be used on a Feature class.
+ """
+ if feature_type in _FEATURE_TYPES:
+ logger.warning(
+ f"Overwriting feature type '{feature_type}' ({_FEATURE_TYPES[feature_type].__name__} -> {feature_cls.__name__})"
+ )
+ _FEATURE_TYPES[feature_type] = feature_cls
+
+
+def generate_from_dict(obj: Any):
+ """Regenerate the nested feature object from a deserialized dict.
+ We use the '_type' fields to get the dataclass name to load.
+
+ generate_from_dict is the recursive helper for Features.from_dict, and allows for a convenient constructor syntax
+ to define features from deserialized JSON dictionaries. This function is used in particular when deserializing
+ a :class:`DatasetInfo` that was dumped to a JSON object. This acts as an analogue to
+ :meth:`Features.from_arrow_schema` and handles the recursive field-by-field instantiation, but doesn't require any
+ mapping to/from pyarrow, except for the fact that it takes advantage of the mapping of pyarrow primitive dtypes
+ that :class:`Value` automatically performs.
+ """
+ # Nested structures: we allow dict, list/tuples, sequences
+ if isinstance(obj, list):
+ return [generate_from_dict(value) for value in obj]
+ # Otherwise we have a dict or a dataclass
+ if "_type" not in obj or isinstance(obj["_type"], dict):
+ return {key: generate_from_dict(value) for key, value in obj.items()}
+ obj = dict(obj)
+ _type = obj.pop("_type")
+ class_type = _FEATURE_TYPES.get(_type, None) or globals().get(_type, None)
+
+ if class_type is None:
+ raise ValueError(f"Feature type '{_type}' not found. Available feature types: {list(_FEATURE_TYPES.keys())}")
+
+ if class_type == LargeList:
+ feature = obj.pop("feature")
+ return LargeList(generate_from_dict(feature), **obj)
+ if class_type == List:
+ feature = obj.pop("feature")
+ return List(generate_from_dict(feature), **obj)
+ if class_type == Sequence: # backward compatibility, this translates to a List or a dict
+ feature = obj.pop("feature")
+ return Sequence(feature=generate_from_dict(feature), **obj)
+
+ field_names = {f.name for f in fields(class_type)}
+ return class_type(**{k: v for k, v in obj.items() if k in field_names})
+
+
+def generate_from_arrow_type(pa_type: pa.DataType) -> FeatureType:
+ """
+ generate_from_arrow_type accepts an arrow DataType and returns a datasets FeatureType to be used as the type for
+ a single field.
+
+ This is the high-level arrow->datasets type conversion and is inverted by get_nested_type().
+
+ This operates at the individual *field* level, whereas Features.from_arrow_schema() operates at the
+ full schema level and holds the methods that represent the bijection from Features<->pyarrow.Schema
+ """
+ if isinstance(pa_type, pa.StructType):
+ return {field.name: generate_from_arrow_type(field.type) for field in pa_type}
+ elif isinstance(pa_type, pa.FixedSizeListType):
+ return List(generate_from_arrow_type(pa_type.value_type), length=pa_type.list_size)
+ elif isinstance(pa_type, pa.ListType):
+ return List(generate_from_arrow_type(pa_type.value_type))
+ elif isinstance(pa_type, pa.LargeListType):
+ return LargeList(generate_from_arrow_type(pa_type.value_type))
+ elif isinstance(pa_type, _ArrayXDExtensionType):
+ array_feature = [None, None, Array2D, Array3D, Array4D, Array5D][pa_type.ndims]
+ return array_feature(shape=pa_type.shape, dtype=pa_type.value_type)
+ elif isinstance(pa_type, pa.DataType):
+ return Value(dtype=_arrow_to_datasets_dtype(pa_type))
+ else:
+ raise ValueError(f"Cannot convert {pa_type} to a Feature type.")
+
+
+def numpy_to_pyarrow_listarray(arr: np.ndarray, type: pa.DataType = None) -> pa.ListArray:
+ """Build a PyArrow ListArray from a multidimensional NumPy array"""
+ arr = np.array(arr)
+ values = pa.array(arr.flatten(), type=type)
+ for i in range(arr.ndim - 1):
+ n_offsets = reduce(mul, arr.shape[: arr.ndim - i - 1], 1)
+ step_offsets = arr.shape[arr.ndim - i - 1]
+ offsets = pa.array(np.arange(n_offsets + 1) * step_offsets, type=pa.int32())
+ values = pa.ListArray.from_arrays(offsets, values)
+ return values
+
+
+def list_of_pa_arrays_to_pyarrow_listarray(l_arr: list[Optional[pa.Array]]) -> pa.ListArray:
+ null_mask = np.array([arr is None for arr in l_arr])
+ null_indices = np.arange(len(null_mask))[null_mask] - np.arange(np.sum(null_mask))
+ l_arr = [arr for arr in l_arr if arr is not None]
+ offsets = np.cumsum(
+ [0] + [len(arr) for arr in l_arr], dtype=object
+ ) # convert to dtype object to allow None insertion
+ offsets = np.insert(offsets, null_indices, None)
+ offsets = pa.array(offsets, type=pa.int32())
+ values = pa.concat_arrays(l_arr)
+ return pa.ListArray.from_arrays(offsets, values)
+
+
+def list_of_np_array_to_pyarrow_listarray(l_arr: list[np.ndarray], type: pa.DataType = None) -> pa.ListArray:
+ """Build a PyArrow ListArray from a possibly nested list of NumPy arrays"""
+ if len(l_arr) > 0:
+ return list_of_pa_arrays_to_pyarrow_listarray(
+ [numpy_to_pyarrow_listarray(arr, type=type) if arr is not None else None for arr in l_arr]
+ )
+ else:
+ return pa.array([], type=type)
+
+
+def contains_any_np_array(data: Any):
+ """Return `True` if data is a NumPy ndarray or (recursively) if first non-null value in list is a NumPy ndarray.
+
+ Args:
+ data (Any): Data.
+
+ Returns:
+ bool
+ """
+ if isinstance(data, np.ndarray):
+ return True
+ elif isinstance(data, list):
+ return contains_any_np_array(first_non_null_value(data)[1])
+ else:
+ return False
+
+
+def any_np_array_to_pyarrow_listarray(data: Union[np.ndarray, list], type: pa.DataType = None) -> pa.ListArray:
+ """Convert to PyArrow ListArray either a NumPy ndarray or (recursively) a list that may contain any NumPy ndarray.
+
+ Args:
+ data (Union[np.ndarray, List]): Data.
+ type (pa.DataType): Explicit PyArrow DataType passed to coerce the ListArray data type.
+
+ Returns:
+ pa.ListArray
+ """
+ if isinstance(data, np.ndarray):
+ return numpy_to_pyarrow_listarray(data, type=type)
+ elif isinstance(data, list):
+ return list_of_pa_arrays_to_pyarrow_listarray([any_np_array_to_pyarrow_listarray(i, type=type) for i in data])
+
+
+def to_pyarrow_listarray(data: Any, pa_type: _ArrayXDExtensionType) -> pa.Array:
+ """Convert to PyArrow ListArray.
+
+ Args:
+ data (Any): List, iterable, np.ndarray or pd.Series.
+ pa_type (_ArrayXDExtensionType): Any of the ArrayNDExtensionType.
+
+ Returns:
+ pyarrow.Array
+ """
+ if contains_any_np_array(data):
+ return any_np_array_to_pyarrow_listarray(data, type=pa_type.value_type)
+ else:
+ return pa.array(data, pa_type.storage_dtype)
+
+
+def _visit(feature: FeatureType, func: Callable[[FeatureType], Optional[FeatureType]]) -> FeatureType:
+ """Visit a (possibly nested) feature.
+
+ Args:
+ feature (FeatureType): the feature type to be checked
+ Returns:
+ visited feature (FeatureType)
+ """
+ if isinstance(feature, Features):
+ out = func(Features({k: _visit(f, func) for k, f in feature.items()}))
+ elif isinstance(feature, dict):
+ out = func({k: _visit(f, func) for k, f in feature.items()})
+ elif isinstance(feature, LargeList):
+ out = func(LargeList(_visit(feature.feature, func)))
+ elif isinstance(feature, List):
+ out = func(List(_visit(feature.feature, func), length=feature.length))
+ else:
+ out = func(feature)
+ return feature if out is None else out
+
+
+_VisitPath = list[Union[str, Literal[0]]]
+
+
+def _visit_with_path(
+ feature: FeatureType, func: Callable[[FeatureType, _VisitPath], Optional[FeatureType]], visit_path: _VisitPath = []
+) -> FeatureType:
+ """Visit a (possibly nested) feature with its path in the Feature object.
+
+ A path in a nested feature object is the list of keys that need to be
+ sequentially accessed to get to the sub-feature.
+
+ For example:
+ - ["foo"] corresponds to the column "foo"
+ - ["foo", 0] corresponds to the sub-feature of the lists in "foo"
+ - ["foo", "bar"] corresponds to the sub-feature of the dicts in "foo" with key "bar"
+
+ Args:
+ feature (`FeatureType`): the feature type to be checked.
+
+ Returns:
+ `FeatureType`: the visited feature.
+ """
+ if isinstance(feature, Features):
+ out = func(Features({k: _visit_with_path(f, func, visit_path + [k]) for k, f in feature.items()}), visit_path)
+ elif isinstance(feature, dict):
+ out = func({k: _visit_with_path(f, func, visit_path + [k]) for k, f in feature.items()}, visit_path)
+ elif isinstance(feature, List):
+ out = func(List(_visit_with_path(feature.feature, func, visit_path + [0]), length=feature.length), visit_path)
+ elif isinstance(feature, LargeList):
+ out = func(LargeList(_visit_with_path(feature.feature, func, visit_path + [0])), visit_path)
+ else:
+ out = func(feature, visit_path)
+ return feature if out is None else out
+
+
+def require_decoding(feature: FeatureType, ignore_decode_attribute: bool = False) -> bool:
+ """Check if a (possibly nested) feature requires decoding.
+
+ Args:
+ feature (FeatureType): the feature type to be checked
+ ignore_decode_attribute (:obj:`bool`, default ``False``): Whether to ignore the current value
+ of the `decode` attribute of the decodable feature types.
+ Returns:
+ :obj:`bool`
+ """
+ if isinstance(feature, dict):
+ return any(require_decoding(f) for f in feature.values())
+ elif isinstance(feature, (list, tuple)):
+ return require_decoding(feature[0])
+ elif isinstance(feature, LargeList):
+ return require_decoding(feature.feature)
+ elif isinstance(feature, List):
+ return require_decoding(feature.feature)
+ else:
+ return hasattr(feature, "decode_example") and (
+ getattr(feature, "decode", True) if not ignore_decode_attribute else True
+ )
+
+
+def require_storage_cast(feature: FeatureType) -> bool:
+ """Check if a (possibly nested) feature requires storage casting.
+
+ Args:
+ feature (FeatureType): the feature type to be checked
+ Returns:
+ :obj:`bool`
+ """
+ if isinstance(feature, dict):
+ return any(require_storage_cast(f) for f in feature.values())
+ elif isinstance(feature, LargeList):
+ return require_storage_cast(feature.feature)
+ elif isinstance(feature, List):
+ return require_storage_cast(feature.feature)
+ else:
+ return hasattr(feature, "cast_storage")
+
+
+def require_storage_embed(feature: FeatureType) -> bool:
+ """Check if a (possibly nested) feature requires embedding data into storage.
+
+ Args:
+ feature (FeatureType): the feature type to be checked
+ Returns:
+ :obj:`bool`
+ """
+ if isinstance(feature, dict):
+ return any(require_storage_cast(f) for f in feature.values())
+ elif isinstance(feature, LargeList):
+ return require_storage_cast(feature.feature)
+ elif isinstance(feature, List):
+ return require_storage_cast(feature.feature)
+ else:
+ return hasattr(feature, "embed_storage")
+
+
+def keep_features_dicts_synced(func):
+ """
+ Wrapper to keep the secondary dictionary, which tracks whether keys are decodable, of the :class:`datasets.Features` object
+ in sync with the main dictionary.
+ """
+
+ @wraps(func)
+ def wrapper(*args, **kwargs):
+ if args:
+ self: "Features" = args[0]
+ args = args[1:]
+ else:
+ self: "Features" = kwargs.pop("self")
+ out = func(self, *args, **kwargs)
+ assert hasattr(self, "_column_requires_decoding")
+ self._column_requires_decoding = {col: require_decoding(feature) for col, feature in self.items()}
+ return out
+
+ wrapper._decorator_name_ = "_keep_dicts_synced"
+ return wrapper
+
+
+class Features(dict):
+ """A special dictionary that defines the internal structure of a dataset.
+
+ Instantiated with a dictionary of type `dict[str, FieldType]`, where keys are the desired column names,
+ and values are the type of that column.
+
+ `FieldType` can be one of the following:
+ - [`Value`] feature specifies a single data type value, e.g. `int64` or `string`.
+ - [`ClassLabel`] feature specifies a predefined set of classes which can have labels associated to them and
+ will be stored as integers in the dataset.
+ - Python `dict` specifies a composite feature containing a mapping of sub-fields to sub-features.
+ It's possible to have nested fields of nested fields in an arbitrary manner.
+ - [`List`] or [`LargeList`] specifies a composite feature containing a sequence of
+ sub-features, all of the same feature type.
+ - [`Array2D`], [`Array3D`], [`Array4D`] or [`Array5D`] feature for multidimensional arrays.
+ - [`Audio`] feature to store the absolute path to an audio file or a dictionary with the relative path
+ to an audio file ("path" key) and its bytes content ("bytes" key).
+ This feature loads the audio lazily with a decoder.
+ - [`Image`] feature to store the absolute path to an image file, an `np.ndarray` object, a `PIL.Image.Image` object
+ or a dictionary with the relative path to an image file ("path" key) and its bytes content ("bytes" key).
+ This feature extracts the image data.
+ - [`Video`] feature to store the absolute path to a video file, a `torchcodec.decoders.VideoDecoder` object
+ or a dictionary with the relative path to a video file ("path" key) and its bytes content ("bytes" key).
+ This feature loads the video lazily with a decoder.
+ - [`Pdf`] feature to store the absolute path to a PDF file, a `pdfplumber.pdf.PDF` object
+ or a dictionary with the relative path to a PDF file ("path" key) and its bytes content ("bytes" key).
+ This feature loads the PDF lazily with a PDF reader.
+ - [`Nifti`] feature to store the absolute path to a NIfTI neuroimaging file, a `nibabel.Nifti1Image` object
+ or a dictionary with the relative path to a NIfTI file ("path" key) and its bytes content ("bytes" key).
+ This feature loads the NIfTI file lazily with nibabel.
+ - [`Translation`] or [`TranslationVariableLanguages`] feature specific to Machine Translation.
+ """
+
+ def __init__(*args, **kwargs):
+ # self not in the signature to allow passing self as a kwarg
+ if not args:
+ raise TypeError("descriptor '__init__' of 'Features' object needs an argument")
+ self, *args = args
+ super(Features, self).__init__(*args, **kwargs)
+ # keep track of columns which require decoding
+ self._column_requires_decoding: dict[str, bool] = {
+ col: require_decoding(feature) for col, feature in self.items()
+ }
+
+ # backward compatibility with datasets<4 : [feature] -> List(feature)
+ def _check_old_list(feature):
+ if isinstance(feature, list):
+ return List(_visit(feature[0], _check_old_list))
+ return feature
+
+ for column_name, feature in self.items():
+ self[column_name] = _visit(feature, _check_old_list)
+
+ __setitem__ = keep_features_dicts_synced(dict.__setitem__)
+ __delitem__ = keep_features_dicts_synced(dict.__delitem__)
+ update = keep_features_dicts_synced(dict.update)
+ setdefault = keep_features_dicts_synced(dict.setdefault)
+ pop = keep_features_dicts_synced(dict.pop)
+ popitem = keep_features_dicts_synced(dict.popitem)
+ clear = keep_features_dicts_synced(dict.clear)
+
+ def __reduce__(self):
+ return Features, (dict(self),)
+
+ @property
+ def type(self):
+ """
+ Features field types.
+
+ Returns:
+ :obj:`pyarrow.DataType`
+ """
+ return get_nested_type(self)
+
+ @property
+ def arrow_schema(self):
+ """
+ Features schema.
+
+ Returns:
+ :obj:`pyarrow.Schema`
+ """
+ hf_metadata = {"info": {"features": self.to_dict()}}
+ return pa.schema(self.type).with_metadata({"huggingface": json.dumps(hf_metadata)})
+
+ @classmethod
+ def from_arrow_schema(cls, pa_schema: pa.Schema) -> "Features":
+ """
+ Construct [`Features`] from Arrow Schema.
+ It also checks the schema metadata for Hugging Face Datasets features.
+ Non-nullable fields are not supported and set to nullable.
+
+ Also, pa.dictionary is not supported and it uses its underlying type instead.
+ Therefore datasets convert DictionaryArray objects to their actual values.
+
+ Args:
+ pa_schema (`pyarrow.Schema`):
+ Arrow Schema.
+
+ Returns:
+ [`Features`]
+ """
+ # try to load features from the arrow schema metadata
+ metadata_features = Features()
+ if pa_schema.metadata is not None and b"huggingface" in pa_schema.metadata:
+ metadata = json.loads(pa_schema.metadata[b"huggingface"].decode())
+ if "info" in metadata and "features" in metadata["info"] and metadata["info"]["features"] is not None:
+ metadata_features = Features.from_dict(metadata["info"]["features"])
+ metadata_features_schema = metadata_features.arrow_schema
+ obj = {
+ field.name: (
+ metadata_features[field.name]
+ if field.name in metadata_features and metadata_features_schema.field(field.name) == field
+ else generate_from_arrow_type(field.type)
+ )
+ for field in pa_schema
+ }
+ return cls(**obj)
+
+ @classmethod
+ def from_dict(cls, dic) -> "Features":
+ """
+ Construct [`Features`] from dict.
+
+ Regenerate the nested feature object from a deserialized dict.
+ We use the `_type` key to infer the dataclass name of the feature `FieldType`.
+
+ It allows for a convenient constructor syntax
+ to define features from deserialized JSON dictionaries. This function is used in particular when deserializing
+ a [`DatasetInfo`] that was dumped to a JSON object. This acts as an analogue to
+ [`Features.from_arrow_schema`] and handles the recursive field-by-field instantiation, but doesn't require
+ any mapping to/from pyarrow, except for the fact that it takes advantage of the mapping of pyarrow primitive
+ dtypes that [`Value`] automatically performs.
+
+ Args:
+ dic (`dict[str, Any]`):
+ Python dictionary.
+
+ Returns:
+ `Features`
+
+ Example::
+ >>> Features.from_dict({'_type': {'dtype': 'string', 'id': None, '_type': 'Value'}})
+ {'_type': Value('string')}
+ """
+ obj = generate_from_dict(dic)
+ return cls(**obj)
+
+ def to_dict(self):
+ return asdict(self)
+
+ def _to_yaml_list(self) -> list:
+ # we compute the YAML list from the dict representation that is used for JSON dump
+ yaml_data = self.to_dict()
+
+ def simplify(feature: dict) -> dict:
+ if not isinstance(feature, dict):
+ raise TypeError(f"Expected a dict but got a {type(feature)}: {feature}")
+
+ for list_type in ["large_list", "list", "sequence"]:
+ #
+ # list_type: -> list_type: int32
+ # dtype: int32 ->
+ #
+ if isinstance(feature.get(list_type), dict) and list(feature[list_type]) == ["dtype"]:
+ feature[list_type] = feature[list_type]["dtype"]
+
+ #
+ # list_type: -> list_type:
+ # struct: -> - name: foo
+ # - name: foo -> dtype: int32
+ # dtype: int32 ->
+ #
+ if isinstance(feature.get(list_type), dict) and list(feature[list_type]) == ["struct"]:
+ feature[list_type] = feature[list_type]["struct"]
+
+ #
+ # class_label: -> class_label:
+ # names: -> names:
+ # - negative -> '0': negative
+ # - positive -> '1': positive
+ #
+ if isinstance(feature.get("class_label"), dict) and isinstance(feature["class_label"].get("names"), list):
+ # server-side requirement: keys must be strings
+ feature["class_label"]["names"] = {
+ str(label_id): label_name for label_id, label_name in enumerate(feature["class_label"]["names"])
+ }
+ return feature
+
+ def to_yaml_inner(obj: Union[dict, list]) -> dict:
+ if isinstance(obj, dict):
+ _type = obj.pop("_type", None)
+ if _type == "LargeList":
+ _feature = obj.pop("feature")
+ return simplify({"large_list": to_yaml_inner(_feature), **obj})
+ elif _type == "List":
+ _feature = obj.pop("feature")
+ return simplify({"list": to_yaml_inner(_feature), **obj})
+ elif _type == "Value":
+ return obj
+ elif _type and not obj:
+ return {"dtype": camelcase_to_snakecase(_type)}
+ elif _type:
+ return {"dtype": simplify({camelcase_to_snakecase(_type): obj})}
+ else:
+ return {"struct": [{"name": name, **to_yaml_inner(_feature)} for name, _feature in obj.items()]}
+ elif isinstance(obj, list):
+ return simplify({"list": simplify(to_yaml_inner(obj[0]))})
+ elif isinstance(obj, tuple):
+ return to_yaml_inner(list(obj))
+ else:
+ raise TypeError(f"Expected a dict or a list but got {type(obj)}: {obj}")
+
+ def to_yaml_types(obj: dict) -> dict:
+ if isinstance(obj, dict):
+ return {k: to_yaml_types(v) for k, v in obj.items()}
+ elif isinstance(obj, list):
+ return [to_yaml_types(v) for v in obj]
+ elif isinstance(obj, tuple):
+ return to_yaml_types(list(obj))
+ else:
+ return obj
+
+ return to_yaml_types(to_yaml_inner(yaml_data)["struct"])
+
+ @classmethod
+ def _from_yaml_list(cls, yaml_data: list) -> "Features":
+ yaml_data = copy.deepcopy(yaml_data)
+
+ # we convert the list obtained from YAML data into the dict representation that is used for JSON dump
+
+ def unsimplify(feature: dict) -> dict:
+ if not isinstance(feature, dict):
+ raise TypeError(f"Expected a dict but got a {type(feature)}: {feature}")
+
+ for list_type in ["large_list", "list", "sequence"]:
+ #
+ # list_type: int32 -> list_type:
+ # -> dtype: int32
+ #
+ if isinstance(feature.get(list_type), str):
+ feature[list_type] = {"dtype": feature[list_type]}
+
+ #
+ # class_label: -> class_label:
+ # names: -> names:
+ # '0': negative -> - negative
+ # '1': positive -> - positive
+ #
+ if isinstance(feature.get("class_label"), dict) and isinstance(feature["class_label"].get("names"), dict):
+ label_ids = sorted(feature["class_label"]["names"], key=int)
+ if label_ids and [int(label_id) for label_id in label_ids] != list(range(int(label_ids[-1]) + 1)):
+ raise ValueError(
+ f"ClassLabel expected a value for all label ids [0:{int(label_ids[-1]) + 1}] but some ids are missing."
+ )
+ feature["class_label"]["names"] = [feature["class_label"]["names"][label_id] for label_id in label_ids]
+ return feature
+
+ def from_yaml_inner(obj: Union[dict, list]) -> Union[dict, list]:
+ if isinstance(obj, dict):
+ if not obj:
+ return {}
+ _type = next(iter(obj))
+ if _type == "large_list":
+ _feature = from_yaml_inner(unsimplify(obj).pop(_type))
+ return {"feature": _feature, **obj, "_type": "LargeList"}
+ if _type == "sequence": # backward compatibility
+ if isinstance(obj[_type], list):
+ _feature = from_yaml_inner(unsimplify(obj).pop(_type))
+ return {
+ name: {"feature": _subfeature, **obj, "_type": "List"}
+ for name, _subfeature in _feature.items()
+ }
+ else:
+ _feature = from_yaml_inner(unsimplify(obj).pop(_type))
+ return {"feature": _feature, **obj, "_type": "List"}
+ if _type == "list":
+ _feature = from_yaml_inner(unsimplify(obj).pop(_type))
+ return {"feature": _feature, **obj, "_type": "List"}
+ if _type == "struct":
+ return from_yaml_inner(obj["struct"])
+ elif _type == "dtype":
+ if isinstance(obj["dtype"], str):
+ # e.g. int32, float64, string, audio, image
+ try:
+ Value(obj["dtype"])
+ return {**obj, "_type": "Value"}
+ except ValueError:
+ # e.g. Audio, Image, ArrayXD
+ return {"_type": snakecase_to_camelcase(obj["dtype"])}
+ else:
+ return from_yaml_inner(obj["dtype"])
+ else:
+ return {"_type": snakecase_to_camelcase(_type), **unsimplify(obj)[_type]}
+ elif isinstance(obj, list):
+ names = [_feature.pop("name") for _feature in obj]
+ return {name: from_yaml_inner(_feature) for name, _feature in zip(names, obj)}
+ else:
+ raise TypeError(f"Expected a dict or a list but got {type(obj)}: {obj}")
+
+ return cls.from_dict(from_yaml_inner(yaml_data))
+
+ def encode_example(self, example):
+ """
+ Encode example into a format for Arrow.
+
+ Args:
+ example (`dict[str, Any]`):
+ Data in a Dataset row.
+
+ Returns:
+ `dict[str, Any]`
+ """
+ example = cast_to_python_objects(example)
+ return encode_nested_example(self, example)
+
+ def encode_column(self, column, column_name: str):
+ """
+ Encode column into a format for Arrow.
+
+ Args:
+ column (`list[Any]`):
+ Data in a Dataset column.
+ column_name (`str`):
+ Dataset column name.
+
+ Returns:
+ `list[Any]`
+ """
+ column = cast_to_python_objects(column)
+ return [encode_nested_example(self[column_name], obj, level=1) for obj in column]
+
+ def encode_batch(self, batch):
+ """
+ Encode batch into a format for Arrow.
+
+ Args:
+ batch (`dict[str, list[Any]]`):
+ Data in a Dataset batch.
+
+ Returns:
+ `dict[str, list[Any]]`
+ """
+ encoded_batch = {}
+ if set(batch) != set(self):
+ raise ValueError(f"Column mismatch between batch {set(batch)} and features {set(self)}")
+ for key, column in batch.items():
+ column = cast_to_python_objects(column)
+ encoded_batch[key] = [encode_nested_example(self[key], obj, level=1) for obj in column]
+ return encoded_batch
+
+ def decode_example(self, example: dict, token_per_repo_id: Optional[dict[str, Union[str, bool, None]]] = None):
+ """Decode example with custom feature decoding.
+
+ Args:
+ example (`dict[str, Any]`):
+ Dataset row data.
+ token_per_repo_id (`dict`, *optional*):
+ To access and decode audio or image files from private repositories on the Hub, you can pass
+ a dictionary `repo_id (str) -> token (bool or str)`.
+
+ Returns:
+ `dict[str, Any]`
+ """
+
+ return {
+ column_name: decode_nested_example(feature, value, token_per_repo_id=token_per_repo_id)
+ if self._column_requires_decoding[column_name]
+ else value
+ for column_name, (feature, value) in zip_dict(
+ {key: value for key, value in self.items() if key in example}, example
+ )
+ }
+
+ def decode_column(
+ self, column: list, column_name: str, token_per_repo_id: Optional[dict[str, Union[str, bool, None]]] = None
+ ):
+ """Decode column with custom feature decoding.
+
+ Args:
+ column (`list[Any]`):
+ Dataset column data.
+ column_name (`str`):
+ Dataset column name.
+
+ Returns:
+ `list[Any]`
+ """
+ return (
+ [
+ decode_nested_example(self[column_name], value, token_per_repo_id=token_per_repo_id)
+ if value is not None
+ else None
+ for value in column
+ ]
+ if self._column_requires_decoding[column_name]
+ else column
+ )
+
+ def decode_batch(self, batch: dict, token_per_repo_id: Optional[dict[str, Union[str, bool, None]]] = None):
+ """Decode batch with custom feature decoding.
+
+ Args:
+ batch (`dict[str, list[Any]]`):
+ Dataset batch data.
+ token_per_repo_id (`dict`, *optional*):
+ To access and decode audio or image files from private repositories on the Hub, you can pass
+ a dictionary repo_id (str) -> token (bool or str)
+
+ Returns:
+ `dict[str, list[Any]]`
+ """
+ decoded_batch = {}
+ for column_name, column in batch.items():
+ decoded_batch[column_name] = (
+ [
+ decode_nested_example(self[column_name], value, token_per_repo_id=token_per_repo_id)
+ if value is not None
+ else None
+ for value in column
+ ]
+ if self._column_requires_decoding[column_name]
+ else column
+ )
+ return decoded_batch
+
+ def copy(self) -> "Features":
+ """
+ Make a deep copy of [`Features`].
+
+ Returns:
+ [`Features`]
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train")
+ >>> copy_of_features = ds.features.copy()
+ >>> copy_of_features
+ {'label': ClassLabel(names=['neg', 'pos']),
+ 'text': Value('string')}
+ ```
+ """
+ return copy.deepcopy(self)
+
+ def reorder_fields_as(self, other: "Features") -> "Features":
+ """
+ Reorder Features fields to match the field order of other [`Features`].
+
+ The order of the fields is important since it matters for the underlying arrow data.
+ Re-ordering the fields allows to make the underlying arrow data type match.
+
+ Args:
+ other ([`Features`]):
+ The other [`Features`] to align with.
+
+ Returns:
+ [`Features`]
+
+ Example::
+
+ >>> from datasets import Features, List, Value
+ >>> # let's say we have two features with a different order of nested fields (for a and b for example)
+ >>> f1 = Features({"root": {"a": Value("string"), "b": Value("string")}})
+ >>> f2 = Features({"root": {"b": Value("string"), "a": Value("string")}})
+ >>> assert f1.type != f2.type
+ >>> # re-ordering keeps the base structure (here List is defined at the root level), but makes the fields order match
+ >>> f1.reorder_fields_as(f2)
+ {'root': List({'b': Value('string'), 'a': Value('string')})}
+ >>> assert f1.reorder_fields_as(f2).type == f2.type
+ """
+
+ def recursive_reorder(source, target, stack=""):
+ stack_position = " at " + stack[1:] if stack else ""
+ if isinstance(source, dict):
+ if not isinstance(target, dict):
+ raise ValueError(f"Type mismatch: between {source} and {target}" + stack_position)
+ if sorted(source) != sorted(target):
+ message = (
+ f"Keys mismatch: between {source} (source) and {target} (target).\n"
+ f"{source.keys() - target.keys()} are missing from target "
+ f"and {target.keys() - source.keys()} are missing from source" + stack_position
+ )
+ raise ValueError(message)
+ return {key: recursive_reorder(source[key], target[key], stack + f".{key}") for key in target}
+ elif isinstance(source, List):
+ if not isinstance(target, List):
+ raise ValueError(f"Type mismatch: between {source} and {target}" + stack_position)
+ return List(recursive_reorder(source.feature, target.feature, stack + "."), length=source.length)
+ elif isinstance(source, LargeList):
+ if not isinstance(target, LargeList):
+ raise ValueError(f"Type mismatch: between {source} and {target}" + stack_position)
+ return LargeList(recursive_reorder(source.feature, target.feature, stack + "."))
+ else:
+ return source
+
+ return Features(recursive_reorder(self, other))
+
+ def flatten(self, max_depth=16) -> "Features":
+ """Flatten the features. Every dictionary column is removed and is replaced by
+ all the subfields it contains. The new fields are named by concatenating the
+ name of the original column and the subfield name like this: `.`.
+
+ If a column contains nested dictionaries, then all the lower-level subfields names are
+ also concatenated to form new columns: `..`, etc.
+
+ Returns:
+ [`Features`]:
+ The flattened features.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("rajpurkar/squad", split="train")
+ >>> ds.features.flatten()
+ {'answers.answer_start': List(Value('int32'), id=None),
+ 'answers.text': List(Value('string'), id=None),
+ 'context': Value('string'),
+ 'id': Value('string'),
+ 'question': Value('string'),
+ 'title': Value('string')}
+ ```
+ """
+ for depth in range(1, max_depth):
+ no_change = True
+ flattened = self.copy()
+ for column_name, subfeature in self.items():
+ if isinstance(subfeature, dict):
+ no_change = False
+ flattened.update({f"{column_name}.{k}": v for k, v in subfeature.items()})
+ del flattened[column_name]
+ elif hasattr(subfeature, "flatten") and subfeature.flatten() != subfeature:
+ no_change = False
+ flattened.update({f"{column_name}.{k}": v for k, v in subfeature.flatten().items()})
+ del flattened[column_name]
+ self = flattened
+ if no_change:
+ break
+ return self
+
+
+def _align_features(features_list: list[Features]) -> list[Features]:
+ """Align dictionaries of features so that the keys that are found in multiple dictionaries share the same feature."""
+ name2feature = {}
+ for features in features_list:
+ for k, v in features.items():
+ if k in name2feature and isinstance(v, dict):
+ # Recursively align features.
+ name2feature[k] = _align_features([name2feature[k], v])[0]
+ elif k not in name2feature or (isinstance(name2feature[k], Value) and name2feature[k].dtype == "null"):
+ name2feature[k] = v
+
+ return [Features({k: name2feature[k] for k in features.keys()}) for features in features_list]
+
+
+def _check_if_features_can_be_aligned(features_list: list[Features]):
+ """Check if the dictionaries of features can be aligned.
+
+ Two dictonaries of features can be aligned if the keys they share have the same type or some of them is of type `Value("null")`.
+ """
+ name2feature = {}
+ for features in features_list:
+ for k, v in features.items():
+ if k not in name2feature or (isinstance(name2feature[k], Value) and name2feature[k].dtype == "null"):
+ name2feature[k] = v
+
+ for features in features_list:
+ for k, v in features.items():
+ if isinstance(v, dict) and isinstance(name2feature[k], dict):
+ # Deep checks for structure.
+ _check_if_features_can_be_aligned([name2feature[k], v])
+ elif not (isinstance(v, Value) and v.dtype == "null") and name2feature[k] != v:
+ raise ValueError(
+ f'The features can\'t be aligned because the key {k} of features {features} has unexpected type - {v} (expected either {name2feature[k]} or Value("null").'
+ )
+
+
+def _fix_for_backward_compatible_features(feature: Any) -> FeatureType:
+ def _fix_old_list(feature):
+ if isinstance(feature, list):
+ return List(_fix_for_backward_compatible_features(feature[0]))
+ return feature
+
+ return _visit(feature, _fix_old_list)
diff --git a/venv/lib/python3.10/site-packages/datasets/features/image.py b/venv/lib/python3.10/site-packages/datasets/features/image.py
new file mode 100644
index 0000000000000000000000000000000000000000..cb746b9219df94bfd69e975a48df6663121207f3
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/features/image.py
@@ -0,0 +1,406 @@
+import os
+import sys
+import warnings
+from dataclasses import dataclass, field
+from io import BytesIO
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union
+
+import numpy as np
+import pyarrow as pa
+
+from .. import config
+from ..download.download_config import DownloadConfig
+from ..table import array_cast
+from ..utils.file_utils import is_local_path, xopen
+from ..utils.py_utils import first_non_null_value, no_op_if_value_is_null, string_to_dict
+
+
+if TYPE_CHECKING:
+ import PIL.Image
+
+ from .features import FeatureType
+
+
+_IMAGE_COMPRESSION_FORMATS: Optional[list[str]] = None
+_NATIVE_BYTEORDER = "<" if sys.byteorder == "little" else ">"
+# Origin: https://github.com/python-pillow/Pillow/blob/698951e19e19972aeed56df686868f1329981c12/src/PIL/Image.py#L3126 minus "|i1" which values are not preserved correctly when saving and loading an image
+_VALID_IMAGE_ARRAY_DTPYES = [
+ np.dtype("|b1"),
+ np.dtype("|u1"),
+ np.dtype("u2"),
+ np.dtype("i2"),
+ np.dtype("u4"),
+ np.dtype("i4"),
+ np.dtype("f4"),
+ np.dtype("f8"),
+]
+
+
+@dataclass
+class Image:
+ """Image [`Feature`] to read image data from an image file.
+
+ Input: The Image feature accepts as input:
+ - A `str`: Absolute path to the image file (i.e. random access is allowed).
+ - A `pathlib.Path`: path to the image file (i.e. random access is allowed).
+ - A `dict` with the keys:
+
+ - `path`: String with relative path of the image file to the archive file.
+ - `bytes`: Bytes of the image file.
+
+ This is useful for parquet or webdataset files which embed image files.
+
+ - An `np.ndarray`: NumPy array representing an image.
+ - A `PIL.Image.Image`: PIL image object.
+
+ Output: The Image features output data as `PIL.Image.Image` objects.
+
+ Args:
+ mode (`str`, *optional*):
+ The mode to convert the image to. If `None`, the native mode of the image is used.
+ decode (`bool`, defaults to `True`):
+ Whether to decode the image data. If `False`,
+ returns the underlying dictionary in the format `{"path": image_path, "bytes": image_bytes}`.
+
+ Examples:
+
+ ```py
+ >>> from datasets import load_dataset, Image
+ >>> ds = load_dataset("AI-Lab-Makerere/beans", split="train")
+ >>> ds.features["image"]
+ Image(decode=True, id=None)
+ >>> ds[0]["image"]
+
+ >>> ds = ds.cast_column('image', Image(decode=False))
+ {'bytes': None,
+ 'path': '/root/.cache/huggingface/datasets/downloads/extracted/b0a21163f78769a2cf11f58dfc767fb458fc7cea5c05dccc0144a2c0f0bc1292/train/healthy/healthy_train.85.jpg'}
+ ```
+ """
+
+ mode: Optional[str] = None
+ decode: bool = True
+ id: Optional[str] = field(default=None, repr=False)
+ # Automatically constructed
+ dtype: ClassVar[str] = "PIL.Image.Image"
+ pa_type: ClassVar[Any] = pa.struct({"bytes": pa.binary(), "path": pa.string()})
+ _type: str = field(default="Image", init=False, repr=False)
+
+ def __call__(self):
+ return self.pa_type
+
+ def encode_example(self, value: Union[str, bytes, bytearray, dict, np.ndarray, "PIL.Image.Image"]) -> dict:
+ """Encode example into a format for Arrow.
+
+ Args:
+ value (`str`, `np.ndarray`, `PIL.Image.Image` or `dict`):
+ Data passed as input to Image feature.
+
+ Returns:
+ `dict` with "path" and "bytes" fields
+ """
+ if config.PIL_AVAILABLE:
+ import PIL.Image
+ else:
+ raise ImportError("To support encoding images, please install 'Pillow'.")
+
+ if isinstance(value, list):
+ value = np.array(value)
+
+ if isinstance(value, str):
+ return {"path": value, "bytes": None}
+ elif isinstance(value, Path):
+ return {"path": str(value.absolute()), "bytes": None}
+ elif isinstance(value, (bytes, bytearray)):
+ return {"path": None, "bytes": value}
+ elif isinstance(value, np.ndarray):
+ # convert the image array to PNG/TIFF bytes
+ return encode_np_array(value)
+ elif isinstance(value, PIL.Image.Image):
+ # convert the PIL image to bytes (default format is PNG/TIFF)
+ return encode_pil_image(value)
+ elif value.get("path") is not None and os.path.isfile(value["path"]):
+ # we set "bytes": None to not duplicate the data if they're already available locally
+ return {"bytes": None, "path": value.get("path")}
+ elif value.get("bytes") is not None or value.get("path") is not None:
+ # store the image bytes, and path is used to infer the image format using the file extension
+ return {"bytes": value.get("bytes"), "path": value.get("path")}
+ else:
+ raise ValueError(
+ f"An image sample should have one of 'path' or 'bytes' but they are missing or None in {value}."
+ )
+
+ def decode_example(self, value: dict, token_per_repo_id=None) -> "PIL.Image.Image":
+ """Decode example image file into image data.
+
+ Args:
+ value (`str` or `dict`):
+ A string with the absolute image file path, a dictionary with
+ keys:
+
+ - `path`: String with absolute or relative image file path.
+ - `bytes`: The bytes of the image file.
+ token_per_repo_id (`dict`, *optional*):
+ To access and decode
+ image files from private repositories on the Hub, you can pass
+ a dictionary repo_id (`str`) -> token (`bool` or `str`).
+
+ Returns:
+ `PIL.Image.Image`
+ """
+ if not self.decode:
+ raise RuntimeError("Decoding is disabled for this feature. Please use Image(decode=True) instead.")
+
+ if config.PIL_AVAILABLE:
+ import PIL.Image
+ import PIL.ImageOps
+ else:
+ raise ImportError("To support decoding images, please install 'Pillow'.")
+
+ if token_per_repo_id is None:
+ token_per_repo_id = {}
+
+ path, bytes_ = value["path"], value["bytes"]
+ if bytes_ is None:
+ if path is None:
+ raise ValueError(f"An image should have one of 'path' or 'bytes' but both are None in {value}.")
+ else:
+ if is_local_path(path):
+ image = PIL.Image.open(path)
+ else:
+ source_url = path.split("::")[-1]
+ pattern = (
+ config.HUB_DATASETS_URL
+ if source_url.startswith(config.HF_ENDPOINT)
+ else config.HUB_DATASETS_HFFS_URL
+ )
+ source_url_fields = string_to_dict(source_url, pattern)
+ token = (
+ token_per_repo_id.get(source_url_fields["repo_id"]) if source_url_fields is not None else None
+ )
+ download_config = DownloadConfig(token=token)
+ with xopen(path, "rb", download_config=download_config) as f:
+ bytes_ = BytesIO(f.read())
+ image = PIL.Image.open(bytes_)
+ else:
+ image = PIL.Image.open(BytesIO(bytes_))
+ image.load() # to avoid "Too many open files" errors
+ if image.getexif().get(PIL.Image.ExifTags.Base.Orientation) is not None:
+ image = PIL.ImageOps.exif_transpose(image)
+ if self.mode and self.mode != image.mode:
+ image = image.convert(self.mode)
+ return image
+
+ def flatten(self) -> Union["FeatureType", dict[str, "FeatureType"]]:
+ """If in the decodable state, return the feature itself, otherwise flatten the feature into a dictionary."""
+ from .features import Value
+
+ return (
+ self
+ if self.decode
+ else {
+ "bytes": Value("binary"),
+ "path": Value("string"),
+ }
+ )
+
+ def cast_storage(self, storage: Union[pa.StringArray, pa.StructArray, pa.ListArray]) -> pa.StructArray:
+ """Cast an Arrow array to the Image arrow storage type.
+ The Arrow types that can be converted to the Image pyarrow storage type are:
+
+ - `pa.string()` - it must contain the "path" data
+ - `pa.large_string()` - it must contain the "path" data (will be cast to string if possible)
+ - `pa.binary()` - it must contain the image bytes
+ - `pa.struct({"bytes": pa.binary()})`
+ - `pa.struct({"path": pa.string()})`
+ - `pa.struct({"bytes": pa.binary(), "path": pa.string()})` - order doesn't matter
+ - `pa.list(*)` - it must contain the image array data
+
+ Args:
+ storage (`Union[pa.StringArray, pa.StructArray, pa.ListArray]`):
+ PyArrow array to cast.
+
+ Returns:
+ `pa.StructArray`: Array in the Image arrow storage type, that is
+ `pa.struct({"bytes": pa.binary(), "path": pa.string()})`.
+ """
+ if pa.types.is_large_string(storage.type):
+ try:
+ storage = storage.cast(pa.string())
+ except pa.ArrowInvalid as e:
+ raise ValueError(
+ f"Failed to cast large_string to string for Image feature. "
+ f"This can happen if string values exceed 2GB. "
+ f"Original error: {e}"
+ ) from e
+ if pa.types.is_string(storage.type):
+ bytes_array = pa.array([None] * len(storage), type=pa.binary())
+ storage = pa.StructArray.from_arrays([bytes_array, storage], ["bytes", "path"], mask=storage.is_null())
+ elif pa.types.is_binary(storage.type):
+ path_array = pa.array([None] * len(storage), type=pa.string())
+ storage = pa.StructArray.from_arrays([storage, path_array], ["bytes", "path"], mask=storage.is_null())
+ elif pa.types.is_struct(storage.type):
+ if storage.type.get_field_index("bytes") >= 0:
+ bytes_array = storage.field("bytes")
+ else:
+ bytes_array = pa.array([None] * len(storage), type=pa.binary())
+ if storage.type.get_field_index("path") >= 0:
+ path_array = storage.field("path")
+ else:
+ path_array = pa.array([None] * len(storage), type=pa.string())
+ storage = pa.StructArray.from_arrays([bytes_array, path_array], ["bytes", "path"], mask=storage.is_null())
+ elif pa.types.is_list(storage.type):
+ bytes_array = pa.array(
+ [encode_np_array(np.array(arr))["bytes"] if arr is not None else None for arr in storage.to_pylist()],
+ type=pa.binary(),
+ )
+ path_array = pa.array([None] * len(storage), type=pa.string())
+ storage = pa.StructArray.from_arrays(
+ [bytes_array, path_array], ["bytes", "path"], mask=bytes_array.is_null()
+ )
+ return array_cast(storage, self.pa_type)
+
+ def embed_storage(self, storage: pa.StructArray, token_per_repo_id=None) -> pa.StructArray:
+ """Embed image files into the Arrow array.
+
+ Args:
+ storage (`pa.StructArray`):
+ PyArrow array to embed.
+
+ Returns:
+ `pa.StructArray`: Array in the Image arrow storage type, that is
+ `pa.struct({"bytes": pa.binary(), "path": pa.string()})`.
+ """
+ if token_per_repo_id is None:
+ token_per_repo_id = {}
+
+ @no_op_if_value_is_null
+ def path_to_bytes(path):
+ source_url = path.split("::")[-1]
+ pattern = (
+ config.HUB_DATASETS_URL if source_url.startswith(config.HF_ENDPOINT) else config.HUB_DATASETS_HFFS_URL
+ )
+ source_url_fields = string_to_dict(source_url, pattern)
+ token = token_per_repo_id.get(source_url_fields["repo_id"]) if source_url_fields is not None else None
+ download_config = DownloadConfig(token=token)
+ with xopen(path, "rb", download_config=download_config) as f:
+ return f.read()
+
+ bytes_array = pa.array(
+ [
+ (path_to_bytes(x["path"]) if x["bytes"] is None else x["bytes"]) if x is not None else None
+ for x in storage.to_pylist()
+ ],
+ type=pa.binary(),
+ )
+ path_array = pa.array(
+ [os.path.basename(path) if path is not None else None for path in storage.field("path").to_pylist()],
+ type=pa.string(),
+ )
+ storage = pa.StructArray.from_arrays([bytes_array, path_array], ["bytes", "path"], mask=bytes_array.is_null())
+ return array_cast(storage, self.pa_type)
+
+
+def list_image_compression_formats() -> list[str]:
+ if config.PIL_AVAILABLE:
+ import PIL.Image
+ else:
+ raise ImportError("To support encoding images, please install 'Pillow'.")
+
+ global _IMAGE_COMPRESSION_FORMATS
+ if _IMAGE_COMPRESSION_FORMATS is None:
+ PIL.Image.init()
+ _IMAGE_COMPRESSION_FORMATS = list(set(PIL.Image.OPEN.keys()) & set(PIL.Image.SAVE.keys()))
+ return _IMAGE_COMPRESSION_FORMATS
+
+
+def image_to_bytes(image: "PIL.Image.Image") -> bytes:
+ """Convert a PIL Image object to bytes using native compression if possible, otherwise use PNG/TIFF compression."""
+ buffer = BytesIO()
+ if image.format in list_image_compression_formats():
+ format = image.format
+ else:
+ format = "PNG" if image.mode in ["1", "L", "LA", "RGB", "RGBA"] else "TIFF"
+ image.save(buffer, format=format)
+ return buffer.getvalue()
+
+
+def encode_pil_image(image: "PIL.Image.Image") -> dict:
+ if hasattr(image, "filename") and image.filename != "":
+ return {"path": image.filename, "bytes": None}
+ else:
+ return {"path": None, "bytes": image_to_bytes(image)}
+
+
+def encode_np_array(array: np.ndarray) -> dict:
+ if config.PIL_AVAILABLE:
+ import PIL.Image
+ else:
+ raise ImportError("To support encoding images, please install 'Pillow'.")
+
+ dtype = array.dtype
+ dtype_byteorder = dtype.byteorder if dtype.byteorder != "=" else _NATIVE_BYTEORDER
+ dtype_kind = dtype.kind
+ dtype_itemsize = dtype.itemsize
+
+ dest_dtype = None
+
+ # Multi-channel array case (only np.dtype("|u1") is allowed)
+ if array.shape[2:]:
+ if dtype_kind not in ["u", "i"]:
+ raise TypeError(
+ f"Unsupported array dtype {dtype} for image encoding. Only {dest_dtype} is supported for multi-channel arrays."
+ )
+ dest_dtype = np.dtype("|u1")
+ if dtype != dest_dtype:
+ warnings.warn(f"Downcasting array dtype {dtype} to {dest_dtype} to be compatible with 'Pillow'")
+ # Exact match
+ elif dtype in _VALID_IMAGE_ARRAY_DTPYES:
+ dest_dtype = dtype
+ else: # Downcast the type within the kind (np.can_cast(from_type, to_type, casting="same_kind") doesn't behave as expected, so do it manually)
+ while dtype_itemsize >= 1:
+ dtype_str = dtype_byteorder + dtype_kind + str(dtype_itemsize)
+ if np.dtype(dtype_str) in _VALID_IMAGE_ARRAY_DTPYES:
+ dest_dtype = np.dtype(dtype_str)
+ warnings.warn(f"Downcasting array dtype {dtype} to {dest_dtype} to be compatible with 'Pillow'")
+ break
+ else:
+ dtype_itemsize //= 2
+ if dest_dtype is None:
+ raise TypeError(
+ f"Cannot downcast dtype {dtype} to a valid image dtype. Valid image dtypes: {_VALID_IMAGE_ARRAY_DTPYES}"
+ )
+
+ image = PIL.Image.fromarray(array.astype(dest_dtype))
+ return {"path": None, "bytes": image_to_bytes(image)}
+
+
+def objects_to_list_of_image_dicts(
+ objs: Union[list[str], list[dict], list[np.ndarray], list["PIL.Image.Image"]],
+) -> list[dict]:
+ """Encode a list of objects into a format suitable for creating an extension array of type `ImageExtensionType`."""
+ if config.PIL_AVAILABLE:
+ import PIL.Image
+ else:
+ raise ImportError("To support encoding images, please install 'Pillow'.")
+
+ if objs:
+ _, obj = first_non_null_value(objs)
+ if isinstance(obj, str):
+ return [{"path": obj, "bytes": None} if obj is not None else None for obj in objs]
+ if isinstance(obj, np.ndarray):
+ obj_to_image_dict_func = no_op_if_value_is_null(encode_np_array)
+ return [obj_to_image_dict_func(obj) for obj in objs]
+ elif isinstance(obj, PIL.Image.Image):
+ obj_to_image_dict_func = no_op_if_value_is_null(encode_pil_image)
+ return [obj_to_image_dict_func(obj) for obj in objs]
+ else:
+ return objs
+ else:
+ return objs
diff --git a/venv/lib/python3.10/site-packages/datasets/features/nifti.py b/venv/lib/python3.10/site-packages/datasets/features/nifti.py
new file mode 100644
index 0000000000000000000000000000000000000000..bac91e2af4bdf8f7a04a28ed0a293677f01bca32
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/features/nifti.py
@@ -0,0 +1,243 @@
+import os
+from dataclasses import dataclass, field
+from io import BytesIO
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Union
+
+import pyarrow as pa
+
+from .. import config
+from ..download.download_config import DownloadConfig
+from ..table import array_cast
+from ..utils.file_utils import is_local_path, xopen
+from ..utils.py_utils import string_to_dict
+
+
+if TYPE_CHECKING:
+ import nibabel as nib
+
+ from .features import FeatureType
+
+
+@dataclass
+class Nifti:
+ """
+ **Experimental.**
+ Nifti [`Feature`] to read NIfTI neuroimaging files.
+
+ Input: The Nifti feature accepts as input:
+ - A `str`: Absolute path to the NIfTI file (i.e. random access is allowed).
+ - A `pathlib.Path`: path to the NIfTI file (i.e. random access is allowed).
+ - A `dict` with the keys:
+ - `path`: String with relative path of the NIfTI file in a dataset repository.
+ - `bytes`: Bytes of the NIfTI file.
+ This is useful for archived files with sequential access.
+
+ - A `nibabel` image object (e.g., `nibabel.nifti1.Nifti1Image`).
+
+ Args:
+ decode (`bool`, defaults to `True`):
+ Whether to decode the NIfTI data. If `False` a string with the bytes is returned. `decode=False` is not supported when decoding examples.
+
+ Examples:
+
+ ```py
+ >>> from datasets import Dataset, Nifti
+ >>> ds = Dataset.from_dict({"nifti": ["path/to/file.nii.gz"]}).cast_column("nifti", Nifti())
+ >>> ds.features["nifti"]
+ Nifti(decode=True, id=None)
+ >>> ds[0]["nifti"]
+
+ >>> ds = ds.cast_column("nifti", Nifti(decode=False))
+ >>> ds[0]["nifti"]
+ {'bytes': None,
+ 'path': 'path/to/file.nii.gz'}
+ ```
+ """
+
+ decode: bool = True
+ id: Optional[str] = field(default=None, repr=False)
+
+ # Automatically constructed
+ dtype: ClassVar[str] = "nibabel.nifti1.Nifti1Image"
+ pa_type: ClassVar[Any] = pa.struct({"bytes": pa.binary(), "path": pa.string()})
+ _type: str = field(default="Nifti", init=False, repr=False)
+
+ def __call__(self):
+ return self.pa_type
+
+ def encode_example(self, value: Union[str, bytes, bytearray, dict, "nib.Nifti1Image"]) -> dict:
+ """Encode example into a format for Arrow.
+
+ Args:
+ value (`str`, `bytes`, `nibabel.Nifti1Image` or `dict`):
+ Data passed as input to Nifti feature.
+
+ Returns:
+ `dict` with "path" and "bytes" fields
+ """
+ if config.NIBABEL_AVAILABLE:
+ import nibabel as nib
+ else:
+ nib = None
+
+ if isinstance(value, str):
+ return {"path": value, "bytes": None}
+ elif isinstance(value, Path):
+ return {"path": str(value.absolute()), "bytes": None}
+ elif isinstance(value, (bytes, bytearray)):
+ return {"path": None, "bytes": value}
+ elif nib is not None and isinstance(value, nib.spatialimages.SpatialImage):
+ # nibabel image object - try to get path or convert to bytes
+ return encode_nibabel_image(value)
+ elif isinstance(value, dict):
+ if value.get("path") is not None and os.path.isfile(value["path"]):
+ # we set "bytes": None to not duplicate the data if they're already available locally
+ return {"bytes": None, "path": value.get("path")}
+ elif value.get("bytes") is not None or value.get("path") is not None:
+ # store the nifti bytes, and path is used to infer the format using the file extension
+ return {"bytes": value.get("bytes"), "path": value.get("path")}
+ else:
+ raise ValueError(
+ f"A nifti sample should have one of 'path' or 'bytes' but they are missing or None in {value}."
+ )
+ else:
+ raise ValueError(
+ f"A nifti sample should be a string, bytes, Path, nibabel image, or dict, but got {type(value)}."
+ )
+
+ def decode_example(self, value: dict, token_per_repo_id=None) -> "nib.nifti1.Nifti1Image":
+ """Decode example NIfTI file into nibabel image object.
+
+ Args:
+ value (`str` or `dict`):
+ A string with the absolute NIfTI file path, a dictionary with
+ keys:
+
+ - `path`: String with absolute or relative NIfTI file path.
+ - `bytes`: The bytes of the NIfTI file.
+
+ token_per_repo_id (`dict`, *optional*):
+ To access and decode NIfTI files from private repositories on
+ the Hub, you can pass a dictionary
+ repo_id (`str`) -> token (`bool` or `str`).
+
+ Returns:
+ `nibabel.Nifti1Image` objects
+ """
+ if not self.decode:
+ raise NotImplementedError("Decoding is disabled for this feature. Please use Nifti(decode=True) instead.")
+
+ if config.NIBABEL_AVAILABLE:
+ import nibabel as nib
+ else:
+ raise ImportError("To support decoding NIfTI files, please install 'nibabel'.")
+
+ if token_per_repo_id is None:
+ token_per_repo_id = {}
+
+ path, bytes_ = value["path"], value["bytes"]
+ if bytes_ is None:
+ if path is None:
+ raise ValueError(f"A nifti should have one of 'path' or 'bytes' but both are None in {value}.")
+ else:
+ if is_local_path(path):
+ nifti = nib.load(path)
+ else:
+ source_url = path.split("::")[-1]
+ pattern = (
+ config.HUB_DATASETS_URL
+ if source_url.startswith(config.HF_ENDPOINT)
+ else config.HUB_DATASETS_HFFS_URL
+ )
+ try:
+ repo_id = string_to_dict(source_url, pattern)["repo_id"]
+ token = token_per_repo_id.get(repo_id)
+ except ValueError:
+ token = None
+ download_config = DownloadConfig(token=token)
+ with xopen(path, "rb", download_config=download_config) as f:
+ nifti = nib.load(f)
+ else:
+ import gzip
+
+ if (
+ bytes_[:2] == b"\x1f\x8b"
+ ): # gzip magic number, see https://stackoverflow.com/a/76055284/9534390 or "Magic number" on https://en.wikipedia.org/wiki/Gzip
+ bytes_ = gzip.decompress(bytes_)
+
+ bio = BytesIO(bytes_)
+ fh = nib.FileHolder(fileobj=bio)
+ nifti = nib.Nifti1Image.from_file_map({"header": fh, "image": fh})
+
+ return nifti
+
+ def flatten(self) -> Union["FeatureType", Dict[str, "FeatureType"]]:
+ """If in the decodable state, return the feature itself, otherwise flatten the feature into a dictionary."""
+ from .features import Value
+
+ return (
+ self
+ if self.decode
+ else {
+ "bytes": Value("binary"),
+ "path": Value("string"),
+ }
+ )
+
+ def cast_storage(self, storage: Union[pa.StringArray, pa.StructArray, pa.BinaryArray]) -> pa.StructArray:
+ """Cast an Arrow array to the Nifti arrow storage type.
+ The Arrow types that can be converted to the Nifti pyarrow storage type are:
+
+ - `pa.string()` - it must contain the "path" data
+ - `pa.binary()` - it must contain the NIfTI bytes
+ - `pa.struct({"bytes": pa.binary()})`
+ - `pa.struct({"path": pa.string()})`
+ - `pa.struct({"bytes": pa.binary(), "path": pa.string()})` - order doesn't matter
+
+ Args:
+ storage (`Union[pa.StringArray, pa.StructArray, pa.BinaryArray]`):
+ PyArrow array to cast.
+
+ Returns:
+ `pa.StructArray`: Array in the Nifti arrow storage type, that is
+ `pa.struct({"bytes": pa.binary(), "path": pa.string()})`.
+ """
+ if pa.types.is_string(storage.type):
+ bytes_array = pa.array([None] * len(storage), type=pa.binary())
+ storage = pa.StructArray.from_arrays([bytes_array, storage], ["bytes", "path"], mask=storage.is_null())
+ elif pa.types.is_binary(storage.type):
+ path_array = pa.array([None] * len(storage), type=pa.string())
+ storage = pa.StructArray.from_arrays([storage, path_array], ["bytes", "path"], mask=storage.is_null())
+ elif pa.types.is_struct(storage.type):
+ if storage.type.get_field_index("bytes") >= 0:
+ bytes_array = storage.field("bytes")
+ else:
+ bytes_array = pa.array([None] * len(storage), type=pa.binary())
+ if storage.type.get_field_index("path") >= 0:
+ path_array = storage.field("path")
+ else:
+ path_array = pa.array([None] * len(storage), type=pa.string())
+ storage = pa.StructArray.from_arrays([bytes_array, path_array], ["bytes", "path"], mask=storage.is_null())
+ return array_cast(storage, self.pa_type)
+
+
+def encode_nibabel_image(img: "nib.Nifti1Image") -> dict[str, Optional[Union[str, bytes]]]:
+ """
+ Encode a nibabel image object into a dictionary.
+
+ If the image has an associated file path, returns the path. Otherwise, serializes
+ the image content into bytes.
+
+ Args:
+ img: A nibabel image object (e.g., Nifti1Image).
+
+ Returns:
+ dict: A dictionary with "path" or "bytes" field.
+ """
+ if hasattr(img, "file_map") and img.file_map is not None:
+ filename = img.file_map["image"].filename
+ return {"path": filename, "bytes": None}
+
+ bytes_data = img.to_bytes()
+ return {"path": None, "bytes": bytes_data}
diff --git a/venv/lib/python3.10/site-packages/datasets/features/pdf.py b/venv/lib/python3.10/site-packages/datasets/features/pdf.py
new file mode 100644
index 0000000000000000000000000000000000000000..756530554d456b9e57660839bb8c75f286c42569
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/features/pdf.py
@@ -0,0 +1,280 @@
+import os
+from dataclasses import dataclass, field
+from io import BytesIO
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Union
+
+import pyarrow as pa
+
+from .. import config
+from ..download.download_config import DownloadConfig
+from ..table import array_cast
+from ..utils.file_utils import is_local_path, xopen
+from ..utils.py_utils import no_op_if_value_is_null, string_to_dict
+
+
+if TYPE_CHECKING:
+ import pdfplumber
+
+ from .features import FeatureType
+
+
+def pdf_to_bytes(pdf: "pdfplumber.pdf.PDF") -> bytes:
+ """Convert a pdfplumber.pdf.PDF object to bytes."""
+ with BytesIO() as buffer:
+ for page in pdf.pages:
+ buffer.write(page.pdf.stream)
+ return buffer.getvalue()
+
+
+@dataclass
+class Pdf:
+ """
+ **Experimental.**
+ Pdf [`Feature`] to read pdf documents from a pdf file.
+
+ Input: The Pdf feature accepts as input:
+ - A `str`: Absolute path to the pdf file (i.e. random access is allowed).
+ - A `pathlib.Path`: path to the pdf file (i.e. random access is allowed).
+ - A `dict` with the keys:
+ - `path`: String with relative path of the pdf file in a dataset repository.
+ - `bytes`: Bytes of the pdf file.
+ This is useful for archived files with sequential access.
+
+ - A `pdfplumber.pdf.PDF`: pdfplumber pdf object.
+
+ Args:
+ decode (`bool`, defaults to `True`):
+ Whether to decode the pdf data. If `False`,
+ returns the underlying dictionary in the format `{"path": pdf_path, "bytes": pdf_bytes}`.
+
+ Examples:
+
+ ```py
+ >>> from datasets import Dataset, Pdf
+ >>> ds = Dataset.from_dict({"pdf": ["path/to/pdf/file.pdf"]}).cast_column("pdf", Pdf())
+ >>> ds.features["pdf"]
+ Pdf(decode=True, id=None)
+ >>> ds[0]["pdf"]
+
+ >>> ds = ds.cast_column("pdf", Pdf(decode=False))
+ >>> ds[0]["pdf"]
+ {'bytes': None,
+ 'path': 'path/to/pdf/file.pdf'}
+ ```
+ """
+
+ decode: bool = True
+ id: Optional[str] = field(default=None, repr=False)
+
+ # Automatically constructed
+ dtype: ClassVar[str] = "pdfplumber.pdf.PDF"
+ pa_type: ClassVar[Any] = pa.struct({"bytes": pa.binary(), "path": pa.string()})
+ _type: str = field(default="Pdf", init=False, repr=False)
+
+ def __call__(self):
+ return self.pa_type
+
+ def encode_example(self, value: Union[str, bytes, bytearray, dict, "pdfplumber.pdf.PDF"]) -> dict:
+ """Encode example into a format for Arrow.
+
+ Args:
+ value (`str`, `bytes`, `pdfplumber.pdf.PDF` or `dict`):
+ Data passed as input to Pdf feature.
+
+ Returns:
+ `dict` with "path" and "bytes" fields
+ """
+ if config.PDFPLUMBER_AVAILABLE:
+ import pdfplumber
+ else:
+ pdfplumber = None
+
+ if isinstance(value, str):
+ return {"path": value, "bytes": None}
+ elif isinstance(value, Path):
+ return {"path": str(value.absolute()), "bytes": None}
+ elif isinstance(value, (bytes, bytearray)):
+ return {"path": None, "bytes": value}
+ elif pdfplumber is not None and isinstance(value, pdfplumber.pdf.PDF):
+ # convert the pdfplumber.pdf.PDF to bytes
+ return encode_pdfplumber_pdf(value)
+ elif value.get("path") is not None and os.path.isfile(value["path"]):
+ # we set "bytes": None to not duplicate the data if they're already available locally
+ return {"bytes": None, "path": value.get("path")}
+ elif value.get("bytes") is not None or value.get("path") is not None:
+ # store the pdf bytes, and path is used to infer the pdf format using the file extension
+ return {"bytes": value.get("bytes"), "path": value.get("path")}
+ else:
+ raise ValueError(
+ f"A pdf sample should have one of 'path' or 'bytes' but they are missing or None in {value}."
+ )
+
+ def decode_example(self, value: dict, token_per_repo_id=None) -> "pdfplumber.pdf.PDF":
+ """Decode example pdf file into pdf data.
+
+ Args:
+ value (`str` or `dict`):
+ A string with the absolute pdf file path, a dictionary with
+ keys:
+
+ - `path`: String with absolute or relative pdf file path.
+ - `bytes`: The bytes of the pdf file.
+
+ token_per_repo_id (`dict`, *optional*):
+ To access and decode pdf files from private repositories on
+ the Hub, you can pass a dictionary
+ repo_id (`str`) -> token (`bool` or `str`).
+
+ Returns:
+ `pdfplumber.pdf.PDF`
+ """
+ if not self.decode:
+ raise RuntimeError("Decoding is disabled for this feature. Please use Pdf(decode=True) instead.")
+
+ if config.PDFPLUMBER_AVAILABLE:
+ import pdfplumber
+ else:
+ raise ImportError("To support decoding pdfs, please install 'pdfplumber'.")
+
+ if token_per_repo_id is None:
+ token_per_repo_id = {}
+
+ path, bytes_ = value["path"], value["bytes"]
+ if bytes_ is None:
+ if path is None:
+ raise ValueError(f"A pdf should have one of 'path' or 'bytes' but both are None in {value}.")
+ else:
+ if is_local_path(path):
+ pdf = pdfplumber.open(path)
+ else:
+ source_url = path.split("::")[-1]
+ pattern = (
+ config.HUB_DATASETS_URL
+ if source_url.startswith(config.HF_ENDPOINT)
+ else config.HUB_DATASETS_HFFS_URL
+ )
+ try:
+ repo_id = string_to_dict(source_url, pattern)["repo_id"]
+ token = token_per_repo_id.get(repo_id)
+ except ValueError:
+ token = None
+ download_config = DownloadConfig(token=token)
+ f = xopen(path, "rb", download_config=download_config)
+ return pdfplumber.open(f)
+ else:
+ with pdfplumber.open(BytesIO(bytes_)) as p:
+ pdf = p
+
+ return pdf
+
+ def flatten(self) -> Union["FeatureType", Dict[str, "FeatureType"]]:
+ """If in the decodable state, return the feature itself, otherwise flatten the feature into a dictionary."""
+ from .features import Value
+
+ return (
+ self
+ if self.decode
+ else {
+ "bytes": Value("binary"),
+ "path": Value("string"),
+ }
+ )
+
+ def cast_storage(self, storage: Union[pa.StringArray, pa.StructArray, pa.ListArray]) -> pa.StructArray:
+ """Cast an Arrow array to the Pdf arrow storage type.
+ The Arrow types that can be converted to the Pdf pyarrow storage type are:
+
+ - `pa.string()` - it must contain the "path" data
+ - `pa.binary()` - it must contain the image bytes
+ - `pa.struct({"bytes": pa.binary()})`
+ - `pa.struct({"path": pa.string()})`
+ - `pa.struct({"bytes": pa.binary(), "path": pa.string()})` - order doesn't matter
+ - `pa.list(*)` - it must contain the pdf array data
+
+ Args:
+ storage (`Union[pa.StringArray, pa.StructArray, pa.ListArray]`):
+ PyArrow array to cast.
+
+ Returns:
+ `pa.StructArray`: Array in the Pdf arrow storage type, that is
+ `pa.struct({"bytes": pa.binary(), "path": pa.string()})`.
+ """
+ if pa.types.is_string(storage.type):
+ bytes_array = pa.array([None] * len(storage), type=pa.binary())
+ storage = pa.StructArray.from_arrays([bytes_array, storage], ["bytes", "path"], mask=storage.is_null())
+ elif pa.types.is_binary(storage.type):
+ path_array = pa.array([None] * len(storage), type=pa.string())
+ storage = pa.StructArray.from_arrays([storage, path_array], ["bytes", "path"], mask=storage.is_null())
+ elif pa.types.is_struct(storage.type):
+ if storage.type.get_field_index("bytes") >= 0:
+ bytes_array = storage.field("bytes")
+ else:
+ bytes_array = pa.array([None] * len(storage), type=pa.binary())
+ if storage.type.get_field_index("path") >= 0:
+ path_array = storage.field("path")
+ else:
+ path_array = pa.array([None] * len(storage), type=pa.string())
+ storage = pa.StructArray.from_arrays([bytes_array, path_array], ["bytes", "path"], mask=storage.is_null())
+ return array_cast(storage, self.pa_type)
+
+ def embed_storage(self, storage: pa.StructArray, token_per_repo_id=None) -> pa.StructArray:
+ """Embed PDF files into the Arrow array.
+
+ Args:
+ storage (`pa.StructArray`):
+ PyArrow array to embed.
+
+ Returns:
+ `pa.StructArray`: Array in the PDF arrow storage type, that is
+ `pa.struct({"bytes": pa.binary(), "path": pa.string()})`.
+ """
+ if token_per_repo_id is None:
+ token_per_repo_id = {}
+
+ @no_op_if_value_is_null
+ def path_to_bytes(path):
+ source_url = path.split("::")[-1]
+ pattern = (
+ config.HUB_DATASETS_URL if source_url.startswith(config.HF_ENDPOINT) else config.HUB_DATASETS_HFFS_URL
+ )
+ source_url_fields = string_to_dict(source_url, pattern)
+ token = token_per_repo_id.get(source_url_fields["repo_id"]) if source_url_fields is not None else None
+ download_config = DownloadConfig(token=token)
+ with xopen(path, "rb", download_config=download_config) as f:
+ return f.read()
+
+ bytes_array = pa.array(
+ [
+ (path_to_bytes(x["path"]) if x["bytes"] is None else x["bytes"]) if x is not None else None
+ for x in storage.to_pylist()
+ ],
+ type=pa.binary(),
+ )
+ path_array = pa.array(
+ [os.path.basename(path) if path is not None else None for path in storage.field("path").to_pylist()],
+ type=pa.string(),
+ )
+ storage = pa.StructArray.from_arrays([bytes_array, path_array], ["bytes", "path"], mask=bytes_array.is_null())
+ return array_cast(storage, self.pa_type)
+
+
+def encode_pdfplumber_pdf(pdf: "pdfplumber.pdf.PDF") -> dict:
+ """
+ Encode a pdfplumber.pdf.PDF object into a dictionary.
+
+ If the PDF has an associated file path, returns the path. Otherwise, serializes
+ the PDF content into bytes.
+
+ Args:
+ pdf (pdfplumber.pdf.PDF): A pdfplumber PDF object.
+
+ Returns:
+ dict: A dictionary with "path" or "bytes" field.
+ """
+ if hasattr(pdf, "stream") and hasattr(pdf.stream, "name") and pdf.stream.name:
+ # Return the path if the PDF has an associated file path
+ return {"path": pdf.stream.name, "bytes": None}
+ else:
+ # Convert the PDF to bytes if no path is available
+ return {"path": None, "bytes": pdf_to_bytes(pdf)}
diff --git a/venv/lib/python3.10/site-packages/datasets/features/translation.py b/venv/lib/python3.10/site-packages/datasets/features/translation.py
new file mode 100644
index 0000000000000000000000000000000000000000..9bee3aa2c11f26edbe0d17138a71fde96e3d3541
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/features/translation.py
@@ -0,0 +1,129 @@
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union
+
+import pyarrow as pa
+
+
+if TYPE_CHECKING:
+ from .features import FeatureType
+
+
+@dataclass
+class Translation:
+ """`Feature` for translations with fixed languages per example.
+ Here for compatibility with tfds.
+
+ Args:
+ languages (`dict`):
+ A dictionary for each example mapping string language codes to string translations.
+
+ Example:
+
+ ```python
+ >>> # At construction time:
+ >>> datasets.features.Translation(languages=['en', 'fr', 'de'])
+ >>> # During data generation:
+ >>> yield {
+ ... 'en': 'the cat',
+ ... 'fr': 'le chat',
+ ... 'de': 'die katze'
+ ... }
+ ```
+ """
+
+ languages: list[str]
+ id: Optional[str] = field(default=None, repr=False)
+ # Automatically constructed
+ dtype: ClassVar[str] = "dict"
+ pa_type: ClassVar[Any] = None
+ _type: str = field(default="Translation", init=False, repr=False)
+
+ def __call__(self):
+ return pa.struct({lang: pa.string() for lang in sorted(self.languages)})
+
+ def flatten(self) -> Union["FeatureType", dict[str, "FeatureType"]]:
+ """Flatten the Translation feature into a dictionary."""
+ from .features import Value
+
+ return {k: Value("string") for k in sorted(self.languages)}
+
+
+@dataclass
+class TranslationVariableLanguages:
+ """`Feature` for translations with variable languages per example.
+ Here for compatibility with tfds.
+
+ Args:
+ languages (`dict`):
+ A dictionary for each example mapping string language codes to one or more string translations.
+ The languages present may vary from example to example.
+
+ Returns:
+ - `language` or `translation` (variable-length 1D `tf.Tensor` of `tf.string`):
+ Language codes sorted in ascending order or plain text translations, sorted to align with language codes.
+
+ Example:
+
+ ```python
+ >>> # At construction time:
+ >>> datasets.features.TranslationVariableLanguages(languages=['en', 'fr', 'de'])
+ >>> # During data generation:
+ >>> yield {
+ ... 'en': 'the cat',
+ ... 'fr': ['le chat', 'la chatte,']
+ ... 'de': 'die katze'
+ ... }
+ >>> # Tensor returned :
+ >>> {
+ ... 'language': ['en', 'de', 'fr', 'fr'],
+ ... 'translation': ['the cat', 'die katze', 'la chatte', 'le chat'],
+ ... }
+ ```
+ """
+
+ languages: Optional[list] = None
+ num_languages: Optional[int] = None
+ id: Optional[str] = field(default=None, repr=False)
+ # Automatically constructed
+ dtype: ClassVar[str] = "dict"
+ pa_type: ClassVar[Any] = None
+ _type: str = field(default="TranslationVariableLanguages", init=False, repr=False)
+
+ def __post_init__(self):
+ self.languages = sorted(set(self.languages)) if self.languages else None
+ self.num_languages = len(self.languages) if self.languages else None
+
+ def __call__(self):
+ return pa.struct({"language": pa.list_(pa.string()), "translation": pa.list_(pa.string())})
+
+ def encode_example(self, translation_dict):
+ lang_set = set(self.languages)
+ if set(translation_dict) == {"language", "translation"}:
+ return translation_dict
+ elif self.languages and set(translation_dict) - lang_set:
+ raise ValueError(
+ f"Some languages in example ({', '.join(sorted(set(translation_dict) - lang_set))}) are not in valid set ({', '.join(lang_set)})."
+ )
+
+ # Convert dictionary into tuples, splitting out cases where there are
+ # multiple translations for a single language.
+ translation_tuples = []
+ for lang, text in translation_dict.items():
+ if isinstance(text, str):
+ translation_tuples.append((lang, text))
+ else:
+ translation_tuples.extend([(lang, el) for el in text])
+
+ # Ensure translations are in ascending order by language code.
+ languages, translations = zip(*sorted(translation_tuples))
+
+ return {"language": languages, "translation": translations}
+
+ def flatten(self) -> Union["FeatureType", dict[str, "FeatureType"]]:
+ """Flatten the TranslationVariableLanguages feature into a dictionary."""
+ from .features import List, Value
+
+ return {
+ "language": List(Value("string")),
+ "translation": List(Value("string")),
+ }
diff --git a/venv/lib/python3.10/site-packages/datasets/features/video.py b/venv/lib/python3.10/site-packages/datasets/features/video.py
new file mode 100644
index 0000000000000000000000000000000000000000..8d7f3e3be51275d9a205afd5de3cecd539bbdebe
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/features/video.py
@@ -0,0 +1,340 @@
+import os
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, ClassVar, Literal, Optional, TypedDict, Union
+
+import numpy as np
+import pyarrow as pa
+
+from .. import config
+from ..download.download_config import DownloadConfig
+from ..table import array_cast
+from ..utils.file_utils import is_local_path, xopen
+from ..utils.py_utils import string_to_dict
+
+
+if TYPE_CHECKING:
+ import torch
+ from torchcodec.decoders import VideoDecoder
+
+ from .features import FeatureType
+
+
+class Example(TypedDict):
+ path: Optional[str]
+ bytes: Optional[bytes]
+
+
+@dataclass
+class Video:
+ """
+ Video [`Feature`] to read video data from a video file.
+
+ Input: The Video feature accepts as input:
+ - A `str`: Absolute path to the video file (i.e. random access is allowed).
+ - A `pathlib.Path`: path to the video file (i.e. random access is allowed).
+ - A `dict` with the keys:
+
+ - `path`: String with relative path of the video file in a dataset repository.
+ - `bytes`: Bytes of the video file.
+
+ This is useful for parquet or webdataset files which embed video files.
+
+ - A `torchcodec.decoders.VideoDecoder`: torchcodec video decoder object.
+
+ Output: The Video features output data as `torchcodec.decoders.VideoDecoder` objects.
+
+ Args:
+ decode (`bool`, defaults to `True`):
+ Whether to decode the video data. If `False`,
+ returns the underlying dictionary in the format `{"path": video_path, "bytes": video_bytes}`.
+ stream_index (`int`, *optional*):
+ The streaming index to use from the file. If `None` defaults to the "best" index.
+ dimension_order (`str`, defaults to `NCHW`):
+ The dimension order of the decoded frames.
+ where N is the batch size, C is the number of channels,
+ H is the height, and W is the width of the frames.
+ num_ffmpeg_threads (`int`, defaults to `1`):
+ The number of threads to use for decoding the video. (Recommended to keep this at 1)
+ device (`str` or `torch.device`, defaults to `cpu`):
+ The device to use for decoding the video.
+ seek_mode (`str`, defaults to `exact`):
+ Determines if frame access will be “exact” or “approximate”.
+ Exact guarantees that requesting frame i will always return frame i, but doing so requires an initial scan of the file.
+ Approximate is faster as it avoids scanning the file, but less accurate as it uses the file's metadata to calculate where i probably is.
+ read more [here](https://docs.pytorch.org/torchcodec/stable/generated_examples/approximate_mode.html#sphx-glr-generated-examples-approximate-mode-py)
+
+ Examples:
+
+ ```py
+ >>> from datasets import Dataset, Video
+ >>> ds = Dataset.from_dict({"video":["path/to/Screen Recording.mov"]}).cast_column("video", Video())
+ >>> ds.features["video"]
+ Video(decode=True, id=None)
+ >>> ds[0]["video"]
+
+ >>> video = ds[0]["video"]
+ >>> video.get_frames_in_range(0, 10)
+ FrameBatch:
+ data (shape): torch.Size([10, 3, 50, 66])
+ pts_seconds: tensor([0.4333, 0.4333, 0.4333, 0.4333, 0.4333, 0.4333, 0.4333, 0.4333, 0.4333,
+ 0.4333], dtype=torch.float64)
+ duration_seconds: tensor([0.0167, 0.0167, 0.0167, 0.0167, 0.0167, 0.0167, 0.0167, 0.0167, 0.0167,
+ 0.0167], dtype=torch.float64)
+ >>> ds.cast_column('video', Video(decode=False))[0]["video]
+ {'bytes': None,
+ 'path': 'path/to/Screen Recording.mov'}
+ ```
+ """
+
+ decode: bool = True
+ stream_index: Optional[int] = None
+ dimension_order: Literal["NCHW", "NHWC"] = "NCHW"
+ num_ffmpeg_threads: int = 1
+ device: Optional[Union[str, "torch.device"]] = "cpu"
+ seek_mode: Literal["exact", "approximate"] = "exact"
+ id: Optional[str] = field(default=None, repr=False)
+ # Automatically constructed
+ dtype: ClassVar[str] = "torchcodec.decoders.VideoDecoder"
+ pa_type: ClassVar[Any] = pa.struct({"bytes": pa.binary(), "path": pa.string()})
+ _type: str = field(default="Video", init=False, repr=False)
+
+ def __call__(self):
+ return self.pa_type
+
+ def encode_example(self, value: Union[str, bytes, bytearray, Example, np.ndarray, "VideoDecoder"]) -> Example:
+ """Encode example into a format for Arrow.
+
+ Args:
+ value (`str`, `np.ndarray`, `bytes`, `bytearray`, `VideoDecoder` or `dict`):
+ Data passed as input to Video feature.
+
+ Returns:
+ `dict` with "path" and "bytes" fields
+ """
+ if value is None:
+ raise ValueError("value must be provided")
+
+ if config.TORCHCODEC_AVAILABLE:
+ from torchcodec.decoders import VideoDecoder
+ else:
+ VideoDecoder = None
+
+ if isinstance(value, list):
+ value = np.array(value)
+
+ if isinstance(value, str):
+ return {"path": value, "bytes": None}
+ elif isinstance(value, Path):
+ return {"path": str(value.absolute()), "bytes": None}
+ elif isinstance(value, (bytes, bytearray)):
+ return {"path": None, "bytes": value}
+ elif isinstance(value, np.ndarray):
+ # convert the video array to bytes
+ return encode_np_array(value)
+ elif VideoDecoder is not None and isinstance(value, VideoDecoder):
+ # convert the torchcodec video decoder to bytes
+ return encode_torchcodec_video(value)
+ elif isinstance(value, dict):
+ path, bytes_ = value.get("path"), value.get("bytes")
+ if path is not None and os.path.isfile(path):
+ # we set "bytes": None to not duplicate the data if they're already available locally
+ return {"bytes": None, "path": path}
+ elif bytes_ is not None or path is not None:
+ # store the video bytes, and path is used to infer the video format using the file extension
+ return {"bytes": bytes_, "path": path}
+ else:
+ raise ValueError(
+ f"A video sample should have one of 'path' or 'bytes' but they are missing or None in {value}."
+ )
+ else:
+ raise TypeError(f"Unsupported encode_example type: {type(value)}")
+
+ def decode_example(
+ self,
+ value: Union[str, Example],
+ token_per_repo_id: Optional[dict[str, Union[bool, str]]] = None,
+ ) -> "VideoDecoder":
+ """Decode example video file into video data.
+
+ Args:
+ value (`str` or `dict`):
+ A string with the absolute video file path, a dictionary with
+ keys:
+
+ - `path`: String with absolute or relative video file path.
+ - `bytes`: The bytes of the video file.
+ token_per_repo_id (`dict`, *optional*):
+ To access and decode
+ video files from private repositories on the Hub, you can pass
+ a dictionary repo_id (`str`) -> token (`bool` or `str`).
+
+ Returns:
+ `torchcodec.decoders.VideoDecoder`
+ """
+ if not self.decode:
+ raise RuntimeError("Decoding is disabled for this feature. Please use Video(decode=True) instead.")
+
+ if config.TORCHCODEC_AVAILABLE:
+ from torchcodec.decoders import VideoDecoder
+
+ else:
+ raise ImportError("To support decoding videos, please install 'torchcodec'.")
+
+ if token_per_repo_id is None:
+ token_per_repo_id = {}
+
+ if isinstance(value, str):
+ path, bytes_ = value, None
+ else:
+ path, bytes_ = value["path"], value["bytes"]
+
+ if bytes_ is None:
+ if path is None:
+ raise ValueError(f"A video should have one of 'path' or 'bytes' but both are None in {value}.")
+ elif is_local_path(path):
+ video = VideoDecoder(
+ path,
+ stream_index=self.stream_index,
+ dimension_order=self.dimension_order,
+ num_ffmpeg_threads=self.num_ffmpeg_threads,
+ device=self.device,
+ seek_mode=self.seek_mode,
+ )
+ else:
+ video = hf_video_reader(
+ path,
+ token_per_repo_id=token_per_repo_id,
+ dimension_order=self.dimension_order,
+ num_ffmpeg_threads=self.num_ffmpeg_threads,
+ device=self.device,
+ seek_mode=self.seek_mode,
+ )
+ else:
+ video = VideoDecoder(
+ bytes_,
+ stream_index=self.stream_index,
+ dimension_order=self.dimension_order,
+ num_ffmpeg_threads=self.num_ffmpeg_threads,
+ device=self.device,
+ seek_mode=self.seek_mode,
+ )
+ video._hf_encoded = {"path": path, "bytes": bytes_}
+ video.metadata.path = path
+ return video
+
+ def flatten(self) -> Union["FeatureType", dict[str, "FeatureType"]]:
+ """If in the decodable state, return the feature itself, otherwise flatten the feature into a dictionary."""
+ from .features import Value
+
+ return (
+ self
+ if self.decode
+ else {
+ "bytes": Value("binary"),
+ "path": Value("string"),
+ }
+ )
+
+ def cast_storage(self, storage: Union[pa.StringArray, pa.StructArray, pa.ListArray]) -> pa.StructArray:
+ """Cast an Arrow array to the Video arrow storage type.
+ The Arrow types that can be converted to the Video pyarrow storage type are:
+
+ - `pa.string()` - it must contain the "path" data
+ - `pa.binary()` - it must contain the video bytes
+ - `pa.struct({"bytes": pa.binary()})`
+ - `pa.struct({"path": pa.string()})`
+ - `pa.struct({"bytes": pa.binary(), "path": pa.string()})` - order doesn't matter
+ - `pa.list(*)` - it must contain the video array data
+
+ Args:
+ storage (`Union[pa.StringArray, pa.StructArray, pa.ListArray]`):
+ PyArrow array to cast.
+
+ Returns:
+ `pa.StructArray`: Array in the Video arrow storage type, that is
+ `pa.struct({"bytes": pa.binary(), "path": pa.string()})`.
+ """
+ if pa.types.is_string(storage.type):
+ bytes_array = pa.array([None] * len(storage), type=pa.binary())
+ storage = pa.StructArray.from_arrays([bytes_array, storage], ["bytes", "path"], mask=storage.is_null())
+ elif pa.types.is_binary(storage.type):
+ path_array = pa.array([None] * len(storage), type=pa.string())
+ storage = pa.StructArray.from_arrays([storage, path_array], ["bytes", "path"], mask=storage.is_null())
+ elif pa.types.is_struct(storage.type):
+ if storage.type.get_field_index("bytes") >= 0:
+ bytes_array = storage.field("bytes")
+ else:
+ bytes_array = pa.array([None] * len(storage), type=pa.binary())
+ if storage.type.get_field_index("path") >= 0:
+ path_array = storage.field("path")
+ else:
+ path_array = pa.array([None] * len(storage), type=pa.string())
+ storage = pa.StructArray.from_arrays([bytes_array, path_array], ["bytes", "path"], mask=storage.is_null())
+ elif pa.types.is_list(storage.type):
+ bytes_array = pa.array(
+ [encode_np_array(np.array(arr))["bytes"] if arr is not None else None for arr in storage.to_pylist()],
+ type=pa.binary(),
+ )
+ path_array = pa.array([None] * len(storage), type=pa.string())
+ storage = pa.StructArray.from_arrays(
+ [bytes_array, path_array], ["bytes", "path"], mask=bytes_array.is_null()
+ )
+ return array_cast(storage, self.pa_type)
+
+
+def video_to_bytes(video: "VideoDecoder") -> bytes:
+ """Convert a torchcodec Video object to bytes using native compression if possible"""
+ raise NotImplementedError()
+
+
+def encode_torchcodec_video(video: "VideoDecoder") -> Example:
+ if hasattr(video, "_hf_encoded"):
+ return video._hf_encoded
+ else:
+ raise NotImplementedError(
+ "Encoding a VideoDecoder that doesn't come from datasets.Video.decode() is not implemented"
+ )
+
+
+def encode_np_array(array: np.ndarray) -> Example:
+ raise NotImplementedError()
+
+
+# No monkey patch needed!
+# 1. store the encoded video data {"path": ..., "bytes": ...} in `video._hf_encoded``
+# 2. add support for hf:// files
+
+
+def hf_video_reader(
+ path: str,
+ token_per_repo_id: Optional[dict[str, Union[bool, str]]] = None,
+ stream: str = "video",
+ dimension_order: Literal["NCHW", "NHWC"] = "NCHW",
+ num_ffmpeg_threads: int = 1,
+ device: Optional[Union[str, "torch.device"]] = "cpu",
+ seek_mode: Literal["exact", "approximate"] = "exact",
+) -> "VideoDecoder":
+ from torchcodec.decoders import VideoDecoder
+
+ # Load the file from HF
+ if token_per_repo_id is None:
+ token_per_repo_id = {}
+ source_url = path.split("::")[-1]
+ pattern = config.HUB_DATASETS_URL if source_url.startswith(config.HF_ENDPOINT) else config.HUB_DATASETS_HFFS_URL
+ source_url_fields = string_to_dict(source_url, pattern)
+ token = token_per_repo_id.get(source_url_fields["repo_id"]) if source_url_fields is not None else None
+ download_config = DownloadConfig(token=token)
+ f = xopen(path, "rb", download_config=download_config)
+
+ # Instantiate the VideoDecoder
+ stream_id = 0 if len(stream.split(":")) == 1 else int(stream.split(":")[1])
+ vd = VideoDecoder(
+ f,
+ stream_index=stream_id,
+ dimension_order=dimension_order,
+ num_ffmpeg_threads=num_ffmpeg_threads,
+ device=device,
+ seek_mode=seek_mode,
+ )
+ return vd
diff --git a/venv/lib/python3.10/site-packages/datasets/filesystems/__init__.py b/venv/lib/python3.10/site-packages/datasets/filesystems/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..d19d58bca1fd8f48e2ef9e5fcdfe17f7738692ac
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/filesystems/__init__.py
@@ -0,0 +1,47 @@
+import importlib
+import shutil
+import warnings
+from typing import List
+
+import fsspec
+import fsspec.asyn
+from fsspec.implementations.local import LocalFileSystem
+
+from . import compression
+
+
+COMPRESSION_FILESYSTEMS: list[compression.BaseCompressedFileFileSystem] = [
+ compression.Bz2FileSystem,
+ compression.GzipFileSystem,
+ compression.Lz4FileSystem,
+ compression.XzFileSystem,
+ compression.ZstdFileSystem,
+]
+
+# Register custom filesystems
+for fs_class in COMPRESSION_FILESYSTEMS:
+ if fs_class.protocol in fsspec.registry and fsspec.registry[fs_class.protocol] is not fs_class:
+ warnings.warn(f"A filesystem protocol was already set for {fs_class.protocol} and will be overwritten.")
+ fsspec.register_implementation(fs_class.protocol, fs_class, clobber=True)
+
+
+def is_remote_filesystem(fs: fsspec.AbstractFileSystem) -> bool:
+ """
+ Checks if `fs` is a remote filesystem.
+
+ Args:
+ fs (`fsspec.spec.AbstractFileSystem`):
+ An abstract super-class for pythonic file-systems, e.g. `fsspec.filesystem(\'file\')` or `s3fs.S3FileSystem`.
+ """
+ return not isinstance(fs, LocalFileSystem)
+
+
+def rename(fs: fsspec.AbstractFileSystem, src: str, dst: str):
+ """
+ Renames the file `src` in `fs` to `dst`.
+ """
+ if not is_remote_filesystem(fs):
+ # LocalFileSystem.mv does copy + rm, it is more efficient to simply move a local directory
+ shutil.move(fs._strip_protocol(src), fs._strip_protocol(dst))
+ else:
+ fs.mv(src, dst, recursive=True)
diff --git a/venv/lib/python3.10/site-packages/datasets/filesystems/__pycache__/__init__.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/filesystems/__pycache__/__init__.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..72d6f4ee6fae4f210267ce0a0ea5ebec50e46331
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/filesystems/__pycache__/__init__.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/filesystems/__pycache__/compression.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/filesystems/__pycache__/compression.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..75bd4b83d23cef66fac8fd36872a1ae0bd13ac20
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/filesystems/__pycache__/compression.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/filesystems/compression.py b/venv/lib/python3.10/site-packages/datasets/filesystems/compression.py
new file mode 100644
index 0000000000000000000000000000000000000000..237b06d75008f8354a4b43d19f6a023e02991393
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/filesystems/compression.py
@@ -0,0 +1,127 @@
+import os
+from functools import partial
+from typing import Optional
+
+import fsspec
+from fsspec.archive import AbstractArchiveFileSystem
+
+
+class BaseCompressedFileFileSystem(AbstractArchiveFileSystem):
+ """Read contents of compressed file as a filesystem with one file inside."""
+
+ root_marker = ""
+ protocol: str = (
+ None # protocol passed in prefix to the url. ex: "gzip", for gzip://file.txt::http://foo.bar/file.txt.gz
+ )
+ compression: str = None # compression type in fsspec. ex: "gzip"
+ extensions: list[str] = None # extensions of the filename to strip. ex: ".gz" to get file.txt from file.txt.gz
+
+ def __init__(
+ self, fo: str = "", target_protocol: Optional[str] = None, target_options: Optional[dict] = None, **kwargs
+ ):
+ """
+ The compressed file system can be instantiated from any compressed file.
+ It reads the contents of compressed file as a filesystem with one file inside, as if it was an archive.
+
+ The single file inside the filesystem is named after the compresssed file,
+ without the compression extension at the end of the filename.
+
+ Args:
+ fo (:obj:``str``): Path to compressed file. Will fetch file using ``fsspec.open()``
+ mode (:obj:``str``): Currently, only 'rb' accepted
+ target_protocol(:obj:``str``, optional): To override the FS protocol inferred from a URL.
+ target_options (:obj:``dict``, optional): Kwargs passed when instantiating the target FS.
+ """
+ super().__init__(self, **kwargs)
+ self.fo = fo.__fspath__() if hasattr(fo, "__fspath__") else fo
+ # always open as "rb" since fsspec can then use the TextIOWrapper to make it work for "r" mode
+ self._open_with_fsspec = partial(
+ fsspec.open,
+ self.fo,
+ mode="rb",
+ protocol=target_protocol,
+ compression=self.compression,
+ client_kwargs={
+ "requote_redirect_url": False, # see https://github.com/huggingface/datasets/pull/5459
+ "trust_env": True, # Enable reading proxy env variables.
+ **(target_options or {}).pop("client_kwargs", {}), # To avoid issues if it was already passed.
+ },
+ **(target_options or {}),
+ )
+ self.compressed_name = os.path.basename(self.fo.split("::")[0])
+ self.uncompressed_name = (
+ self.compressed_name[: self.compressed_name.rindex(".")]
+ if "." in self.compressed_name
+ else self.compressed_name
+ )
+ self.dir_cache = None
+
+ @classmethod
+ def _strip_protocol(cls, path):
+ # compressed file paths are always relative to the archive root
+ return super()._strip_protocol(path).lstrip("/")
+
+ def _get_dirs(self):
+ if self.dir_cache is None:
+ f = {**self._open_with_fsspec().fs.info(self.fo), "name": self.uncompressed_name}
+ self.dir_cache = {f["name"]: f}
+
+ def cat(self, path: str):
+ with self._open_with_fsspec().open() as f:
+ return f.read()
+
+ def _open(
+ self,
+ path: str,
+ mode: str = "rb",
+ block_size=None,
+ autocommit=True,
+ cache_options=None,
+ **kwargs,
+ ):
+ path = self._strip_protocol(path)
+ if mode != "rb":
+ raise ValueError(f"Tried to read with mode {mode} on file {self.fo} opened with mode 'rb'")
+ return self._open_with_fsspec().open()
+
+
+class Bz2FileSystem(BaseCompressedFileFileSystem):
+ """Read contents of BZ2 file as a filesystem with one file inside."""
+
+ protocol = "bz2"
+ compression = "bz2"
+ extensions = [".bz2"]
+
+
+class GzipFileSystem(BaseCompressedFileFileSystem):
+ """Read contents of GZIP file as a filesystem with one file inside."""
+
+ protocol = "gzip"
+ compression = "gzip"
+ extensions = [".gz", ".gzip"]
+
+
+class Lz4FileSystem(BaseCompressedFileFileSystem):
+ """Read contents of LZ4 file as a filesystem with one file inside."""
+
+ protocol = "lz4"
+ compression = "lz4"
+ extensions = [".lz4"]
+
+
+class XzFileSystem(BaseCompressedFileFileSystem):
+ """Read contents of .xz (LZMA) file as a filesystem with one file inside."""
+
+ protocol = "xz"
+ compression = "xz"
+ extensions = [".xz"]
+
+
+class ZstdFileSystem(BaseCompressedFileFileSystem):
+ """
+ Read contents of .zstd file as a filesystem with one file inside.
+ """
+
+ protocol = "zstd"
+ compression = "zstd"
+ extensions = [".zst", ".zstd"]
diff --git a/venv/lib/python3.10/site-packages/datasets/fingerprint.py b/venv/lib/python3.10/site-packages/datasets/fingerprint.py
new file mode 100644
index 0000000000000000000000000000000000000000..91e8bd01eebba7773ce0b335bfc928c0796b943e
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/fingerprint.py
@@ -0,0 +1,454 @@
+import inspect
+import os
+import random
+import shutil
+import tempfile
+import weakref
+from functools import wraps
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Callable, Optional, Union
+
+import numpy as np
+import xxhash
+
+from . import config
+from .naming import INVALID_WINDOWS_CHARACTERS_IN_PATH
+from .utils._dill import dumps
+from .utils.logging import get_logger
+
+
+if TYPE_CHECKING:
+ from .arrow_dataset import Dataset
+
+
+logger = get_logger(__name__)
+
+
+# Fingerprinting allows to have one deterministic fingerprint per dataset state.
+# A dataset fingerprint is updated after each transform.
+# Re-running the same transforms on a dataset in a different session results in the same fingerprint.
+# This is possible thanks to a custom hashing function that works with most python objects.
+
+# Fingerprinting is the main mechanism that enables caching.
+# The caching mechanism allows to reload an existing cache file if it's already been computed.
+
+
+#################
+# Caching
+#################
+
+_CACHING_ENABLED = True
+_TEMP_DIR_FOR_TEMP_CACHE_FILES: Optional["_TempCacheDir"] = None
+_DATASETS_WITH_TABLE_IN_TEMP_DIR: Optional[weakref.WeakSet] = None
+
+
+class _TempCacheDir:
+ """
+ A temporary directory for storing cached Arrow files with a cleanup that frees references to the Arrow files
+ before deleting the directory itself to avoid permission errors on Windows.
+ """
+
+ def __init__(self):
+ self.name = tempfile.mkdtemp(prefix=config.TEMP_CACHE_DIR_PREFIX)
+ self._finalizer = weakref.finalize(self, self._cleanup)
+
+ def _cleanup(self):
+ for dset in get_datasets_with_cache_file_in_temp_dir():
+ dset.__del__()
+ if os.path.exists(self.name):
+ try:
+ shutil.rmtree(self.name)
+ except Exception as e:
+ raise OSError(
+ f"An error occurred while trying to delete temporary cache directory {self.name}. Please delete it manually."
+ ) from e
+
+ def cleanup(self):
+ if self._finalizer.detach():
+ self._cleanup()
+
+
+def maybe_register_dataset_for_temp_dir_deletion(dataset):
+ """
+ This function registers the datasets that have cache files in _TEMP_DIR_FOR_TEMP_CACHE_FILES in order
+ to properly delete them before deleting the temporary directory.
+ The temporary directory _TEMP_DIR_FOR_TEMP_CACHE_FILES is used when caching is disabled.
+ """
+ if _TEMP_DIR_FOR_TEMP_CACHE_FILES is None:
+ return
+
+ global _DATASETS_WITH_TABLE_IN_TEMP_DIR
+ if _DATASETS_WITH_TABLE_IN_TEMP_DIR is None:
+ _DATASETS_WITH_TABLE_IN_TEMP_DIR = weakref.WeakSet()
+ if any(
+ Path(_TEMP_DIR_FOR_TEMP_CACHE_FILES.name) in Path(cache_file["filename"]).parents
+ for cache_file in dataset.cache_files
+ ):
+ _DATASETS_WITH_TABLE_IN_TEMP_DIR.add(dataset)
+
+
+def get_datasets_with_cache_file_in_temp_dir():
+ return list(_DATASETS_WITH_TABLE_IN_TEMP_DIR) if _DATASETS_WITH_TABLE_IN_TEMP_DIR is not None else []
+
+
+def enable_caching():
+ """
+ When applying transforms on a dataset, the data are stored in cache files.
+ The caching mechanism allows to reload an existing cache file if it's already been computed.
+
+ Reloading a dataset is possible since the cache files are named using the dataset fingerprint, which is updated
+ after each transform.
+
+ If disabled, the library will no longer reload cached datasets files when applying transforms to the datasets.
+ More precisely, if the caching is disabled:
+ - cache files are always recreated
+ - cache files are written to a temporary directory that is deleted when session closes
+ - cache files are named using a random hash instead of the dataset fingerprint
+ - use [`~datasets.Dataset.save_to_disk`] to save a transformed dataset or it will be deleted when session closes
+ - caching doesn't affect [`~datasets.load_dataset`]. If you want to regenerate a dataset from scratch you should use
+ the `download_mode` parameter in [`~datasets.load_dataset`].
+ """
+ global _CACHING_ENABLED
+ _CACHING_ENABLED = True
+
+
+def disable_caching():
+ """
+ When applying transforms on a dataset, the data are stored in cache files.
+ The caching mechanism allows to reload an existing cache file if it's already been computed.
+
+ Reloading a dataset is possible since the cache files are named using the dataset fingerprint, which is updated
+ after each transform.
+
+ If disabled, the library will no longer reload cached datasets files when applying transforms to the datasets.
+ More precisely, if the caching is disabled:
+ - cache files are always recreated
+ - cache files are written to a temporary directory that is deleted when session closes
+ - cache files are named using a random hash instead of the dataset fingerprint
+ - use [`~datasets.Dataset.save_to_disk`] to save a transformed dataset or it will be deleted when session closes
+ - caching doesn't affect [`~datasets.load_dataset`]. If you want to regenerate a dataset from scratch you should use
+ the `download_mode` parameter in [`~datasets.load_dataset`].
+ """
+ global _CACHING_ENABLED
+ _CACHING_ENABLED = False
+
+
+def is_caching_enabled() -> bool:
+ """
+ When applying transforms on a dataset, the data are stored in cache files.
+ The caching mechanism allows to reload an existing cache file if it's already been computed.
+
+ Reloading a dataset is possible since the cache files are named using the dataset fingerprint, which is updated
+ after each transform.
+
+ If disabled, the library will no longer reload cached datasets files when applying transforms to the datasets.
+ More precisely, if the caching is disabled:
+ - cache files are always recreated
+ - cache files are written to a temporary directory that is deleted when session closes
+ - cache files are named using a random hash instead of the dataset fingerprint
+ - use [`~datasets.Dataset.save_to_disk`]] to save a transformed dataset or it will be deleted when session closes
+ - caching doesn't affect [`~datasets.load_dataset`]. If you want to regenerate a dataset from scratch you should use
+ the `download_mode` parameter in [`~datasets.load_dataset`].
+ """
+ global _CACHING_ENABLED
+ return bool(_CACHING_ENABLED)
+
+
+def get_temporary_cache_files_directory() -> str:
+ """Return a directory that is deleted when session closes."""
+ global _TEMP_DIR_FOR_TEMP_CACHE_FILES
+ if _TEMP_DIR_FOR_TEMP_CACHE_FILES is None:
+ _TEMP_DIR_FOR_TEMP_CACHE_FILES = _TempCacheDir()
+ return _TEMP_DIR_FOR_TEMP_CACHE_FILES.name
+
+
+#################
+# Hashing
+#################
+
+
+class Hasher:
+ """Hasher that accepts python objects as inputs."""
+
+ dispatch: dict = {}
+
+ def __init__(self):
+ self.m = xxhash.xxh64()
+
+ @classmethod
+ def hash_bytes(cls, value: Union[bytes, list[bytes]]) -> str:
+ value = [value] if isinstance(value, bytes) else value
+ m = xxhash.xxh64()
+ for x in value:
+ m.update(x)
+ return m.hexdigest()
+
+ @classmethod
+ def hash(cls, value: Any) -> str:
+ return cls.hash_bytes(dumps(value))
+
+ def update(self, value: Any) -> None:
+ header_for_update = f"=={type(value)}=="
+ value_for_update = self.hash(value)
+ self.m.update(header_for_update.encode("utf8"))
+ self.m.update(value_for_update.encode("utf-8"))
+
+ def hexdigest(self) -> str:
+ return self.m.hexdigest()
+
+
+#################
+# Fingerprinting
+#################
+
+fingerprint_rng = random.Random()
+# we show a warning only once when fingerprinting fails to avoid spam
+fingerprint_warnings: dict[str, bool] = {}
+
+
+def generate_fingerprint(dataset: "Dataset") -> str:
+ state = dataset.__dict__
+ hasher = Hasher()
+ for key in sorted(state):
+ if key == "_fingerprint":
+ continue
+ hasher.update(key)
+ hasher.update(state[key])
+ # hash data files last modification timestamps as well
+ for cache_file in dataset.cache_files:
+ hasher.update(os.path.getmtime(cache_file["filename"]))
+ return hasher.hexdigest()
+
+
+def generate_random_fingerprint(nbits: int = 64) -> str:
+ return f"{fingerprint_rng.getrandbits(nbits):0{nbits // 4}x}"
+
+
+def update_fingerprint(fingerprint, transform, transform_args):
+ global fingerprint_warnings
+ hasher = Hasher()
+ hasher.update(fingerprint)
+ try:
+ hasher.update(transform)
+ except: # noqa various errors might raise here from pickle or dill
+ if _CACHING_ENABLED:
+ if not fingerprint_warnings.get("update_fingerprint_transform_hash_failed", False):
+ logger.warning(
+ f"Transform {transform} couldn't be hashed properly, a random hash was used instead. "
+ "Make sure your transforms and parameters are serializable with pickle or dill for the dataset fingerprinting and caching to work. "
+ "If you reuse this transform, the caching mechanism will consider it to be different from the previous calls and recompute everything. "
+ "This warning is only shown once. Subsequent hashing failures won't be shown."
+ )
+ fingerprint_warnings["update_fingerprint_transform_hash_failed"] = True
+ else:
+ logger.info(f"Transform {transform} couldn't be hashed properly, a random hash was used instead.")
+ else:
+ logger.info(
+ f"Transform {transform} couldn't be hashed properly, a random hash was used instead. This doesn't affect caching since it's disabled."
+ )
+
+ return generate_random_fingerprint()
+ for key in sorted(transform_args):
+ hasher.update(key)
+ try:
+ hasher.update(transform_args[key])
+ except: # noqa various errors might raise here from pickle or dill
+ if _CACHING_ENABLED:
+ if not fingerprint_warnings.get("update_fingerprint_transform_hash_failed", False):
+ logger.warning(
+ f"Parameter '{key}'={transform_args[key]} of the transform {transform} couldn't be hashed properly, a random hash was used instead. "
+ "Make sure your transforms and parameters are serializable with pickle or dill for the dataset fingerprinting and caching to work. "
+ "If you reuse this transform, the caching mechanism will consider it to be different from the previous calls and recompute everything. "
+ "This warning is only shown once. Subsequent hashing failures won't be shown."
+ )
+ fingerprint_warnings["update_fingerprint_transform_hash_failed"] = True
+ else:
+ logger.info(
+ f"Parameter '{key}'={transform_args[key]} of the transform {transform} couldn't be hashed properly, a random hash was used instead."
+ )
+ else:
+ logger.info(
+ f"Parameter '{key}'={transform_args[key]} of the transform {transform} couldn't be hashed properly, a random hash was used instead. This doesn't affect caching since it's disabled."
+ )
+ return generate_random_fingerprint()
+ return hasher.hexdigest()
+
+
+def validate_fingerprint(fingerprint: str, max_length=64):
+ """
+ Make sure the fingerprint is a non-empty string that is not longer that max_length=64 by default,
+ so that the fingerprint can be used to name cache files without issues.
+ """
+ if not isinstance(fingerprint, str) or not fingerprint:
+ raise ValueError(f"Invalid fingerprint '{fingerprint}': it should be a non-empty string.")
+ for invalid_char in INVALID_WINDOWS_CHARACTERS_IN_PATH:
+ if invalid_char in fingerprint:
+ raise ValueError(
+ f"Invalid fingerprint. Bad characters from black list '{INVALID_WINDOWS_CHARACTERS_IN_PATH}' found in '{fingerprint}'. "
+ f"They could create issues when creating cache files."
+ )
+ if len(fingerprint) > max_length:
+ raise ValueError(
+ f"Invalid fingerprint. Maximum lenth is {max_length} but '{fingerprint}' has length {len(fingerprint)}."
+ "It could create issues when creating cache files."
+ )
+
+
+def format_transform_for_fingerprint(func: Callable, version: Optional[str] = None) -> str:
+ """
+ Format a transform to the format that will be used to update the fingerprint.
+ """
+ transform = f"{func.__module__}.{func.__qualname__}"
+ if version is not None:
+ transform += f"@{version}"
+ return transform
+
+
+def format_kwargs_for_fingerprint(
+ func: Callable,
+ args: tuple,
+ kwargs: dict[str, Any],
+ use_kwargs: Optional[list[str]] = None,
+ ignore_kwargs: Optional[list[str]] = None,
+ randomized_function: bool = False,
+) -> dict[str, Any]:
+ """
+ Format the kwargs of a transform to the format that will be used to update the fingerprint.
+ """
+ kwargs_for_fingerprint = kwargs.copy()
+ if args:
+ params = [p.name for p in inspect.signature(func).parameters.values() if p != p.VAR_KEYWORD]
+ args = args[1:] # assume the first argument is the dataset
+ params = params[1:]
+ kwargs_for_fingerprint.update(zip(params, args))
+ else:
+ del kwargs_for_fingerprint[
+ next(iter(inspect.signature(func).parameters))
+ ] # assume the first key is the dataset
+
+ # keep the right kwargs to be hashed to generate the fingerprint
+
+ if use_kwargs:
+ kwargs_for_fingerprint = {k: v for k, v in kwargs_for_fingerprint.items() if k in use_kwargs}
+ if ignore_kwargs:
+ kwargs_for_fingerprint = {k: v for k, v in kwargs_for_fingerprint.items() if k not in ignore_kwargs}
+ if randomized_function: # randomized functions have `seed` and `generator` parameters
+ if kwargs_for_fingerprint.get("seed") is None and kwargs_for_fingerprint.get("generator") is None:
+ _, seed, pos, *_ = np.random.get_state()
+ seed = seed[pos] if pos < 624 else seed[0]
+ kwargs_for_fingerprint["generator"] = np.random.default_rng(seed)
+
+ # remove kwargs that are the default values
+
+ default_values = {
+ p.name: p.default for p in inspect.signature(func).parameters.values() if p.default != inspect._empty
+ }
+ for default_varname, default_value in default_values.items():
+ if default_varname in kwargs_for_fingerprint and kwargs_for_fingerprint[default_varname] == default_value:
+ kwargs_for_fingerprint.pop(default_varname)
+ return kwargs_for_fingerprint
+
+
+def fingerprint_transform(
+ inplace: bool,
+ use_kwargs: Optional[list[str]] = None,
+ ignore_kwargs: Optional[list[str]] = None,
+ fingerprint_names: Optional[list[str]] = None,
+ randomized_function: bool = False,
+ version: Optional[str] = None,
+):
+ """
+ Wrapper for dataset transforms to update the dataset fingerprint using ``update_fingerprint``
+ Args:
+ inplace (:obj:`bool`): If inplace is True, the fingerprint of the dataset is updated inplace.
+ Otherwise, a parameter "new_fingerprint" is passed to the wrapped method that should take care of
+ setting the fingerprint of the returned Dataset.
+ use_kwargs (:obj:`List[str]`, optional): optional white list of argument names to take into account
+ to update the fingerprint to the wrapped method that should take care of
+ setting the fingerprint of the returned Dataset. By default all the arguments are used.
+ ignore_kwargs (:obj:`List[str]`, optional): optional black list of argument names to take into account
+ to update the fingerprint. Note that ignore_kwargs prevails on use_kwargs.
+ fingerprint_names (:obj:`List[str]`, optional, defaults to ["new_fingerprint"]):
+ If the dataset transforms is not inplace and returns a DatasetDict, then it can require
+ several fingerprints (one per dataset in the DatasetDict). By specifying fingerprint_names,
+ one fingerprint named after each element of fingerprint_names is going to be passed.
+ randomized_function (:obj:`bool`, defaults to False): If the dataset transform is random and has
+ optional parameters "seed" and "generator", then you can set randomized_function to True.
+ This way, even if users set "seed" and "generator" to None, then the fingerprint is
+ going to be randomly generated depending on numpy's current state. In this case, the
+ generator is set to np.random.default_rng(np.random.get_state()[1][0]).
+ version (:obj:`str`, optional): version of the transform. The version is taken into account when
+ computing the fingerprint. If a datase transform changes (or at least if the output data
+ that are cached changes), then one should increase the version. If the version stays the
+ same, then old cached data could be reused that are not compatible with the new transform.
+ It should be in the format "MAJOR.MINOR.PATCH".
+ """
+
+ if use_kwargs is not None and not isinstance(use_kwargs, list):
+ raise ValueError(f"use_kwargs is supposed to be a list, not {type(use_kwargs)}")
+
+ if ignore_kwargs is not None and not isinstance(ignore_kwargs, list):
+ raise ValueError(f"ignore_kwargs is supposed to be a list, not {type(use_kwargs)}")
+
+ if inplace and fingerprint_names:
+ raise ValueError("fingerprint_names are only used when inplace is False")
+
+ fingerprint_names = fingerprint_names if fingerprint_names is not None else ["new_fingerprint"]
+
+ def _fingerprint(func):
+ if not inplace and not all(name in func.__code__.co_varnames for name in fingerprint_names):
+ raise ValueError(f"function {func} is missing parameters {fingerprint_names} in signature")
+
+ if randomized_function: # randomized function have seed and generator parameters
+ if "seed" not in func.__code__.co_varnames:
+ raise ValueError(f"'seed' must be in {func}'s signature")
+ if "generator" not in func.__code__.co_varnames:
+ raise ValueError(f"'generator' must be in {func}'s signature")
+ # this call has to be outside the wrapper or since __qualname__ changes in multiprocessing
+ transform = format_transform_for_fingerprint(func, version=version)
+
+ @wraps(func)
+ def wrapper(*args, **kwargs):
+ kwargs_for_fingerprint = format_kwargs_for_fingerprint(
+ func,
+ args,
+ kwargs,
+ use_kwargs=use_kwargs,
+ ignore_kwargs=ignore_kwargs,
+ randomized_function=randomized_function,
+ )
+
+ if args:
+ dataset: Dataset = args[0]
+ args = args[1:]
+ else:
+ dataset: Dataset = kwargs.pop(next(iter(inspect.signature(func).parameters)))
+
+ # compute new_fingerprint and add it to the args of not in-place transforms
+ if inplace:
+ new_fingerprint = update_fingerprint(dataset._fingerprint, transform, kwargs_for_fingerprint)
+ else:
+ for fingerprint_name in fingerprint_names: # transforms like `train_test_split` have several hashes
+ if kwargs.get(fingerprint_name) is None:
+ kwargs_for_fingerprint["fingerprint_name"] = fingerprint_name
+ kwargs[fingerprint_name] = update_fingerprint(
+ dataset._fingerprint, transform, kwargs_for_fingerprint
+ )
+ else:
+ validate_fingerprint(kwargs[fingerprint_name])
+
+ # Call actual function
+
+ out = func(dataset, *args, **kwargs)
+
+ # Update fingerprint of in-place transforms + update in-place history of transforms
+
+ if inplace: # update after calling func so that the fingerprint doesn't change if the function fails
+ dataset._fingerprint = new_fingerprint
+
+ return out
+
+ wrapper._decorator_name_ = "fingerprint"
+ return wrapper
+
+ return _fingerprint
diff --git a/venv/lib/python3.10/site-packages/datasets/formatting/__init__.py b/venv/lib/python3.10/site-packages/datasets/formatting/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e29421df342f172cece9e28ff619c7760b34c6a0
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/formatting/__init__.py
@@ -0,0 +1,136 @@
+# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Dict, List, Optional, Type
+
+from .. import config
+from ..utils import logging
+from .formatting import (
+ ArrowFormatter,
+ CustomFormatter,
+ Formatter,
+ PandasFormatter,
+ PythonFormatter,
+ TableFormatter,
+ TensorFormatter,
+ format_table,
+ query_table,
+)
+from .np_formatter import NumpyFormatter
+
+
+logger = logging.get_logger(__name__)
+
+_FORMAT_TYPES: dict[Optional[str], type[Formatter]] = {}
+_FORMAT_TYPES_ALIASES: dict[Optional[str], str] = {}
+_FORMAT_TYPES_ALIASES_UNAVAILABLE: dict[Optional[str], Exception] = {}
+
+
+def _register_formatter(
+ formatter_cls: type,
+ format_type: Optional[str],
+ aliases: Optional[list[str]] = None,
+):
+ """
+ Register a Formatter object using a name and optional aliases.
+ This function must be used on a Formatter class.
+ """
+ aliases = aliases if aliases is not None else []
+ if format_type in _FORMAT_TYPES:
+ logger.warning(
+ f"Overwriting format type '{format_type}' ({_FORMAT_TYPES[format_type].__name__} -> {formatter_cls.__name__})"
+ )
+ _FORMAT_TYPES[format_type] = formatter_cls
+ for alias in set(aliases + [format_type]):
+ if alias in _FORMAT_TYPES_ALIASES:
+ logger.warning(
+ f"Overwriting format type alias '{alias}' ({_FORMAT_TYPES_ALIASES[alias]} -> {format_type})"
+ )
+ _FORMAT_TYPES_ALIASES[alias] = format_type
+
+
+def _register_unavailable_formatter(
+ unavailable_error: Exception, format_type: Optional[str], aliases: Optional[list[str]] = None
+):
+ """
+ Register an unavailable Formatter object using a name and optional aliases.
+ This function must be used on an Exception object that is raised when trying to get the unavailable formatter.
+ """
+ aliases = aliases if aliases is not None else []
+ for alias in set(aliases + [format_type]):
+ _FORMAT_TYPES_ALIASES_UNAVAILABLE[alias] = unavailable_error
+
+
+# Here we define all the available formatting functions that can be used by `Dataset.set_format`
+_register_formatter(PythonFormatter, None, aliases=["python"])
+_register_formatter(ArrowFormatter, "arrow", aliases=["pa", "pyarrow"])
+_register_formatter(NumpyFormatter, "numpy", aliases=["np"])
+_register_formatter(PandasFormatter, "pandas", aliases=["pd"])
+_register_formatter(CustomFormatter, "custom")
+
+if config.POLARS_AVAILABLE:
+ from .polars_formatter import PolarsFormatter
+
+ _register_formatter(PolarsFormatter, "polars", aliases=["pl"])
+else:
+ _polars_error = ValueError("Polars needs to be installed to be able to return Polars dataframes.")
+ _register_unavailable_formatter(_polars_error, "polars", aliases=["pl"])
+
+if config.TORCH_AVAILABLE:
+ from .torch_formatter import TorchFormatter
+
+ _register_formatter(TorchFormatter, "torch", aliases=["pt", "pytorch"])
+else:
+ _torch_error = ValueError("PyTorch needs to be installed to be able to return PyTorch tensors.")
+ _register_unavailable_formatter(_torch_error, "torch", aliases=["pt", "pytorch"])
+
+if config.TF_AVAILABLE:
+ from .tf_formatter import TFFormatter
+
+ _register_formatter(TFFormatter, "tensorflow", aliases=["tf"])
+else:
+ _tf_error = ValueError("Tensorflow needs to be installed to be able to return Tensorflow tensors.")
+ _register_unavailable_formatter(_tf_error, "tensorflow", aliases=["tf"])
+
+if config.JAX_AVAILABLE:
+ from .jax_formatter import JaxFormatter
+
+ _register_formatter(JaxFormatter, "jax", aliases=[])
+else:
+ _jax_error = ValueError("JAX needs to be installed to be able to return JAX arrays.")
+ _register_unavailable_formatter(_jax_error, "jax", aliases=[])
+
+
+def get_format_type_from_alias(format_type: Optional[str]) -> Optional[str]:
+ """If the given format type is a known alias, then return its main type name. Otherwise return the type with no change."""
+ if format_type in _FORMAT_TYPES_ALIASES:
+ return _FORMAT_TYPES_ALIASES[format_type]
+ else:
+ return format_type
+
+
+def get_formatter(format_type: Optional[str], **format_kwargs) -> Formatter:
+ """
+ Factory function to get a Formatter given its type name and keyword arguments.
+ A formatter is an object that extracts and formats data from pyarrow table.
+ It defines the formatting for rows, columns and batches.
+ If the formatter for a given type name doesn't exist or is not available, an error is raised.
+ """
+ format_type = get_format_type_from_alias(format_type)
+ if format_type in _FORMAT_TYPES:
+ return _FORMAT_TYPES[format_type](**format_kwargs)
+ if format_type in _FORMAT_TYPES_ALIASES_UNAVAILABLE:
+ raise _FORMAT_TYPES_ALIASES_UNAVAILABLE[format_type]
+ else:
+ raise ValueError(f"Format type should be one of {list(_FORMAT_TYPES.keys())}, but got '{format_type}'")
diff --git a/venv/lib/python3.10/site-packages/datasets/formatting/__pycache__/__init__.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/formatting/__pycache__/__init__.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..e9371d43215ec02e4e08146a5ee66f9e34e51379
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/formatting/__pycache__/__init__.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/formatting/__pycache__/formatting.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/formatting/__pycache__/formatting.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..c32ce38026c307e1a6db98112d4962d37d0fe97b
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/formatting/__pycache__/formatting.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/formatting/__pycache__/jax_formatter.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/formatting/__pycache__/jax_formatter.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..d6b618909800ceab63158a1f178f7486ba24ef48
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/formatting/__pycache__/jax_formatter.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/formatting/__pycache__/np_formatter.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/formatting/__pycache__/np_formatter.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..4945fa125321a63f8d210a715c192421c1ef566b
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/formatting/__pycache__/np_formatter.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/formatting/__pycache__/polars_formatter.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/formatting/__pycache__/polars_formatter.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..8926ca1820470167a0ad9316f420d476841e9211
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/formatting/__pycache__/polars_formatter.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/formatting/__pycache__/tf_formatter.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/formatting/__pycache__/tf_formatter.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..a4c3cc79d66c3582cf3ea1f3dc53348fbf901718
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/formatting/__pycache__/tf_formatter.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/formatting/__pycache__/torch_formatter.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/formatting/__pycache__/torch_formatter.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..e487e140fc145dd3c15270ac0aa5471d7f93bd26
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/formatting/__pycache__/torch_formatter.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/formatting/formatting.py b/venv/lib/python3.10/site-packages/datasets/formatting/formatting.py
new file mode 100644
index 0000000000000000000000000000000000000000..24805dc1e79a22c6639091d45f9d4ba46ecfa193
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/formatting/formatting.py
@@ -0,0 +1,678 @@
+# Copyright 2020 The HuggingFace Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numbers
+import operator
+from collections.abc import Iterable, Mapping, MutableMapping
+from functools import partial
+
+# Lint as: python3
+from typing import Any, Callable, Generic, Optional, TypeVar, Union
+
+import numpy as np
+import pandas as pd
+import pyarrow as pa
+
+from ..features import Features
+from ..features.features import _ArrayXDExtensionType, _is_zero_copy_only, decode_nested_example, pandas_types_mapper
+from ..table import Table
+from ..utils.py_utils import no_op_if_value_is_null
+
+
+T = TypeVar("T")
+
+RowFormat = TypeVar("RowFormat")
+ColumnFormat = TypeVar("ColumnFormat")
+BatchFormat = TypeVar("BatchFormat")
+
+
+def _is_range_contiguous(key: range) -> bool:
+ return key.step == 1 and key.stop >= key.start
+
+
+def _raise_bad_key_type(key: Any):
+ raise TypeError(
+ f"Wrong key type: '{key}' of type '{type(key)}'. Expected one of int, slice, range, str or Iterable."
+ )
+
+
+def _query_table_with_indices_mapping(
+ table: Table, key: Union[int, slice, range, str, Iterable], indices: Table
+) -> pa.Table:
+ """
+ Query a pyarrow Table to extract the subtable that correspond to the given key.
+ The :obj:`indices` parameter corresponds to the indices mapping in case we cant to take into
+ account a shuffling or an indices selection for example.
+ The indices table must contain one column named "indices" of type uint64.
+ """
+ if isinstance(key, int):
+ key = indices.fast_slice(key % indices.num_rows, 1).column(0)[0].as_py()
+ return _query_table(table, key)
+ if isinstance(key, slice):
+ key = range(*key.indices(indices.num_rows))
+ if isinstance(key, range):
+ if _is_range_contiguous(key) and key.start >= 0:
+ return _query_table(
+ table, [i.as_py() for i in indices.fast_slice(key.start, key.stop - key.start).column(0)]
+ )
+ else:
+ pass # treat as an iterable
+ if isinstance(key, str):
+ table = table.select([key])
+ return _query_table(table, indices.column(0).to_pylist())
+ if isinstance(key, Iterable):
+ return _query_table(table, [indices.fast_slice(i, 1).column(0)[0].as_py() for i in key])
+
+ _raise_bad_key_type(key)
+
+
+def _query_table(table: Table, key: Union[int, slice, range, str, Iterable]) -> pa.Table:
+ """
+ Query a pyarrow Table to extract the subtable that correspond to the given key.
+ """
+ if isinstance(key, int):
+ return table.fast_slice(key % table.num_rows, 1)
+ if isinstance(key, slice):
+ key = range(*key.indices(table.num_rows))
+ if isinstance(key, range):
+ if _is_range_contiguous(key) and key.start >= 0:
+ return table.fast_slice(key.start, key.stop - key.start)
+ else:
+ pass # treat as an iterable
+ if isinstance(key, str):
+ return table.table.drop([column for column in table.column_names if column != key])
+ if isinstance(key, Iterable):
+ key = np.fromiter(key, np.int64)
+ if len(key) == 0:
+ return table.table.slice(0, 0)
+ # don't use pyarrow.Table.take even for pyarrow >=1.0 (see https://issues.apache.org/jira/browse/ARROW-9773)
+ return table.fast_gather(key % table.num_rows)
+
+ _raise_bad_key_type(key)
+
+
+def _is_array_with_nulls(pa_array: pa.Array) -> bool:
+ return pa_array.null_count > 0
+
+
+class BaseArrowExtractor(Generic[RowFormat, ColumnFormat, BatchFormat]):
+ """
+ Arrow extractor are used to extract data from pyarrow tables.
+ It makes it possible to extract rows, columns and batches.
+ These three extractions types have to be implemented.
+ """
+
+ def extract_row(self, pa_table: pa.Table) -> RowFormat:
+ raise NotImplementedError
+
+ def extract_column(self, pa_table: pa.Table) -> ColumnFormat:
+ raise NotImplementedError
+
+ def extract_batch(self, pa_table: pa.Table) -> BatchFormat:
+ raise NotImplementedError
+
+
+def _unnest(py_dict: dict[str, list[T]]) -> dict[str, T]:
+ """Return the first element of a batch (dict) as a row (dict)"""
+ return {key: array[0] for key, array in py_dict.items()}
+
+
+class SimpleArrowExtractor(BaseArrowExtractor[pa.Table, pa.Array, pa.Table]):
+ def extract_row(self, pa_table: pa.Table) -> pa.Table:
+ return pa_table
+
+ def extract_column(self, pa_table: pa.Table) -> pa.Array:
+ return pa_table.column(0)
+
+ def extract_batch(self, pa_table: pa.Table) -> pa.Table:
+ return pa_table
+
+
+class PythonArrowExtractor(BaseArrowExtractor[dict, list, dict]):
+ def extract_row(self, pa_table: pa.Table) -> dict:
+ return _unnest(pa_table.to_pydict())
+
+ def extract_column(self, pa_table: pa.Table) -> list:
+ return pa_table.column(0).to_pylist()
+
+ def extract_batch(self, pa_table: pa.Table) -> dict:
+ return pa_table.to_pydict()
+
+
+class NumpyArrowExtractor(BaseArrowExtractor[dict, np.ndarray, dict]):
+ def __init__(self, **np_array_kwargs):
+ self.np_array_kwargs = np_array_kwargs
+
+ def extract_row(self, pa_table: pa.Table) -> dict:
+ return _unnest(self.extract_batch(pa_table))
+
+ def extract_column(self, pa_table: pa.Table) -> np.ndarray:
+ return self._arrow_array_to_numpy(pa_table[pa_table.column_names[0]])
+
+ def extract_batch(self, pa_table: pa.Table) -> dict:
+ return {col: self._arrow_array_to_numpy(pa_table[col]) for col in pa_table.column_names}
+
+ def _arrow_array_to_numpy(self, pa_array: pa.Array) -> np.ndarray:
+ if isinstance(pa_array, pa.ChunkedArray):
+ if isinstance(pa_array.type, _ArrayXDExtensionType):
+ # don't call to_pylist() to preserve dtype of the fixed-size array
+ zero_copy_only = _is_zero_copy_only(pa_array.type.storage_dtype, unnest=True)
+ array: list = [
+ row for chunk in pa_array.chunks for row in chunk.to_numpy(zero_copy_only=zero_copy_only)
+ ]
+ else:
+ zero_copy_only = _is_zero_copy_only(pa_array.type) and all(
+ not _is_array_with_nulls(chunk) for chunk in pa_array.chunks
+ )
+ array: list = [
+ row for chunk in pa_array.chunks for row in chunk.to_numpy(zero_copy_only=zero_copy_only)
+ ]
+ else:
+ if isinstance(pa_array.type, _ArrayXDExtensionType):
+ # don't call to_pylist() to preserve dtype of the fixed-size array
+ zero_copy_only = _is_zero_copy_only(pa_array.type.storage_dtype, unnest=True)
+ array: list = pa_array.to_numpy(zero_copy_only=zero_copy_only)
+ else:
+ zero_copy_only = _is_zero_copy_only(pa_array.type) and not _is_array_with_nulls(pa_array)
+ array: list = pa_array.to_numpy(zero_copy_only=zero_copy_only).tolist()
+
+ if len(array) > 0:
+ if any(
+ (isinstance(x, np.ndarray) and (x.dtype == object or x.shape != array[0].shape))
+ or (isinstance(x, float) and np.isnan(x))
+ for x in array
+ ):
+ if np.lib.NumpyVersion(np.__version__) >= "2.0.0b1":
+ return np.asarray(array, dtype=object)
+ return np.array(array, copy=False, dtype=object)
+ if np.lib.NumpyVersion(np.__version__) >= "2.0.0b1":
+ return np.asarray(array)
+ else:
+ return np.array(array, copy=False)
+
+
+class PandasArrowExtractor(BaseArrowExtractor[pd.DataFrame, pd.Series, pd.DataFrame]):
+ def extract_row(self, pa_table: pa.Table) -> pd.DataFrame:
+ return pa_table.slice(length=1).to_pandas(types_mapper=pandas_types_mapper)
+
+ def extract_column(self, pa_table: pa.Table) -> pd.Series:
+ return pa_table.select([0]).to_pandas(types_mapper=pandas_types_mapper)[pa_table.column_names[0]]
+
+ def extract_batch(self, pa_table: pa.Table) -> pd.DataFrame:
+ return pa_table.to_pandas(types_mapper=pandas_types_mapper)
+
+
+class PythonFeaturesDecoder:
+ def __init__(
+ self, features: Optional[Features], token_per_repo_id: Optional[dict[str, Union[str, bool, None]]] = None
+ ):
+ self.features = features
+ self.token_per_repo_id = token_per_repo_id
+
+ def decode_row(self, row: dict) -> dict:
+ return self.features.decode_example(row, token_per_repo_id=self.token_per_repo_id) if self.features else row
+
+ def decode_column(self, column: list, column_name: str) -> list:
+ return (
+ self.features.decode_column(column, column_name, token_per_repo_id=self.token_per_repo_id)
+ if self.features
+ else column
+ )
+
+ def decode_batch(self, batch: dict) -> dict:
+ return self.features.decode_batch(batch, token_per_repo_id=self.token_per_repo_id) if self.features else batch
+
+
+class PandasFeaturesDecoder:
+ def __init__(self, features: Optional[Features]):
+ self.features = features
+
+ def decode_row(self, row: pd.DataFrame) -> pd.DataFrame:
+ decode = (
+ {
+ column_name: no_op_if_value_is_null(partial(decode_nested_example, feature))
+ for column_name, feature in self.features.items()
+ if self.features._column_requires_decoding[column_name]
+ }
+ if self.features
+ else {}
+ )
+ if decode:
+ row[list(decode.keys())] = row.transform(decode)
+ return row
+
+ def decode_column(self, column: pd.Series, column_name: str) -> pd.Series:
+ decode = (
+ no_op_if_value_is_null(partial(decode_nested_example, self.features[column_name]))
+ if self.features and column_name in self.features and self.features._column_requires_decoding[column_name]
+ else None
+ )
+ if decode:
+ column = column.transform(decode)
+ return column
+
+ def decode_batch(self, batch: pd.DataFrame) -> pd.DataFrame:
+ return self.decode_row(batch)
+
+
+class LazyDict(MutableMapping):
+ """A dictionary backed by Arrow data. The values are formatted on-the-fly when accessing the dictionary."""
+
+ def __init__(self, pa_table: pa.Table, formatter: "Formatter"):
+ self.pa_table = pa_table
+ self.formatter = formatter
+
+ self.data = dict.fromkeys(pa_table.column_names)
+ self.keys_to_format = set(self.data.keys())
+
+ def __len__(self):
+ return len(self.data)
+
+ def __getitem__(self, key):
+ value = self.data[key]
+ if key in self.keys_to_format:
+ value = self.format(key)
+ self.data[key] = value
+ self.keys_to_format.remove(key)
+ return value
+
+ def __setitem__(self, key, value):
+ if key in self.keys_to_format:
+ self.keys_to_format.remove(key)
+ self.data[key] = value
+
+ def __delitem__(self, key) -> None:
+ if key in self.keys_to_format:
+ self.keys_to_format.remove(key)
+ del self.data[key]
+
+ def __iter__(self):
+ return iter(self.data)
+
+ def __contains__(self, key):
+ return key in self.data
+
+ def __repr__(self):
+ self._format_all()
+ return repr(self.data)
+
+ def __or__(self, other):
+ if isinstance(other, LazyDict):
+ inst = self.copy()
+ other = other.copy()
+ other._format_all()
+ inst.keys_to_format -= other.data.keys()
+ inst.data = inst.data | other.data
+ return inst
+ if isinstance(other, dict):
+ inst = self.copy()
+ inst.keys_to_format -= other.keys()
+ inst.data = inst.data | other
+ return inst
+ return NotImplemented
+
+ def __ror__(self, other):
+ if isinstance(other, LazyDict):
+ inst = self.copy()
+ other = other.copy()
+ other._format_all()
+ inst.keys_to_format -= other.data.keys()
+ inst.data = other.data | inst.data
+ return inst
+ if isinstance(other, dict):
+ inst = self.copy()
+ inst.keys_to_format -= other.keys()
+ inst.data = other | inst.data
+ return inst
+ return NotImplemented
+
+ def __ior__(self, other):
+ if isinstance(other, LazyDict):
+ other = other.copy()
+ other._format_all()
+ self.keys_to_format -= other.data.keys()
+ self.data |= other.data
+ else:
+ self.keys_to_format -= other.keys()
+ self.data |= other
+ return self
+
+ def __copy__(self):
+ # Identical to `UserDict.__copy__`
+ inst = self.__class__.__new__(self.__class__)
+ inst.__dict__.update(self.__dict__)
+ # Create a copy and avoid triggering descriptors
+ inst.__dict__["data"] = self.__dict__["data"].copy()
+ inst.__dict__["keys_to_format"] = self.__dict__["keys_to_format"].copy()
+ return inst
+
+ def copy(self):
+ import copy
+
+ return copy.copy(self)
+
+ @classmethod
+ def fromkeys(cls, iterable, value=None):
+ raise NotImplementedError
+
+ def format(self, key):
+ raise NotImplementedError
+
+ def _format_all(self):
+ for key in self.keys_to_format:
+ self.data[key] = self.format(key)
+ self.keys_to_format.clear()
+
+
+class LazyRow(LazyDict):
+ def format(self, key):
+ return self.formatter.format_column(self.pa_table.select([key]))[0]
+
+
+class LazyBatch(LazyDict):
+ def format(self, key):
+ return self.formatter.format_column(self.pa_table.select([key]))
+
+
+class Formatter(Generic[RowFormat, ColumnFormat, BatchFormat]):
+ """
+ A formatter is an object that extracts and formats data from pyarrow tables.
+ It defines the formatting for rows, columns and batches.
+ """
+
+ simple_arrow_extractor = SimpleArrowExtractor
+ python_arrow_extractor = PythonArrowExtractor
+ numpy_arrow_extractor = NumpyArrowExtractor
+ pandas_arrow_extractor = PandasArrowExtractor
+
+ def __init__(
+ self,
+ features: Optional[Features] = None,
+ token_per_repo_id: Optional[dict[str, Union[str, bool, None]]] = None,
+ ):
+ self.features = features
+ self.token_per_repo_id = token_per_repo_id
+ self.python_features_decoder = PythonFeaturesDecoder(self.features, self.token_per_repo_id)
+ self.pandas_features_decoder = PandasFeaturesDecoder(self.features)
+
+ def __call__(self, pa_table: pa.Table, query_type: str) -> Union[RowFormat, ColumnFormat, BatchFormat]:
+ if query_type == "row":
+ return self.format_row(pa_table)
+ elif query_type == "column":
+ return self.format_column(pa_table)
+ elif query_type == "batch":
+ return self.format_batch(pa_table)
+
+ def format_row(self, pa_table: pa.Table) -> RowFormat:
+ raise NotImplementedError
+
+ def format_column(self, pa_table: pa.Table) -> ColumnFormat:
+ raise NotImplementedError
+
+ def format_batch(self, pa_table: pa.Table) -> BatchFormat:
+ raise NotImplementedError
+
+
+class TensorFormatter(Formatter[RowFormat, ColumnFormat, BatchFormat]):
+ def recursive_tensorize(self, data_struct: dict):
+ raise NotImplementedError
+
+
+class TableFormatter(Formatter[RowFormat, ColumnFormat, BatchFormat]):
+ table_type: str
+ column_type: str
+
+
+class ArrowFormatter(TableFormatter[pa.Table, pa.Array, pa.Table]):
+ table_type = "arrow table"
+ column_type = "arrow array"
+
+ def format_row(self, pa_table: pa.Table) -> pa.Table:
+ return self.simple_arrow_extractor().extract_row(pa_table)
+
+ def format_column(self, pa_table: pa.Table) -> pa.Array:
+ return self.simple_arrow_extractor().extract_column(pa_table)
+
+ def format_batch(self, pa_table: pa.Table) -> pa.Table:
+ return self.simple_arrow_extractor().extract_batch(pa_table)
+
+
+class PythonFormatter(Formatter[Mapping, list, Mapping]):
+ def __init__(self, features=None, lazy=False, token_per_repo_id=None):
+ super().__init__(features, token_per_repo_id)
+ self.lazy = lazy
+
+ def format_row(self, pa_table: pa.Table) -> Mapping:
+ if self.lazy:
+ return LazyRow(pa_table, self)
+ row = self.python_arrow_extractor().extract_row(pa_table)
+ row = self.python_features_decoder.decode_row(row)
+ return row
+
+ def format_column(self, pa_table: pa.Table) -> list:
+ column = self.python_arrow_extractor().extract_column(pa_table)
+ column = self.python_features_decoder.decode_column(column, pa_table.column_names[0])
+ return column
+
+ def format_batch(self, pa_table: pa.Table) -> Mapping:
+ if self.lazy:
+ return LazyBatch(pa_table, self)
+ batch = self.python_arrow_extractor().extract_batch(pa_table)
+ batch = self.python_features_decoder.decode_batch(batch)
+ return batch
+
+
+class PandasFormatter(TableFormatter[pd.DataFrame, pd.Series, pd.DataFrame]):
+ table_type = "pandas dataframe"
+ column_type = "pandas series"
+
+ def format_row(self, pa_table: pa.Table) -> pd.DataFrame:
+ row = self.pandas_arrow_extractor().extract_row(pa_table)
+ row = self.pandas_features_decoder.decode_row(row)
+ return row
+
+ def format_column(self, pa_table: pa.Table) -> pd.Series:
+ column = self.pandas_arrow_extractor().extract_column(pa_table)
+ column = self.pandas_features_decoder.decode_column(column, pa_table.column_names[0])
+ return column
+
+ def format_batch(self, pa_table: pa.Table) -> pd.DataFrame:
+ row = self.pandas_arrow_extractor().extract_batch(pa_table)
+ row = self.pandas_features_decoder.decode_batch(row)
+ return row
+
+
+class CustomFormatter(Formatter[dict, ColumnFormat, dict]):
+ """
+ A user-defined custom formatter function defined by a ``transform``.
+ The transform must take as input a batch of data extracted for an arrow table using the python extractor,
+ and return a batch.
+ If the output batch is not a dict, then output_all_columns won't work.
+ If the output batch has several fields, then querying a single column won't work since we don't know which field
+ to return.
+ """
+
+ def __init__(self, transform: Callable[[dict], dict], features=None, token_per_repo_id=None, **kwargs):
+ super().__init__(features=features, token_per_repo_id=token_per_repo_id)
+ self.transform = transform
+
+ def format_row(self, pa_table: pa.Table) -> dict:
+ formatted_batch = self.format_batch(pa_table)
+ try:
+ return _unnest(formatted_batch)
+ except Exception as exc:
+ raise TypeError(
+ f"Custom formatting function must return a dict of sequences to be able to pick a row, but got {formatted_batch}"
+ ) from exc
+
+ def format_column(self, pa_table: pa.Table) -> ColumnFormat:
+ formatted_batch = self.format_batch(pa_table)
+ if hasattr(formatted_batch, "keys"):
+ if len(formatted_batch.keys()) > 1:
+ raise TypeError(
+ "Tried to query a column but the custom formatting function returns too many columns. "
+ f"Only one column was expected but got columns {list(formatted_batch.keys())}."
+ )
+ else:
+ raise TypeError(
+ f"Custom formatting function must return a dict to be able to pick a row, but got {formatted_batch}"
+ )
+ try:
+ return formatted_batch[pa_table.column_names[0]]
+ except Exception as exc:
+ raise TypeError(
+ f"Custom formatting function must return a dict to be able to pick a row, but got {formatted_batch}"
+ ) from exc
+
+ def format_batch(self, pa_table: pa.Table) -> dict:
+ batch = self.python_arrow_extractor().extract_batch(pa_table)
+ batch = self.python_features_decoder.decode_batch(batch)
+ return self.transform(batch)
+
+
+def _check_valid_column_key(key: str, columns: list[str]) -> None:
+ if key not in columns:
+ raise KeyError(f"Column {key} not in the dataset. Current columns in the dataset: {columns}")
+
+
+def _check_valid_index_key(key: Union[int, slice, range, Iterable], size: int) -> None:
+ if isinstance(key, int):
+ if (key < 0 and key + size < 0) or (key >= size):
+ raise IndexError(f"Invalid key: {key} is out of bounds for size {size}")
+ return
+ elif isinstance(key, slice):
+ pass
+ elif isinstance(key, range):
+ if len(key) > 0:
+ _check_valid_index_key(max(key), size=size)
+ _check_valid_index_key(min(key), size=size)
+ elif isinstance(key, Iterable):
+ if len(key) > 0:
+ _check_valid_index_key(int(max(key)), size=size)
+ _check_valid_index_key(int(min(key)), size=size)
+ else:
+ _raise_bad_key_type(key)
+
+
+def key_to_query_type(key: Union[int, slice, range, str, Iterable]) -> str:
+ if isinstance(key, numbers.Integral):
+ return "row"
+ elif isinstance(key, str):
+ return "column"
+ elif isinstance(key, (slice, range, Iterable)):
+ return "batch"
+ _raise_bad_key_type(key)
+
+
+def query_table(
+ table: Table,
+ key: Union[int, slice, range, str, Iterable],
+ indices: Optional[Table] = None,
+) -> pa.Table:
+ """
+ Query a Table to extract the subtable that correspond to the given key.
+
+ Args:
+ table (``datasets.table.Table``): The input Table to query from
+ key (``Union[int, slice, range, str, Iterable]``): The key can be of different types:
+ - an integer i: the subtable containing only the i-th row
+ - a slice [i:j:k]: the subtable containing the rows that correspond to this slice
+ - a range(i, j, k): the subtable containing the rows that correspond to this range
+ - a string c: the subtable containing all the rows but only the column c
+ - an iterable l: the subtable that is the concatenation of all the i-th rows for all i in the iterable
+ indices (Optional ``datasets.table.Table``): If not None, it is used to re-map the given key to the table rows.
+ The indices table must contain one column named "indices" of type uint64.
+ This is used in case of shuffling or rows selection.
+
+
+ Returns:
+ ``pyarrow.Table``: the result of the query on the input table
+ """
+ # Check if key is valid
+ if not isinstance(key, (int, slice, range, str, Iterable)):
+ try:
+ key = operator.index(key)
+ except TypeError:
+ _raise_bad_key_type(key)
+ if isinstance(key, str):
+ _check_valid_column_key(key, table.column_names)
+ else:
+ size = indices.num_rows if indices is not None else table.num_rows
+ _check_valid_index_key(key, size)
+ # Query the main table
+ if indices is None:
+ pa_subtable = _query_table(table, key)
+ else:
+ pa_subtable = _query_table_with_indices_mapping(table, key, indices=indices)
+ return pa_subtable
+
+
+def format_table(
+ table: Table,
+ key: Union[int, slice, range, str, Iterable],
+ formatter: Formatter,
+ format_columns: Optional[list] = None,
+ output_all_columns=False,
+):
+ """
+ Format a Table depending on the key that was used and a Formatter object.
+
+ Args:
+ table (``datasets.table.Table``): The input Table to format
+ key (``Union[int, slice, range, str, Iterable]``): Depending on the key that was used, the formatter formats
+ the table as either a row, a column or a batch.
+ formatter (``datasets.formatting.formatting.Formatter``): Any subclass of a Formatter such as
+ PythonFormatter, NumpyFormatter, etc.
+ format_columns (:obj:`List[str]`, optional): if not None, it defines the columns that will be formatted using the
+ given formatter. Other columns are discarded (unless ``output_all_columns`` is True)
+ output_all_columns (:obj:`bool`, defaults to False). If True, the formatted output is completed using the columns
+ that are not in the ``format_columns`` list. For these columns, the PythonFormatter is used.
+
+
+ Returns:
+ A row, column or batch formatted object defined by the Formatter:
+ - the PythonFormatter returns a dictionary for a row or a batch, and a list for a column.
+ - the NumpyFormatter returns a dictionary for a row or a batch, and a np.array for a column.
+ - the PandasFormatter returns a pd.DataFrame for a row or a batch, and a pd.Series for a column.
+ - the TorchFormatter returns a dictionary for a row or a batch, and a torch.Tensor for a column.
+ - the TFFormatter returns a dictionary for a row or a batch, and a tf.Tensor for a column.
+ """
+ if isinstance(table, Table):
+ pa_table = table.table
+ else:
+ pa_table = table
+ query_type = key_to_query_type(key)
+ python_formatter = PythonFormatter(features=formatter.features)
+ if format_columns is None:
+ return formatter(pa_table, query_type=query_type)
+ elif query_type == "column":
+ if key in format_columns:
+ return formatter(pa_table, query_type)
+ else:
+ return python_formatter(pa_table, query_type=query_type)
+ else:
+ pa_table_to_format = pa_table.drop(col for col in pa_table.column_names if col not in format_columns)
+ formatted_output = formatter(pa_table_to_format, query_type=query_type)
+ if output_all_columns:
+ if isinstance(formatted_output, MutableMapping):
+ pa_table_with_remaining_columns = pa_table.drop(
+ col for col in pa_table.column_names if col in format_columns
+ )
+ remaining_columns_dict = python_formatter(pa_table_with_remaining_columns, query_type=query_type)
+ formatted_output.update(remaining_columns_dict)
+ else:
+ raise TypeError(
+ f"Custom formatting function must return a dict to work with output_all_columns=True, but got {formatted_output}"
+ )
+ return formatted_output
diff --git a/venv/lib/python3.10/site-packages/datasets/formatting/jax_formatter.py b/venv/lib/python3.10/site-packages/datasets/formatting/jax_formatter.py
new file mode 100644
index 0000000000000000000000000000000000000000..c52ef7a4d59f1efbb55630791d5bafcfb74beb94
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/formatting/jax_formatter.py
@@ -0,0 +1,171 @@
+# Copyright 2021 The HuggingFace Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+import sys
+from collections.abc import Mapping
+from typing import TYPE_CHECKING, Optional
+
+import numpy as np
+import pyarrow as pa
+
+from .. import config
+from ..utils.logging import get_logger
+from ..utils.py_utils import map_nested
+from .formatting import TensorFormatter
+
+
+if TYPE_CHECKING:
+ import jax
+ import jaxlib
+
+logger = get_logger()
+
+DEVICE_MAPPING: Optional[dict] = None
+
+
+class JaxFormatter(TensorFormatter[Mapping, "jax.Array", Mapping]):
+ def __init__(self, features=None, device=None, token_per_repo_id=None, **jnp_array_kwargs):
+ super().__init__(features=features, token_per_repo_id=token_per_repo_id)
+ import jax
+ from jaxlib.xla_client import Device
+
+ if isinstance(device, Device):
+ raise ValueError(
+ f"Expected {device} to be a `str` not {type(device)}, as `jaxlib.xla_extension.Device` "
+ "is not serializable neither with `pickle` nor with `dill`. Instead you can surround "
+ "the device with `str()` to get its string identifier that will be internally mapped "
+ "to the actual `jaxlib.xla_extension.Device`."
+ )
+ self.device = device if isinstance(device, str) else str(jax.devices()[0])
+ # using global variable since `jaxlib.xla_extension.Device` is not serializable neither
+ # with `pickle` nor with `dill`, so we need to use a global variable instead
+ global DEVICE_MAPPING
+ if DEVICE_MAPPING is None:
+ DEVICE_MAPPING = self._map_devices_to_str()
+ if self.device not in list(DEVICE_MAPPING.keys()):
+ logger.warning(
+ f"Device with string identifier {self.device} not listed among the available "
+ f"devices: {list(DEVICE_MAPPING.keys())}, so falling back to the default "
+ f"device: {str(jax.devices()[0])}."
+ )
+ self.device = str(jax.devices()[0])
+ self.jnp_array_kwargs = jnp_array_kwargs
+
+ @staticmethod
+ def _map_devices_to_str() -> dict[str, "jaxlib.xla_extension.Device"]:
+ import jax
+
+ return {str(device): device for device in jax.devices()}
+
+ def _consolidate(self, column):
+ import jax
+ import jax.numpy as jnp
+
+ if isinstance(column, list) and column:
+ if all(
+ isinstance(x, jax.Array) and x.shape == column[0].shape and x.dtype == column[0].dtype for x in column
+ ):
+ return jnp.stack(column, axis=0)
+ return column
+
+ def _tensorize(self, value):
+ import jax
+ import jax.numpy as jnp
+
+ if isinstance(value, (str, bytes, type(None))):
+ return value
+ elif isinstance(value, (np.character, np.ndarray)) and np.issubdtype(value.dtype, np.character):
+ return value.tolist()
+
+ default_dtype = {}
+
+ if isinstance(value, (np.number, np.ndarray)) and np.issubdtype(value.dtype, np.integer):
+ # the default int precision depends on the jax config
+ # see https://jax.readthedocs.io/en/latest/notebooks/Common_Gotchas_in_JAX.html#double-64bit-precision
+ if jax.config.jax_enable_x64:
+ default_dtype = {"dtype": jnp.int64}
+ else:
+ default_dtype = {"dtype": jnp.int32}
+ elif isinstance(value, (np.number, np.ndarray)) and np.issubdtype(value.dtype, np.floating):
+ default_dtype = {"dtype": jnp.float32}
+
+ if config.PIL_AVAILABLE and "PIL" in sys.modules:
+ import PIL.Image
+
+ if isinstance(value, PIL.Image.Image):
+ value = np.asarray(value)
+ if config.TORCHVISION_AVAILABLE and "torchvision" in sys.modules:
+ from torchvision.io import VideoReader
+
+ if isinstance(value, VideoReader):
+ return value # TODO(QL): set output to jax arrays ?
+ if config.TORCHCODEC_AVAILABLE and "torchcodec" in sys.modules:
+ from torchcodec.decoders import AudioDecoder, VideoDecoder
+
+ if isinstance(value, (VideoDecoder, AudioDecoder)):
+ return value # TODO(QL): set output to jax arrays ?
+
+ # using global variable since `jaxlib.xla_extension.Device` is not serializable neither
+ # with `pickle` nor with `dill`, so we need to use a global variable instead
+ global DEVICE_MAPPING
+ if DEVICE_MAPPING is None:
+ DEVICE_MAPPING = self._map_devices_to_str()
+
+ with jax.default_device(DEVICE_MAPPING[self.device]):
+ # calling jnp.array on a np.ndarray does copy the data
+ # see https://github.com/google/jax/issues/4486
+ return jnp.array(value, **{**default_dtype, **self.jnp_array_kwargs})
+
+ def _recursive_tensorize(self, data_struct):
+ import jax
+
+ # support for torch, tf, jax etc.
+ if config.TORCH_AVAILABLE and "torch" in sys.modules:
+ import torch
+
+ if isinstance(data_struct, torch.Tensor):
+ return self._tensorize(data_struct.detach().cpu().numpy()[()])
+ if hasattr(data_struct, "__array__") and not isinstance(data_struct, jax.Array):
+ data_struct = data_struct.__array__()
+ # support for nested types like struct of list of struct
+ if isinstance(data_struct, np.ndarray):
+ if data_struct.dtype == object: # jax arrays cannot be instantied from an array of objects
+ return self._consolidate([self.recursive_tensorize(substruct) for substruct in data_struct])
+ elif isinstance(data_struct, (list, tuple)):
+ return self._consolidate([self.recursive_tensorize(substruct) for substruct in data_struct])
+ return self._tensorize(data_struct)
+
+ def recursive_tensorize(self, data_struct: dict):
+ return map_nested(self._recursive_tensorize, data_struct, map_list=False)
+
+ def format_row(self, pa_table: pa.Table) -> Mapping:
+ row = self.numpy_arrow_extractor().extract_row(pa_table)
+ row = self.python_features_decoder.decode_row(row)
+ return self.recursive_tensorize(row)
+
+ def format_column(self, pa_table: pa.Table) -> "jax.Array":
+ column = self.numpy_arrow_extractor().extract_column(pa_table)
+ column = self.python_features_decoder.decode_column(column, pa_table.column_names[0])
+ column = self.recursive_tensorize(column)
+ column = self._consolidate(column)
+ return column
+
+ def format_batch(self, pa_table: pa.Table) -> Mapping:
+ batch = self.numpy_arrow_extractor().extract_batch(pa_table)
+ batch = self.python_features_decoder.decode_batch(batch)
+ batch = self.recursive_tensorize(batch)
+ for column_name in batch:
+ batch[column_name] = self._consolidate(batch[column_name])
+ return batch
diff --git a/venv/lib/python3.10/site-packages/datasets/formatting/np_formatter.py b/venv/lib/python3.10/site-packages/datasets/formatting/np_formatter.py
new file mode 100644
index 0000000000000000000000000000000000000000..062d199c6f68e0d6a07172726a3cf4f445b0425f
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/formatting/np_formatter.py
@@ -0,0 +1,117 @@
+# Copyright 2020 The HuggingFace Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+from collections.abc import Mapping
+
+import numpy as np
+import pyarrow as pa
+
+from .. import config
+from ..utils.py_utils import map_nested
+from .formatting import TensorFormatter
+
+
+class NumpyFormatter(TensorFormatter[Mapping, np.ndarray, Mapping]):
+ def __init__(self, features=None, token_per_repo_id=None, **np_array_kwargs):
+ super().__init__(features=features, token_per_repo_id=token_per_repo_id)
+ self.np_array_kwargs = np_array_kwargs
+
+ def _consolidate(self, column):
+ if isinstance(column, list):
+ if column and all(
+ isinstance(x, np.ndarray) and x.shape == column[0].shape and x.dtype == column[0].dtype for x in column
+ ):
+ return np.stack(column)
+ else:
+ # don't use np.array(column, dtype=object)
+ # since it fails in certain cases
+ # see https://stackoverflow.com/q/51005699
+ out = np.empty(len(column), dtype=object)
+ out[:] = column
+ return out
+ return column
+
+ def _tensorize(self, value):
+ if isinstance(value, (str, bytes, type(None))):
+ return value
+ elif isinstance(value, (np.character, np.ndarray)) and np.issubdtype(value.dtype, np.character):
+ return value
+ elif isinstance(value, np.number):
+ return value
+
+ default_dtype = {}
+
+ if isinstance(value, np.ndarray) and np.issubdtype(value.dtype, np.integer):
+ default_dtype = {"dtype": np.int64}
+ elif isinstance(value, np.ndarray) and np.issubdtype(value.dtype, np.floating):
+ default_dtype = {"dtype": np.float32}
+
+ if config.PIL_AVAILABLE and "PIL" in sys.modules:
+ import PIL.Image
+
+ if isinstance(value, PIL.Image.Image):
+ return np.asarray(value, **self.np_array_kwargs)
+ if config.TORCHVISION_AVAILABLE and "torchvision" in sys.modules:
+ from torchvision.io import VideoReader
+
+ if isinstance(value, VideoReader):
+ return value # TODO(QL): set output to np arrays ?
+ if config.TORCHCODEC_AVAILABLE and "torchcodec" in sys.modules:
+ from torchcodec.decoders import AudioDecoder, VideoDecoder
+
+ if isinstance(value, (VideoDecoder, AudioDecoder)):
+ return value # TODO(QL): set output to np arrays ?
+
+ return np.asarray(value, **{**default_dtype, **self.np_array_kwargs})
+
+ def _recursive_tensorize(self, data_struct):
+ # support for torch, tf, jax etc.
+ if config.TORCH_AVAILABLE and "torch" in sys.modules:
+ import torch
+
+ if isinstance(data_struct, torch.Tensor):
+ return self._tensorize(data_struct.detach().cpu().numpy()[()])
+ if hasattr(data_struct, "__array__") and not isinstance(data_struct, (np.ndarray, np.character, np.number)):
+ data_struct = data_struct.__array__()
+ # support for nested types like struct of list of struct
+ if isinstance(data_struct, np.ndarray):
+ if data_struct.dtype == object:
+ return self._consolidate([self.recursive_tensorize(substruct) for substruct in data_struct])
+ if isinstance(data_struct, (list, tuple)):
+ return self._consolidate([self.recursive_tensorize(substruct) for substruct in data_struct])
+ return self._tensorize(data_struct)
+
+ def recursive_tensorize(self, data_struct: dict):
+ return map_nested(self._recursive_tensorize, data_struct, map_list=False)
+
+ def format_row(self, pa_table: pa.Table) -> Mapping:
+ row = self.numpy_arrow_extractor().extract_row(pa_table)
+ row = self.python_features_decoder.decode_row(row)
+ return self.recursive_tensorize(row)
+
+ def format_column(self, pa_table: pa.Table) -> np.ndarray:
+ column = self.numpy_arrow_extractor().extract_column(pa_table)
+ column = self.python_features_decoder.decode_column(column, pa_table.column_names[0])
+ column = self.recursive_tensorize(column)
+ column = self._consolidate(column)
+ return column
+
+ def format_batch(self, pa_table: pa.Table) -> Mapping:
+ batch = self.numpy_arrow_extractor().extract_batch(pa_table)
+ batch = self.python_features_decoder.decode_batch(batch)
+ batch = self.recursive_tensorize(batch)
+ for column_name in batch:
+ batch[column_name] = self._consolidate(batch[column_name])
+ return batch
diff --git a/venv/lib/python3.10/site-packages/datasets/formatting/polars_formatter.py b/venv/lib/python3.10/site-packages/datasets/formatting/polars_formatter.py
new file mode 100644
index 0000000000000000000000000000000000000000..7ea2f783aec38ac6bbc10d12aff2488655940f3f
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/formatting/polars_formatter.py
@@ -0,0 +1,124 @@
+# Copyright 2020 The HuggingFace Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+from functools import partial
+from typing import TYPE_CHECKING, Optional
+
+import pyarrow as pa
+
+from .. import config
+from ..features import Features
+from ..features.features import decode_nested_example
+from ..utils.py_utils import no_op_if_value_is_null
+from .formatting import BaseArrowExtractor, TableFormatter
+
+
+if TYPE_CHECKING:
+ import polars as pl
+
+
+class PolarsArrowExtractor(BaseArrowExtractor["pl.DataFrame", "pl.Series", "pl.DataFrame"]):
+ def extract_row(self, pa_table: pa.Table) -> "pl.DataFrame":
+ if config.POLARS_AVAILABLE:
+ if "polars" not in sys.modules:
+ import polars
+ else:
+ polars = sys.modules["polars"]
+
+ return polars.from_arrow(pa_table.slice(length=1))
+ else:
+ raise ValueError("Polars needs to be installed to be able to return Polars dataframes.")
+
+ def extract_column(self, pa_table: pa.Table) -> "pl.Series":
+ if config.POLARS_AVAILABLE:
+ if "polars" not in sys.modules:
+ import polars
+ else:
+ polars = sys.modules["polars"]
+
+ return polars.from_arrow(pa_table.select([0]))[pa_table.column_names[0]]
+ else:
+ raise ValueError("Polars needs to be installed to be able to return Polars dataframes.")
+
+ def extract_batch(self, pa_table: pa.Table) -> "pl.DataFrame":
+ if config.POLARS_AVAILABLE:
+ if "polars" not in sys.modules:
+ import polars
+ else:
+ polars = sys.modules["polars"]
+
+ return polars.from_arrow(pa_table)
+ else:
+ raise ValueError("Polars needs to be installed to be able to return Polars dataframes.")
+
+
+class PolarsFeaturesDecoder:
+ def __init__(self, features: Optional[Features]):
+ self.features = features
+ import polars as pl # noqa: F401 - import pl at initialization
+
+ def decode_row(self, row: "pl.DataFrame") -> "pl.DataFrame":
+ decode = (
+ {
+ column_name: no_op_if_value_is_null(partial(decode_nested_example, feature))
+ for column_name, feature in self.features.items()
+ if self.features._column_requires_decoding[column_name]
+ }
+ if self.features
+ else {}
+ )
+ if decode:
+ row[list(decode.keys())] = row.map_rows(decode)
+ return row
+
+ def decode_column(self, column: "pl.Series", column_name: str) -> "pl.Series":
+ decode = (
+ no_op_if_value_is_null(partial(decode_nested_example, self.features[column_name]))
+ if self.features and column_name in self.features and self.features._column_requires_decoding[column_name]
+ else None
+ )
+ if decode:
+ column = column.map_elements(decode)
+ return column
+
+ def decode_batch(self, batch: "pl.DataFrame") -> "pl.DataFrame":
+ return self.decode_row(batch)
+
+
+class PolarsFormatter(TableFormatter["pl.DataFrame", "pl.Series", "pl.DataFrame"]):
+ table_type = "polars dataframe"
+ column_type = "polars series"
+
+ def __init__(self, features=None, **np_array_kwargs):
+ super().__init__(features=features)
+ self.np_array_kwargs = np_array_kwargs
+ self.polars_arrow_extractor = PolarsArrowExtractor
+ self.polars_features_decoder = PolarsFeaturesDecoder(features)
+ import polars as pl # noqa: F401 - import pl at initialization
+
+ def format_row(self, pa_table: pa.Table) -> "pl.DataFrame":
+ row = self.polars_arrow_extractor().extract_row(pa_table)
+ row = self.polars_features_decoder.decode_row(row)
+ return row
+
+ def format_column(self, pa_table: pa.Table) -> "pl.Series":
+ column = self.polars_arrow_extractor().extract_column(pa_table)
+ column = self.polars_features_decoder.decode_column(column, pa_table.column_names[0])
+ return column
+
+ def format_batch(self, pa_table: pa.Table) -> "pl.DataFrame":
+ row = self.polars_arrow_extractor().extract_batch(pa_table)
+ row = self.polars_features_decoder.decode_batch(row)
+ return row
diff --git a/venv/lib/python3.10/site-packages/datasets/formatting/tf_formatter.py b/venv/lib/python3.10/site-packages/datasets/formatting/tf_formatter.py
new file mode 100644
index 0000000000000000000000000000000000000000..1a20eb31d1d081b451f3988c3bb40203ce5c5a65
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/formatting/tf_formatter.py
@@ -0,0 +1,126 @@
+# Copyright 2020 The HuggingFace Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+import sys
+from collections.abc import Mapping
+from typing import TYPE_CHECKING
+
+import numpy as np
+import pyarrow as pa
+
+from .. import config
+from ..utils.py_utils import map_nested
+from .formatting import TensorFormatter
+
+
+if TYPE_CHECKING:
+ import tensorflow as tf
+
+
+class TFFormatter(TensorFormatter[Mapping, "tf.Tensor", Mapping]):
+ def __init__(self, features=None, token_per_repo_id=None, **tf_tensor_kwargs):
+ super().__init__(features=features, token_per_repo_id=token_per_repo_id)
+ self.tf_tensor_kwargs = tf_tensor_kwargs
+ import tensorflow as tf # noqa: F401 - import tf at initialization
+
+ def _consolidate(self, column):
+ import tensorflow as tf
+
+ if isinstance(column, list) and column:
+ if all(
+ isinstance(x, tf.Tensor) and x.shape == column[0].shape and x.dtype == column[0].dtype for x in column
+ ):
+ return tf.stack(column)
+ elif all(
+ isinstance(x, (tf.Tensor, tf.RaggedTensor)) and x.ndim == 1 and x.dtype == column[0].dtype
+ for x in column
+ ):
+ # only rag 1-D tensors, otherwise some dimensions become ragged even though they were consolidated
+ return tf.ragged.stack(column)
+
+ return column
+
+ def _tensorize(self, value):
+ import tensorflow as tf
+
+ if value is None:
+ return value
+
+ default_dtype = {}
+
+ if isinstance(value, (np.number, np.ndarray)) and np.issubdtype(value.dtype, np.integer):
+ default_dtype = {"dtype": tf.int64}
+ elif isinstance(value, (np.number, np.ndarray)) and np.issubdtype(value.dtype, np.floating):
+ default_dtype = {"dtype": tf.float32}
+
+ if config.PIL_AVAILABLE and "PIL" in sys.modules:
+ import PIL.Image
+
+ if isinstance(value, PIL.Image.Image):
+ value = np.asarray(value)
+ if config.TORCHVISION_AVAILABLE and "torchvision" in sys.modules:
+ from torchvision.io import VideoReader
+
+ if isinstance(value, VideoReader):
+ return value # TODO(QL): set output to tf tensors ?
+ if config.TORCHCODEC_AVAILABLE and "torchcodec" in sys.modules:
+ from torchcodec.decoders import AudioDecoder, VideoDecoder
+
+ if isinstance(value, (VideoDecoder, AudioDecoder)):
+ return value # TODO(QL): set output to jax arrays ?
+
+ return tf.convert_to_tensor(value, **{**default_dtype, **self.tf_tensor_kwargs})
+
+ def _recursive_tensorize(self, data_struct):
+ import tensorflow as tf
+
+ # support for torch, tf, jax etc.
+ if config.TORCH_AVAILABLE and "torch" in sys.modules:
+ import torch
+
+ if isinstance(data_struct, torch.Tensor):
+ return self._tensorize(data_struct.detach().cpu().numpy()[()])
+ if hasattr(data_struct, "__array__") and not isinstance(data_struct, tf.Tensor):
+ data_struct = data_struct.__array__()
+ # support for nested types like struct of list of struct
+ if isinstance(data_struct, np.ndarray):
+ if data_struct.dtype == object: # tf tensors cannot be instantied from an array of objects
+ return self._consolidate([self.recursive_tensorize(substruct) for substruct in data_struct])
+ elif isinstance(data_struct, (list, tuple)):
+ return self._consolidate([self.recursive_tensorize(substruct) for substruct in data_struct])
+ return self._tensorize(data_struct)
+
+ def recursive_tensorize(self, data_struct: dict):
+ return map_nested(self._recursive_tensorize, data_struct, map_list=False)
+
+ def format_row(self, pa_table: pa.Table) -> Mapping:
+ row = self.numpy_arrow_extractor().extract_row(pa_table)
+ row = self.python_features_decoder.decode_row(row)
+ return self.recursive_tensorize(row)
+
+ def format_column(self, pa_table: pa.Table) -> "tf.Tensor":
+ column = self.numpy_arrow_extractor().extract_column(pa_table)
+ column = self.python_features_decoder.decode_column(column, pa_table.column_names[0])
+ column = self.recursive_tensorize(column)
+ column = self._consolidate(column)
+ return column
+
+ def format_batch(self, pa_table: pa.Table) -> Mapping:
+ batch = self.numpy_arrow_extractor().extract_batch(pa_table)
+ batch = self.python_features_decoder.decode_batch(batch)
+ batch = self.recursive_tensorize(batch)
+ for column_name in batch:
+ batch[column_name] = self._consolidate(batch[column_name])
+ return batch
diff --git a/venv/lib/python3.10/site-packages/datasets/formatting/torch_formatter.py b/venv/lib/python3.10/site-packages/datasets/formatting/torch_formatter.py
new file mode 100644
index 0000000000000000000000000000000000000000..60d763484459c831859c66bd4805746c6cef1931
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/formatting/torch_formatter.py
@@ -0,0 +1,269 @@
+# Copyright 2020 The HuggingFace Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+import sys
+from collections.abc import Mapping
+from typing import TYPE_CHECKING
+
+import numpy as np
+import pyarrow as pa
+
+from .. import config
+from .formatting import TensorFormatter
+
+
+if TYPE_CHECKING:
+ import torch
+
+# Import torch once at module level once
+try:
+ import torch
+
+ _torch_available = True
+except ImportError:
+ _torch_available = False
+ torch = None
+
+
+class TorchFormatter(TensorFormatter[Mapping, "torch.Tensor", Mapping]):
+ def __init__(self, features=None, token_per_repo_id=None, **torch_tensor_kwargs):
+ super().__init__(features=features, token_per_repo_id=token_per_repo_id)
+ self.torch_tensor_kwargs = torch_tensor_kwargs
+
+ if not _torch_available:
+ raise ImportError("PyTorch is required but not available")
+
+ def _consolidate(self, column):
+ """Smarter consolidation that only stacks when safe and beneficial."""
+ if not isinstance(column, list) or not column:
+ return column
+
+ # Check if all items are tensors with matching properties
+ first = column[0]
+ if not isinstance(first, torch.Tensor):
+ return column
+
+ # Fast check: if all tensors have same shape, dtype, and device, we can stack
+ if all(
+ isinstance(x, torch.Tensor)
+ and x.shape == first.shape
+ and x.dtype == first.dtype
+ and x.device == first.device
+ for x in column
+ ):
+ return torch.stack(column)
+
+ return column
+
+ def _tensorize(self, value):
+ """Zero/low-copy tensor conversion with smart dtype handling."""
+ # Fast path for strings, bytes, None
+ if isinstance(value, (str, bytes, type(None))):
+ return value
+
+ # Handle string arrays
+ if isinstance(value, (np.character, np.ndarray)) and np.issubdtype(value.dtype, np.character):
+ return value.tolist()
+
+ # PIL Image fast path - avoid extra copies
+ if config.PIL_AVAILABLE and "PIL" in sys.modules:
+ import PIL.Image
+
+ if isinstance(value, PIL.Image.Image):
+ # Single conversion path: PIL -> numpy -> torch
+ arr = np.asarray(value)
+ if arr.ndim == 2:
+ arr = arr[:, :, np.newaxis]
+ # Use moveaxis instead of transpose
+ arr = np.moveaxis(arr, -1, 0) # HWC -> CHW
+ # Ensure contiguous for zero-copy conversion
+ if not arr.flags.c_contiguous:
+ arr = np.ascontiguousarray(arr)
+ # Ensure array is writable for torch conversion
+ if not arr.flags.writeable:
+ arr = arr.copy()
+ return torch.from_numpy(arr)
+
+ # Video/Audio decoder passthrough
+ if config.TORCHVISION_AVAILABLE and "torchvision" in sys.modules:
+ from torchvision.io import VideoReader
+
+ if isinstance(value, VideoReader):
+ return value
+
+ if config.TORCHCODEC_AVAILABLE and "torchcodec" in sys.modules:
+ from torchcodec.decoders import AudioDecoder, VideoDecoder
+
+ if isinstance(value, (VideoDecoder, AudioDecoder)):
+ return value
+
+ # Support for other tensor libraries via __array__
+ if hasattr(value, "__array__") and not isinstance(value, torch.Tensor):
+ value = value.__array__()
+
+ # Fast numpy conversion paths
+ if isinstance(value, np.ndarray):
+ # Handle integer types with smart casting
+ if np.issubdtype(value.dtype, np.integer):
+ # Check if user specified a dtype, otherwise default to int64
+ kwargs = self.torch_tensor_kwargs.copy()
+ target_dtype = kwargs.get("dtype", torch.int64)
+
+ # Safe casting for unsigned types
+ if value.dtype in (np.uint16, np.uint32):
+ # Cast to int64 in numpy (fast) then convert to torch
+ value = value.astype(np.int64)
+ if target_dtype == torch.int64:
+ if not value.flags.writeable:
+ value = value.copy()
+ return torch.from_numpy(value)
+ else:
+ if not value.flags.writeable:
+ value = value.copy()
+ kwargs.setdefault("dtype", target_dtype)
+ return torch.as_tensor(value, **kwargs)
+ elif value.dtype == np.uint64:
+ # Check if values fit in int64 range
+ if np.all(value <= np.iinfo(np.int64).max):
+ value = value.astype(np.int64)
+ if target_dtype == torch.int64:
+ if not value.flags.writeable:
+ value = value.copy()
+ return torch.from_numpy(value)
+ else:
+ if not value.flags.writeable:
+ value = value.copy()
+ kwargs.setdefault("dtype", target_dtype)
+ return torch.as_tensor(value, **kwargs)
+ else:
+ # Fallback to safe conversion via Python ints
+ kwargs.setdefault("dtype", target_dtype)
+ return torch.tensor(value, **kwargs)
+ else:
+ # Use zero-copy conversion for compatible integer types
+ if value.dtype == np.int64 and target_dtype == torch.int64:
+ # Perfect match, zero-copy conversion
+ if not value.flags.writeable:
+ value = value.copy()
+ return torch.from_numpy(value)
+ else:
+ # Need dtype conversion, use as_tensor for efficiency
+ if not value.flags.writeable:
+ value = value.copy()
+ kwargs.setdefault("dtype", target_dtype)
+ return torch.as_tensor(value, **kwargs)
+
+ # Handle floating point types
+ elif np.issubdtype(value.dtype, np.floating):
+ # Check if user specified a dtype, otherwise default to float32
+ kwargs = self.torch_tensor_kwargs.copy()
+ target_dtype = kwargs.get("dtype", torch.float32)
+
+ if value.dtype == np.float32 and target_dtype == torch.float32:
+ # Zero-copy conversion, but ensure array is writable
+ if not value.flags.writeable:
+ value = value.copy()
+ return torch.from_numpy(value)
+ else:
+ # Need dtype conversion
+ if not value.flags.writeable:
+ value = value.copy()
+ kwargs.setdefault("dtype", target_dtype)
+ return torch.as_tensor(value, **kwargs)
+ else:
+ # Other numpy types, use zero-copy when possible
+ if not value.flags.writeable:
+ value = value.copy()
+ return torch.from_numpy(value)
+
+ # Handle numpy scalars
+ elif isinstance(value, np.number):
+ kwargs = self.torch_tensor_kwargs.copy()
+ if np.issubdtype(value.dtype, np.integer):
+ # Use torch.as_tensor for scalar conversion with dtype control
+ kwargs.setdefault("dtype", torch.int64)
+ return torch.as_tensor(value, **kwargs)
+ elif np.issubdtype(value.dtype, np.floating):
+ kwargs.setdefault("dtype", torch.float32)
+ return torch.as_tensor(value, **kwargs)
+ else:
+ return torch.as_tensor(value, **kwargs)
+
+ # Handle Python lists/tuples of numbers efficiently
+ elif isinstance(value, (list, tuple)):
+ # Try to convert to numpy first for faster tensor creation
+ try:
+ arr = np.array(value)
+ if arr.dtype.kind in "iuf": # integer, unsigned, float
+ return self._tensorize(arr) # Recursive call to handle numpy path
+ except (ValueError, TypeError):
+ pass # Fall back to torch.tensor
+
+ # Default fallback with dtype defaults
+ default_dtype = {}
+ if isinstance(value, (int, float)):
+ if isinstance(value, int):
+ default_dtype = {"dtype": torch.int64}
+ else:
+ default_dtype = {"dtype": torch.float32}
+
+ return torch.tensor(value, **{**default_dtype, **self.torch_tensor_kwargs})
+
+ def _recursive_tensorize(self, data_struct):
+ """Optimized recursive walker with reduced Python overhead."""
+ # Handle tensor-like objects with __array__ interface
+ if hasattr(data_struct, "__array__") and not isinstance(data_struct, torch.Tensor):
+ data_struct = data_struct.__array__()
+
+ # Handle object arrays (nested structures)
+ if isinstance(data_struct, np.ndarray):
+ if data_struct.dtype == object:
+ # Use list comprehension instead of map_nested
+ result = [self._recursive_tensorize(item) for item in data_struct]
+ return self._consolidate(result)
+ # Handle lists and tuples
+ elif isinstance(data_struct, (list, tuple)):
+ result = [self._recursive_tensorize(item) for item in data_struct]
+ return self._consolidate(result)
+ # Handle dictionaries
+ elif isinstance(data_struct, dict):
+ return {key: self._recursive_tensorize(value) for key, value in data_struct.items()}
+
+ # Base case: tensorize the leaf value
+ return self._tensorize(data_struct)
+
+ def recursive_tensorize(self, data_struct: dict):
+ """Public interface maintaining compatibility."""
+ return self._recursive_tensorize(data_struct)
+
+ def format_row(self, pa_table: pa.Table) -> Mapping:
+ row = self.numpy_arrow_extractor().extract_row(pa_table)
+ row = self.python_features_decoder.decode_row(row)
+ return self.recursive_tensorize(row)
+
+ def format_column(self, pa_table: pa.Table) -> "torch.Tensor":
+ column = self.numpy_arrow_extractor().extract_column(pa_table)
+ column = self.python_features_decoder.decode_column(column, pa_table.column_names[0])
+ column = self.recursive_tensorize(column)
+ column = self._consolidate(column)
+ return column
+
+ def format_batch(self, pa_table: pa.Table) -> Mapping:
+ batch = self.numpy_arrow_extractor().extract_batch(pa_table)
+ batch = self.python_features_decoder.decode_batch(batch)
+ batch = self.recursive_tensorize(batch)
+ for column_name in batch:
+ batch[column_name] = self._consolidate(batch[column_name])
+ return batch
diff --git a/venv/lib/python3.10/site-packages/datasets/hub.py b/venv/lib/python3.10/site-packages/datasets/hub.py
new file mode 100644
index 0000000000000000000000000000000000000000..c6594760de4bb1da4d2726791f4e9726ffaaa611
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/hub.py
@@ -0,0 +1,124 @@
+from itertools import chain
+from typing import Optional, Union
+
+from huggingface_hub import (
+ CommitInfo,
+ CommitOperationAdd,
+ CommitOperationDelete,
+ DatasetCard,
+ DatasetCardData,
+ HfApi,
+ HfFileSystem,
+)
+
+import datasets.config
+from datasets.info import DatasetInfosDict
+from datasets.load import load_dataset_builder
+from datasets.utils.metadata import MetadataConfigs
+
+
+def delete_from_hub(
+ repo_id: str,
+ config_name: str,
+ revision: Optional[str] = None,
+ token: Optional[Union[bool, str]] = None,
+) -> CommitInfo:
+ """Delete a dataset configuration from a [data-only dataset](repository_structure) on the Hub.
+
+ Args:
+ repo_id (`str`): ID of the Hub dataset repository, in the following format: `/` or
+ `/`.
+ config_name (`str`): Name of the dataset configuration.
+ revision (`str`, *optional*): Branch to delete the configuration from. Defaults to the `"main"` branch.
+ token (`bool` or `str`, *optional*): Authentication token for the Hugging Face Hub.
+
+ Returns:
+ `huggingface_hub.CommitInfo`
+ """
+ operations = []
+ # data_files
+ fs = HfFileSystem(endpoint=datasets.config.HF_ENDPOINT, token=token)
+ builder = load_dataset_builder(repo_id, config_name, revision=revision, token=token)
+ for data_file in chain(*builder.config.data_files.values()):
+ data_file_resolved_path = fs.resolve_path(data_file)
+ if data_file_resolved_path.repo_id == repo_id:
+ operations.append(CommitOperationDelete(path_in_repo=data_file_resolved_path.path_in_repo))
+ # README.md
+ dataset_card = DatasetCard.load(repo_id)
+ # config_names
+ if dataset_card.data.get("config_names", None) and config_name in dataset_card.data["config_names"]:
+ dataset_card.data["config_names"].remove(config_name)
+ # metadata_configs
+ metadata_configs = MetadataConfigs.from_dataset_card_data(dataset_card.data)
+ if metadata_configs:
+ _ = metadata_configs.pop(config_name, None)
+ dataset_card_data = DatasetCardData()
+ metadata_configs.to_dataset_card_data(dataset_card_data)
+ if datasets.config.METADATA_CONFIGS_FIELD in dataset_card_data:
+ dataset_card.data[datasets.config.METADATA_CONFIGS_FIELD] = dataset_card_data[
+ datasets.config.METADATA_CONFIGS_FIELD
+ ]
+ else:
+ _ = dataset_card.data.pop(datasets.config.METADATA_CONFIGS_FIELD, None)
+ # dataset_info
+ dataset_infos: DatasetInfosDict = DatasetInfosDict.from_dataset_card_data(dataset_card.data)
+ if dataset_infos:
+ _ = dataset_infos.pop(config_name, None)
+ dataset_card_data = DatasetCardData()
+ dataset_infos.to_dataset_card_data(dataset_card_data)
+ if "dataset_info" in dataset_card_data:
+ dataset_card.data["dataset_info"] = dataset_card_data["dataset_info"]
+ else:
+ _ = dataset_card.data.pop("dataset_info", None)
+ # Commit
+ operations.append(
+ CommitOperationAdd(path_in_repo=datasets.config.REPOCARD_FILENAME, path_or_fileobj=str(dataset_card).encode())
+ )
+ api = HfApi(endpoint=datasets.config.HF_ENDPOINT, token=token)
+ commit_info = api.create_commit(
+ repo_id,
+ operations=operations,
+ commit_message=f"Delete '{config_name}' config",
+ commit_description=f"Delete '{config_name}' config.",
+ token=token,
+ repo_type="dataset",
+ revision=revision,
+ create_pr=True,
+ )
+ print(f"You can find your PR to delete the dataset config at: {commit_info.pr_url}")
+ return commit_info
+
+
+def _delete_files(dataset_id, revision=None, token=None):
+ hf_api = HfApi(endpoint=datasets.config.HF_ENDPOINT, token=token)
+ repo_files = hf_api.list_repo_files(
+ dataset_id,
+ repo_type="dataset",
+ )
+ if repo_files:
+ legacy_json_file = []
+ data_files = []
+ for filename in repo_files:
+ if filename in {".gitattributes", "README.md"}:
+ continue
+ elif filename == "dataset_infos.json":
+ legacy_json_file.append(filename)
+ else:
+ data_files.append(filename)
+ if legacy_json_file:
+ hf_api.delete_file(
+ "dataset_infos.json",
+ dataset_id,
+ repo_type="dataset",
+ revision=revision,
+ commit_message="Delete legacy dataset_infos.json",
+ )
+ if data_files:
+ for filename in data_files:
+ hf_api.delete_file(
+ filename,
+ dataset_id,
+ repo_type="dataset",
+ revision=revision,
+ commit_message="Delete data file",
+ )
diff --git a/venv/lib/python3.10/site-packages/datasets/info.py b/venv/lib/python3.10/site-packages/datasets/info.py
new file mode 100644
index 0000000000000000000000000000000000000000..3723439fb91ab301c1d90c58f4cb98a197cc8a1e
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/info.py
@@ -0,0 +1,430 @@
+# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""DatasetInfo record information we know about a dataset.
+
+This includes things that we know about the dataset statically, i.e.:
+ - description
+ - canonical location
+ - does it have validation and tests splits
+ - size
+ - etc.
+
+This also includes the things that can and should be computed once we've
+processed the dataset as well:
+ - number of examples (in each split)
+ - etc.
+"""
+
+import copy
+import dataclasses
+import json
+import os
+import posixpath
+from dataclasses import dataclass
+from pathlib import Path
+from typing import ClassVar, Optional, Union
+
+import fsspec
+from fsspec.core import url_to_fs
+from huggingface_hub import DatasetCard, DatasetCardData
+
+from . import config
+from .features import Features
+from .splits import SplitDict
+from .utils import Version
+from .utils.logging import get_logger
+from .utils.py_utils import asdict, unique_values
+
+
+logger = get_logger(__name__)
+
+
+@dataclass
+class SupervisedKeysData:
+ input: str = ""
+ output: str = ""
+
+
+@dataclass
+class DownloadChecksumsEntryData:
+ key: str = ""
+ value: str = ""
+
+
+class MissingCachedSizesConfigError(Exception):
+ """The expected cached sizes of the download file are missing."""
+
+
+class NonMatchingCachedSizesError(Exception):
+ """The prepared split doesn't have expected sizes."""
+
+
+@dataclass
+class PostProcessedInfo:
+ features: Optional[Features] = None
+ resources_checksums: Optional[dict] = None
+
+ def __post_init__(self):
+ # Convert back to the correct classes when we reload from dict
+ if self.features is not None and not isinstance(self.features, Features):
+ self.features = Features.from_dict(self.features)
+
+ @classmethod
+ def from_dict(cls, post_processed_info_dict: dict) -> "PostProcessedInfo":
+ field_names = {f.name for f in dataclasses.fields(cls)}
+ return cls(**{k: v for k, v in post_processed_info_dict.items() if k in field_names})
+
+
+@dataclass
+class DatasetInfo:
+ """Information about a dataset.
+
+ `DatasetInfo` documents datasets, including its name, version, and features.
+ See the constructor arguments and properties for a full list.
+
+ Not all fields are known on construction and may be updated later.
+
+ Attributes:
+ description (`str`):
+ A description of the dataset.
+ citation (`str`):
+ A BibTeX citation of the dataset.
+ homepage (`str`):
+ A URL to the official homepage for the dataset.
+ license (`str`):
+ The dataset's license. It can be the name of the license or a paragraph containing the terms of the license.
+ features ([`Features`], *optional*):
+ The features used to specify the dataset's column types.
+ post_processed (`PostProcessedInfo`, *optional*):
+ Information regarding the resources of a possible post-processing of a dataset. For example, it can contain the information of an index.
+ supervised_keys (`SupervisedKeysData`, *optional*):
+ Specifies the input feature and the label for supervised learning if applicable for the dataset (legacy from TFDS).
+ builder_name (`str`, *optional*):
+ The name of the `GeneratorBasedBuilder` subclass used to create the dataset. It is also the snake_case version of the dataset builder class name.
+ config_name (`str`, *optional*):
+ The name of the configuration derived from [`BuilderConfig`].
+ version (`str` or [`Version`], *optional*):
+ The version of the dataset.
+ splits (`dict`, *optional*):
+ The mapping between split name and metadata.
+ download_checksums (`dict`, *optional*):
+ The mapping between the URL to download the dataset's checksums and corresponding metadata.
+ download_size (`int`, *optional*):
+ The size of the files to download to generate the dataset, in bytes.
+ post_processing_size (`int`, *optional*):
+ Size of the dataset in bytes after post-processing, if any.
+ dataset_size (`int`, *optional*):
+ The combined size in bytes of the Arrow tables for all splits.
+ size_in_bytes (`int`, *optional*):
+ The combined size in bytes of all files associated with the dataset (downloaded files + Arrow files).
+ **config_kwargs (additional keyword arguments):
+ Keyword arguments to be passed to the [`BuilderConfig`] and used in the [`DatasetBuilder`].
+ """
+
+ # Set in the dataset builders
+ description: str = dataclasses.field(default_factory=str)
+ citation: str = dataclasses.field(default_factory=str)
+ homepage: str = dataclasses.field(default_factory=str)
+ license: str = dataclasses.field(default_factory=str)
+ features: Optional[Features] = None
+ post_processed: Optional[PostProcessedInfo] = None
+ supervised_keys: Optional[SupervisedKeysData] = None
+
+ # Set later by the builder
+ builder_name: Optional[str] = None
+ dataset_name: Optional[str] = None # for packaged builders, to be different from builder_name
+ config_name: Optional[str] = None
+ version: Optional[Union[str, Version]] = None
+ # Set later by `download_and_prepare`
+ splits: Optional[dict] = None
+ download_checksums: Optional[dict] = None
+ download_size: Optional[int] = None
+ post_processing_size: Optional[int] = None
+ dataset_size: Optional[int] = None
+ size_in_bytes: Optional[int] = None
+
+ _INCLUDED_INFO_IN_YAML: ClassVar[list[str]] = [
+ "config_name",
+ "download_size",
+ "dataset_size",
+ "features",
+ "splits",
+ ]
+
+ def __post_init__(self):
+ # Convert back to the correct classes when we reload from dict
+ if self.features is not None and not isinstance(self.features, Features):
+ self.features = Features.from_dict(self.features)
+ if self.post_processed is not None and not isinstance(self.post_processed, PostProcessedInfo):
+ self.post_processed = PostProcessedInfo.from_dict(self.post_processed)
+ if self.version is not None and not isinstance(self.version, Version):
+ if isinstance(self.version, str):
+ self.version = Version(self.version)
+ else:
+ self.version = Version.from_dict(self.version)
+ if self.splits is not None and not isinstance(self.splits, SplitDict):
+ self.splits = SplitDict.from_split_dict(self.splits)
+ if self.supervised_keys is not None and not isinstance(self.supervised_keys, SupervisedKeysData):
+ if isinstance(self.supervised_keys, (tuple, list)):
+ self.supervised_keys = SupervisedKeysData(*self.supervised_keys)
+ else:
+ self.supervised_keys = SupervisedKeysData(**self.supervised_keys)
+
+ def write_to_directory(self, dataset_info_dir, pretty_print=False, storage_options: Optional[dict] = None):
+ """Write `DatasetInfo` and license (if present) as JSON files to `dataset_info_dir`.
+
+ Args:
+ dataset_info_dir (`str`):
+ Destination directory.
+ pretty_print (`bool`, defaults to `False`):
+ If `True`, the JSON will be pretty-printed with the indent level of 4.
+ storage_options (`dict`, *optional*):
+ Key/value pairs to be passed on to the file-system backend, if any.
+
+
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> ds.info.write_to_directory("/path/to/directory/")
+ ```
+ """
+ fs: fsspec.AbstractFileSystem
+ fs, *_ = url_to_fs(dataset_info_dir, **(storage_options or {}))
+ with fs.open(posixpath.join(dataset_info_dir, config.DATASET_INFO_FILENAME), "wb") as f:
+ self._dump_info(f, pretty_print=pretty_print)
+ if self.license:
+ with fs.open(posixpath.join(dataset_info_dir, config.LICENSE_FILENAME), "wb") as f:
+ self._dump_license(f)
+
+ def _dump_info(self, file, pretty_print=False):
+ """Dump info in `file` file-like object open in bytes mode (to support remote files)"""
+ file.write(json.dumps(asdict(self), indent=4 if pretty_print else None).encode("utf-8"))
+
+ def _dump_license(self, file):
+ """Dump license in `file` file-like object open in bytes mode (to support remote files)"""
+ file.write(self.license.encode("utf-8"))
+
+ @classmethod
+ def from_merge(cls, dataset_infos: list["DatasetInfo"]):
+ dataset_infos = [dset_info.copy() for dset_info in dataset_infos if dset_info is not None]
+
+ if len(dataset_infos) > 0 and all(dataset_infos[0] == dset_info for dset_info in dataset_infos):
+ # if all dataset_infos are equal we don't need to merge. Just return the first.
+ return dataset_infos[0]
+
+ description = "\n\n".join(unique_values(info.description for info in dataset_infos)).strip()
+ citation = "\n\n".join(unique_values(info.citation for info in dataset_infos)).strip()
+ homepage = "\n\n".join(unique_values(info.homepage for info in dataset_infos)).strip()
+ license = "\n\n".join(unique_values(info.license for info in dataset_infos)).strip()
+ features = None
+ supervised_keys = None
+
+ return cls(
+ description=description,
+ citation=citation,
+ homepage=homepage,
+ license=license,
+ features=features,
+ supervised_keys=supervised_keys,
+ )
+
+ @classmethod
+ def from_directory(cls, dataset_info_dir: str, storage_options: Optional[dict] = None) -> "DatasetInfo":
+ """Create [`DatasetInfo`] from the JSON file in `dataset_info_dir`.
+
+ This function updates all the dynamically generated fields (num_examples,
+ hash, time of creation,...) of the [`DatasetInfo`].
+
+ This will overwrite all previous metadata.
+
+ Args:
+ dataset_info_dir (`str`):
+ The directory containing the metadata file. This
+ should be the root directory of a specific dataset version.
+ storage_options (`dict`, *optional*):
+ Key/value pairs to be passed on to the file-system backend, if any.
+
+
+
+ Example:
+
+ ```py
+ >>> from datasets import DatasetInfo
+ >>> ds_info = DatasetInfo.from_directory("/path/to/directory/")
+ ```
+ """
+ fs: fsspec.AbstractFileSystem
+ fs, *_ = url_to_fs(dataset_info_dir, **(storage_options or {}))
+ logger.debug(f"Loading Dataset info from {dataset_info_dir}")
+ if not dataset_info_dir:
+ raise ValueError("Calling DatasetInfo.from_directory() with undefined dataset_info_dir.")
+ with fs.open(posixpath.join(dataset_info_dir, config.DATASET_INFO_FILENAME), "r", encoding="utf-8") as f:
+ dataset_info_dict = json.load(f)
+ return cls.from_dict(dataset_info_dict)
+
+ @classmethod
+ def from_dict(cls, dataset_info_dict: dict) -> "DatasetInfo":
+ field_names = {f.name for f in dataclasses.fields(cls)}
+ return cls(**{k: v for k, v in dataset_info_dict.items() if k in field_names})
+
+ def update(self, other_dataset_info: "DatasetInfo", ignore_none=True):
+ self_dict = self.__dict__
+ self_dict.update(
+ **{
+ k: copy.deepcopy(v)
+ for k, v in other_dataset_info.__dict__.items()
+ if (v is not None or not ignore_none)
+ }
+ )
+
+ def copy(self) -> "DatasetInfo":
+ return self.__class__(**{k: copy.deepcopy(v) for k, v in self.__dict__.items()})
+
+ def _to_yaml_dict(self) -> dict:
+ yaml_dict = {}
+ dataset_info_dict = asdict(self)
+ for key in dataset_info_dict:
+ if key in self._INCLUDED_INFO_IN_YAML:
+ value = getattr(self, key)
+ if hasattr(value, "_to_yaml_list"): # Features, SplitDict
+ yaml_dict[key] = value._to_yaml_list()
+ elif hasattr(value, "_to_yaml_string"): # Version
+ yaml_dict[key] = value._to_yaml_string()
+ else:
+ yaml_dict[key] = value
+ return yaml_dict
+
+ @classmethod
+ def _from_yaml_dict(cls, yaml_data: dict) -> "DatasetInfo":
+ yaml_data = copy.deepcopy(yaml_data)
+ if yaml_data.get("features") is not None:
+ yaml_data["features"] = Features._from_yaml_list(yaml_data["features"])
+ if yaml_data.get("splits") is not None:
+ yaml_data["splits"] = SplitDict._from_yaml_list(yaml_data["splits"])
+ field_names = {f.name for f in dataclasses.fields(cls)}
+ return cls(**{k: v for k, v in yaml_data.items() if k in field_names})
+
+
+class DatasetInfosDict(dict[str, DatasetInfo]):
+ def write_to_directory(self, dataset_infos_dir, overwrite=False, pretty_print=False) -> None:
+ total_dataset_infos = {}
+ dataset_infos_path = os.path.join(dataset_infos_dir, config.DATASETDICT_INFOS_FILENAME)
+ dataset_readme_path = os.path.join(dataset_infos_dir, config.REPOCARD_FILENAME)
+ if not overwrite:
+ total_dataset_infos = self.from_directory(dataset_infos_dir)
+ total_dataset_infos.update(self)
+ if os.path.exists(dataset_infos_path):
+ # for backward compatibility, let's update the JSON file if it exists
+ with open(dataset_infos_path, "w", encoding="utf-8") as f:
+ dataset_infos_dict = {
+ config_name: asdict(dset_info) for config_name, dset_info in total_dataset_infos.items()
+ }
+ json.dump(dataset_infos_dict, f, indent=4 if pretty_print else None)
+ # Dump the infos in the YAML part of the README.md file
+ if os.path.exists(dataset_readme_path):
+ dataset_card = DatasetCard.load(dataset_readme_path)
+ dataset_card_data = dataset_card.data
+ else:
+ dataset_card = None
+ dataset_card_data = DatasetCardData()
+ if total_dataset_infos:
+ total_dataset_infos.to_dataset_card_data(dataset_card_data)
+ dataset_card = (
+ DatasetCard("---\n" + str(dataset_card_data) + "\n---\n") if dataset_card is None else dataset_card
+ )
+ dataset_card.save(Path(dataset_readme_path))
+
+ @classmethod
+ def from_directory(cls, dataset_infos_dir) -> "DatasetInfosDict":
+ logger.debug(f"Loading Dataset Infos from {dataset_infos_dir}")
+ # Load the info from the YAML part of README.md
+ if os.path.exists(os.path.join(dataset_infos_dir, config.REPOCARD_FILENAME)):
+ dataset_card_data = DatasetCard.load(Path(dataset_infos_dir) / config.REPOCARD_FILENAME).data
+ if "dataset_info" in dataset_card_data:
+ return cls.from_dataset_card_data(dataset_card_data)
+ if os.path.exists(os.path.join(dataset_infos_dir, config.DATASETDICT_INFOS_FILENAME)):
+ # this is just to have backward compatibility with dataset_infos.json files
+ with open(os.path.join(dataset_infos_dir, config.DATASETDICT_INFOS_FILENAME), encoding="utf-8") as f:
+ return cls(
+ {
+ config_name: DatasetInfo.from_dict(dataset_info_dict)
+ for config_name, dataset_info_dict in json.load(f).items()
+ }
+ )
+ else:
+ return cls()
+
+ @classmethod
+ def from_dataset_card_data(cls, dataset_card_data: DatasetCardData) -> "DatasetInfosDict":
+ if isinstance(dataset_card_data.get("dataset_info"), (list, dict)):
+ if isinstance(dataset_card_data["dataset_info"], list):
+ return cls(
+ {
+ dataset_info_yaml_dict.get("config_name", "default"): DatasetInfo._from_yaml_dict(
+ dataset_info_yaml_dict
+ )
+ for dataset_info_yaml_dict in dataset_card_data["dataset_info"]
+ }
+ )
+ else:
+ dataset_info = DatasetInfo._from_yaml_dict(dataset_card_data["dataset_info"])
+ dataset_info.config_name = dataset_card_data["dataset_info"].get("config_name", "default")
+ return cls({dataset_info.config_name: dataset_info})
+ else:
+ return cls()
+
+ def to_dataset_card_data(self, dataset_card_data: DatasetCardData) -> None:
+ if self:
+ # first get existing metadata info
+ if "dataset_info" in dataset_card_data and isinstance(dataset_card_data["dataset_info"], dict):
+ dataset_metadata_infos = {
+ dataset_card_data["dataset_info"].get("config_name", "default"): dataset_card_data["dataset_info"]
+ }
+ elif "dataset_info" in dataset_card_data and isinstance(dataset_card_data["dataset_info"], list):
+ dataset_metadata_infos = {
+ config_metadata["config_name"]: config_metadata
+ for config_metadata in dataset_card_data["dataset_info"]
+ }
+ else:
+ dataset_metadata_infos = {}
+ # update/rewrite existing metadata info with the one to dump
+ total_dataset_infos = {
+ **dataset_metadata_infos,
+ **{config_name: dset_info._to_yaml_dict() for config_name, dset_info in self.items()},
+ }
+ # the config_name from the dataset_infos_dict takes over the config_name of the DatasetInfo
+ for config_name, dset_info_yaml_dict in total_dataset_infos.items():
+ dset_info_yaml_dict["config_name"] = config_name
+ if len(total_dataset_infos) == 1:
+ # use a struct instead of a list of configurations, since there's only one
+ dataset_card_data["dataset_info"] = next(iter(total_dataset_infos.values()))
+ config_name = dataset_card_data["dataset_info"].pop("config_name", None)
+ if config_name != "default":
+ # if config_name is not "default" preserve it and put at the first position
+ dataset_card_data["dataset_info"] = {
+ "config_name": config_name,
+ **dataset_card_data["dataset_info"],
+ }
+ else:
+ dataset_card_data["dataset_info"] = []
+ for config_name, dataset_info_yaml_dict in sorted(total_dataset_infos.items()):
+ # add the config_name field in first position
+ dataset_info_yaml_dict.pop("config_name", None)
+ dataset_info_yaml_dict = {"config_name": config_name, **dataset_info_yaml_dict}
+ dataset_card_data["dataset_info"].append(dataset_info_yaml_dict)
diff --git a/venv/lib/python3.10/site-packages/datasets/inspect.py b/venv/lib/python3.10/site-packages/datasets/inspect.py
new file mode 100644
index 0000000000000000000000000000000000000000..81022eadc678df28be247b77b925f6f05b9f6a0c
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/inspect.py
@@ -0,0 +1,353 @@
+# Copyright 2020 The HuggingFace Datasets Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""List and inspect datasets."""
+
+import os
+from collections.abc import Mapping, Sequence
+from typing import Optional, Union
+
+from .download.download_config import DownloadConfig
+from .download.download_manager import DownloadMode
+from .download.streaming_download_manager import StreamingDownloadManager
+from .info import DatasetInfo
+from .load import (
+ dataset_module_factory,
+ get_dataset_builder_class,
+ load_dataset_builder,
+)
+from .utils.logging import get_logger
+from .utils.version import Version
+
+
+logger = get_logger(__name__)
+
+
+class SplitsNotFoundError(ValueError):
+ pass
+
+
+def get_dataset_infos(
+ path: str,
+ data_files: Optional[Union[dict, list, str]] = None,
+ download_config: Optional[DownloadConfig] = None,
+ download_mode: Optional[Union[DownloadMode, str]] = None,
+ revision: Optional[Union[str, Version]] = None,
+ token: Optional[Union[bool, str]] = None,
+ **config_kwargs,
+):
+ """Get the meta information about a dataset, returned as a dict mapping config name to DatasetInfoDict.
+
+ Args:
+ path (`str`): path to the dataset repository. Can be either:
+
+ - a local path to the dataset directory containing the data files,
+ e.g. `'./dataset/squad'`
+ - a dataset identifier on the Hugging Face Hub (list all available datasets and ids with [`huggingface_hub.list_datasets`]),
+ e.g. `'rajpurkar/squad'`, `'nyu-mll/glue'` or``'openai/webtext'`
+ revision (`Union[str, datasets.Version]`, *optional*):
+ If specified, the dataset module will be loaded from the datasets repository at this version.
+ By default:
+ - it is set to the local version of the lib.
+ - it will also try to load it from the main branch if it's not available at the local version of the lib.
+ Specifying a version that is different from your local version of the lib might cause compatibility issues.
+ download_config ([`DownloadConfig`], *optional*):
+ Specific download configuration parameters.
+ download_mode ([`DownloadMode`] or `str`, defaults to `REUSE_DATASET_IF_EXISTS`):
+ Download/generate mode.
+ data_files (`Union[Dict, List, str]`, *optional*):
+ Defining the data_files of the dataset configuration.
+ token (`str` or `bool`, *optional*):
+ Optional string or boolean to use as Bearer token for remote files on the Datasets Hub.
+ If `True`, or not specified, will get token from `"~/.huggingface"`.
+ **config_kwargs (additional keyword arguments):
+ Optional attributes for builder class which will override the attributes if supplied.
+
+ Example:
+
+ ```py
+ >>> from datasets import get_dataset_infos
+ >>> get_dataset_infos('cornell-movie-review-data/rotten_tomatoes')
+ {'default': DatasetInfo(description="Movie Review Dataset.\nThis is a dataset of containing 5,331 positive and 5,331 negative processed\nsentences from Rotten Tomatoes movie reviews...), ...}
+ ```
+ """
+ config_names = get_dataset_config_names(
+ path=path,
+ revision=revision,
+ download_config=download_config,
+ download_mode=download_mode,
+ data_files=data_files,
+ token=token,
+ )
+ return {
+ config_name: get_dataset_config_info(
+ path=path,
+ config_name=config_name,
+ data_files=data_files,
+ download_config=download_config,
+ download_mode=download_mode,
+ revision=revision,
+ token=token,
+ **config_kwargs,
+ )
+ for config_name in config_names
+ }
+
+
+def get_dataset_config_names(
+ path: str,
+ revision: Optional[Union[str, Version]] = None,
+ download_config: Optional[DownloadConfig] = None,
+ download_mode: Optional[Union[DownloadMode, str]] = None,
+ data_files: Optional[Union[dict, list, str]] = None,
+ **download_kwargs,
+):
+ """Get the list of available config names for a particular dataset.
+
+ Args:
+ path (`str`): path to the dataset repository. Can be either:
+
+ - a local path to the dataset directory containing the data files,
+ e.g. `'./dataset/squad'`
+ - a dataset identifier on the Hugging Face Hub (list all available datasets and ids with [`huggingface_hub.list_datasets`]),
+ e.g. `'rajpurkar/squad'`, `'nyu-mll/glue'` or``'openai/webtext'`
+ revision (`Union[str, datasets.Version]`, *optional*):
+ If specified, the dataset module will be loaded from the datasets repository at this version.
+ By default:
+ - it is set to the local version of the lib.
+ - it will also try to load it from the main branch if it's not available at the local version of the lib.
+ Specifying a version that is different from your local version of the lib might cause compatibility issues.
+ download_config ([`DownloadConfig`], *optional*):
+ Specific download configuration parameters.
+ download_mode ([`DownloadMode`] or `str`, defaults to `REUSE_DATASET_IF_EXISTS`):
+ Download/generate mode.
+ data_files (`Union[Dict, List, str]`, *optional*):
+ Defining the data_files of the dataset configuration.
+ **download_kwargs (additional keyword arguments):
+ Optional attributes for [`DownloadConfig`] which will override the attributes in `download_config` if supplied,
+ for example `token`.
+
+ Example:
+
+ ```py
+ >>> from datasets import get_dataset_config_names
+ >>> get_dataset_config_names("nyu-mll/glue")
+ ['cola',
+ 'sst2',
+ 'mrpc',
+ 'qqp',
+ 'stsb',
+ 'mnli',
+ 'mnli_mismatched',
+ 'mnli_matched',
+ 'qnli',
+ 'rte',
+ 'wnli',
+ 'ax']
+ ```
+ """
+ dataset_module = dataset_module_factory(
+ path,
+ revision=revision,
+ download_config=download_config,
+ download_mode=download_mode,
+ data_files=data_files,
+ **download_kwargs,
+ )
+ builder_cls = get_dataset_builder_class(dataset_module, dataset_name=os.path.basename(path))
+ return list(builder_cls.builder_configs.keys()) or [
+ dataset_module.builder_kwargs.get("config_name", builder_cls.DEFAULT_CONFIG_NAME or "default")
+ ]
+
+
+def get_dataset_default_config_name(
+ path: str,
+ revision: Optional[Union[str, Version]] = None,
+ download_config: Optional[DownloadConfig] = None,
+ download_mode: Optional[Union[DownloadMode, str]] = None,
+ data_files: Optional[Union[dict, list, str]] = None,
+ **download_kwargs,
+) -> Optional[str]:
+ """Get the default config name for a particular dataset.
+ Can return None only if the dataset has multiple configurations and no default configuration.
+
+ Args:
+ path (`str`): path to the dataset repository. Can be either:
+
+ - a local path to the dataset directory containing the data files,
+ e.g. `'./dataset/squad'`
+ - a dataset identifier on the Hugging Face Hub (list all available datasets and ids with [`huggingface_hub.list_datasets`]),
+ e.g. `'rajpurkar/squad'`, `'nyu-mll/glue'` or``'openai/webtext'`
+ revision (`Union[str, datasets.Version]`, *optional*):
+ If specified, the dataset module will be loaded from the datasets repository at this version.
+ By default:
+ - it is set to the local version of the lib.
+ - it will also try to load it from the main branch if it's not available at the local version of the lib.
+ Specifying a version that is different from your local version of the lib might cause compatibility issues.
+ download_config ([`DownloadConfig`], *optional*):
+ Specific download configuration parameters.
+ download_mode ([`DownloadMode`] or `str`, defaults to `REUSE_DATASET_IF_EXISTS`):
+ Download/generate mode.
+ data_files (`Union[Dict, List, str]`, *optional*):
+ Defining the data_files of the dataset configuration.
+ **download_kwargs (additional keyword arguments):
+ Optional attributes for [`DownloadConfig`] which will override the attributes in `download_config` if supplied,
+ for example `token`.
+
+ Returns:
+ Optional[str]: the default config name if there is one
+
+ Example:
+
+ ```py
+ >>> from datasets import get_dataset_default_config_name
+ >>> get_dataset_default_config_name("openbookqa")
+ 'main'
+ ```
+ """
+ dataset_module = dataset_module_factory(
+ path,
+ revision=revision,
+ download_config=download_config,
+ download_mode=download_mode,
+ data_files=data_files,
+ **download_kwargs,
+ )
+ builder_cls = get_dataset_builder_class(dataset_module, dataset_name=os.path.basename(path))
+ builder_configs = list(builder_cls.builder_configs.keys())
+ if builder_configs:
+ default_config_name = builder_configs[0] if len(builder_configs) == 1 else None
+ else:
+ default_config_name = "default"
+ return builder_cls.DEFAULT_CONFIG_NAME or default_config_name
+
+
+def get_dataset_config_info(
+ path: str,
+ config_name: Optional[str] = None,
+ data_files: Optional[Union[str, Sequence[str], Mapping[str, Union[str, Sequence[str]]]]] = None,
+ download_config: Optional[DownloadConfig] = None,
+ download_mode: Optional[Union[DownloadMode, str]] = None,
+ revision: Optional[Union[str, Version]] = None,
+ token: Optional[Union[bool, str]] = None,
+ **config_kwargs,
+) -> DatasetInfo:
+ """Get the meta information (DatasetInfo) about a dataset for a particular config
+
+ Args:
+ path (`str`): path to the dataset repository. Can be either:
+
+ - a local path to the dataset directory containing the data files,
+ e.g. `'./dataset/squad'`
+ - a dataset identifier on the Hugging Face Hub (list all available datasets and ids with [`huggingface_hub.list_datasets`]),
+ e.g. `'rajpurkar/squad'`, `'nyu-mll/glue'` or``'openai/webtext'`
+ config_name (:obj:`str`, optional): Defining the name of the dataset configuration.
+ data_files (:obj:`str` or :obj:`Sequence` or :obj:`Mapping`, optional): Path(s) to source data file(s).
+ download_config (:class:`~download.DownloadConfig`, optional): Specific download configuration parameters.
+ download_mode (:class:`DownloadMode` or :obj:`str`, default ``REUSE_DATASET_IF_EXISTS``): Download/generate mode.
+ revision (:class:`~utils.Version` or :obj:`str`, optional): Version of the dataset to load.
+ As datasets have their own git repository on the Datasets Hub, the default version "main" corresponds to their "main" branch.
+ You can specify a different version than the default "main" by using a commit SHA or a git tag of the dataset repository.
+ token (``str`` or :obj:`bool`, optional): Optional string or boolean to use as Bearer token for remote files on the Datasets Hub.
+ If True, or not specified, will get token from `"~/.huggingface"`.
+ **config_kwargs (additional keyword arguments): optional attributes for builder class which will override the attributes if supplied.
+
+ """
+ builder = load_dataset_builder(
+ path,
+ name=config_name,
+ data_files=data_files,
+ download_config=download_config,
+ download_mode=download_mode,
+ revision=revision,
+ token=token,
+ **config_kwargs,
+ )
+ info = builder.info
+ if info.splits is None:
+ download_config = download_config.copy() if download_config else DownloadConfig()
+ if token is not None:
+ download_config.token = token
+ builder._check_manual_download(
+ StreamingDownloadManager(base_path=builder.base_path, download_config=download_config)
+ )
+ try:
+ info.splits = {
+ split_generator.name: {"name": split_generator.name, "dataset_name": path}
+ for split_generator in builder._split_generators(
+ StreamingDownloadManager(base_path=builder.base_path, download_config=download_config)
+ )
+ }
+ except Exception as err:
+ raise SplitsNotFoundError("The split names could not be parsed from the dataset config.") from err
+ return info
+
+
+def get_dataset_split_names(
+ path: str,
+ config_name: Optional[str] = None,
+ data_files: Optional[Union[str, Sequence[str], Mapping[str, Union[str, Sequence[str]]]]] = None,
+ download_config: Optional[DownloadConfig] = None,
+ download_mode: Optional[Union[DownloadMode, str]] = None,
+ revision: Optional[Union[str, Version]] = None,
+ token: Optional[Union[bool, str]] = None,
+ **config_kwargs,
+):
+ """Get the list of available splits for a particular config and dataset.
+
+ Args:
+ path (`str`): path to the dataset repository. Can be either:
+
+ - a local path to the dataset directory containing the data files,
+ e.g. `'./dataset/squad'`
+ - a dataset identifier on the Hugging Face Hub (list all available datasets and ids with [`huggingface_hub.list_datasets`]),
+ e.g. `'rajpurkar/squad'`, `'nyu-mll/glue'` or``'openai/webtext'`
+ config_name (`str`, *optional*):
+ Defining the name of the dataset configuration.
+ data_files (`str` or `Sequence` or `Mapping`, *optional*):
+ Path(s) to source data file(s).
+ download_config ([`DownloadConfig`], *optional*):
+ Specific download configuration parameters.
+ download_mode ([`DownloadMode`] or `str`, defaults to `REUSE_DATASET_IF_EXISTS`):
+ Download/generate mode.
+ revision ([`Version`] or `str`, *optional*):
+ Version of the dataset to load.
+ As datasets have their own git repository on the Datasets Hub, the default version "main" corresponds to their "main" branch.
+ You can specify a different version than the default "main" by using a commit SHA or a git tag of the dataset repository.
+ token (`str` or `bool`, *optional*):
+ Optional string or boolean to use as Bearer token for remote files on the Datasets Hub.
+ If `True`, or not specified, will get token from `"~/.huggingface"`.
+ **config_kwargs (additional keyword arguments):
+ Optional attributes for builder class which will override the attributes if supplied.
+
+ Example:
+
+ ```py
+ >>> from datasets import get_dataset_split_names
+ >>> get_dataset_split_names('cornell-movie-review-data/rotten_tomatoes')
+ ['train', 'validation', 'test']
+ ```
+ """
+ info = get_dataset_config_info(
+ path,
+ config_name=config_name,
+ data_files=data_files,
+ download_config=download_config,
+ download_mode=download_mode,
+ revision=revision,
+ token=token,
+ **config_kwargs,
+ )
+ return list(info.splits.keys())
diff --git a/venv/lib/python3.10/site-packages/datasets/io/__init__.py b/venv/lib/python3.10/site-packages/datasets/io/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/venv/lib/python3.10/site-packages/datasets/io/__pycache__/__init__.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/io/__pycache__/__init__.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..baff3b7e808cf3a7af6ef6aa8b2c8e735f51be68
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/io/__pycache__/__init__.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/io/__pycache__/abc.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/io/__pycache__/abc.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..f996a9e3a1929bf283d1af99908e6be8feaa0bb2
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/io/__pycache__/abc.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/io/__pycache__/csv.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/io/__pycache__/csv.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..00895738a090157996f0d1da112903c1eab03709
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/io/__pycache__/csv.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/io/__pycache__/generator.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/io/__pycache__/generator.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..793390c9a489bd49bb7301625676370c1da831ad
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/io/__pycache__/generator.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/io/__pycache__/json.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/io/__pycache__/json.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..0d03b2620b59f32ea6fb11ad95138d0982af1c77
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/io/__pycache__/json.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/io/__pycache__/parquet.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/io/__pycache__/parquet.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..81fed81fffe05d7d19b44a12f5f930af5db93582
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/io/__pycache__/parquet.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/io/__pycache__/spark.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/io/__pycache__/spark.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..d3b43a55dde313f47d1b501959d5f27000865ed5
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/io/__pycache__/spark.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/io/__pycache__/sql.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/io/__pycache__/sql.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..d27cfeb233dfc3e6aca0a9ac748b9b6714328cfc
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/io/__pycache__/sql.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/io/__pycache__/text.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/io/__pycache__/text.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..839659316b8d11663f15ce0ff1c47ea44f61ab3f
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/io/__pycache__/text.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/io/abc.py b/venv/lib/python3.10/site-packages/datasets/io/abc.py
new file mode 100644
index 0000000000000000000000000000000000000000..a1913cc20e3fd748ef912e2fb3d7c1e18f16ac8c
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/io/abc.py
@@ -0,0 +1,53 @@
+from abc import ABC, abstractmethod
+from typing import Optional, Union
+
+from .. import Dataset, DatasetDict, Features, IterableDataset, IterableDatasetDict, NamedSplit
+from ..utils.typing import NestedDataStructureLike, PathLike
+
+
+class AbstractDatasetReader(ABC):
+ def __init__(
+ self,
+ path_or_paths: Optional[NestedDataStructureLike[PathLike]] = None,
+ split: Optional[NamedSplit] = None,
+ features: Optional[Features] = None,
+ cache_dir: str = None,
+ keep_in_memory: bool = False,
+ streaming: bool = False,
+ num_proc: Optional[int] = None,
+ **kwargs,
+ ):
+ self.path_or_paths = path_or_paths
+ self.split = split if split or isinstance(path_or_paths, dict) else "train"
+ self.features = features
+ self.cache_dir = cache_dir
+ self.keep_in_memory = keep_in_memory
+ self.streaming = streaming
+ self.num_proc = num_proc
+ self.kwargs = kwargs
+
+ @abstractmethod
+ def read(self) -> Union[Dataset, DatasetDict, IterableDataset, IterableDatasetDict]:
+ pass
+
+
+class AbstractDatasetInputStream(ABC):
+ def __init__(
+ self,
+ features: Optional[Features] = None,
+ cache_dir: str = None,
+ keep_in_memory: bool = False,
+ streaming: bool = False,
+ num_proc: Optional[int] = None,
+ **kwargs,
+ ):
+ self.features = features
+ self.cache_dir = cache_dir
+ self.keep_in_memory = keep_in_memory
+ self.streaming = streaming
+ self.num_proc = num_proc
+ self.kwargs = kwargs
+
+ @abstractmethod
+ def read(self) -> Union[Dataset, IterableDataset]:
+ pass
diff --git a/venv/lib/python3.10/site-packages/datasets/io/csv.py b/venv/lib/python3.10/site-packages/datasets/io/csv.py
new file mode 100644
index 0000000000000000000000000000000000000000..4ac2ea1135b579ae0f38f7dafc73ae5c2db1a6b6
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/io/csv.py
@@ -0,0 +1,145 @@
+import multiprocessing
+import os
+from typing import BinaryIO, Optional, Union
+
+import fsspec
+
+from .. import Dataset, Features, NamedSplit, config
+from ..formatting import query_table
+from ..packaged_modules.csv.csv import Csv
+from ..utils import tqdm as hf_tqdm
+from ..utils.typing import NestedDataStructureLike, PathLike
+from .abc import AbstractDatasetReader
+
+
+class CsvDatasetReader(AbstractDatasetReader):
+ def __init__(
+ self,
+ path_or_paths: NestedDataStructureLike[PathLike],
+ split: Optional[NamedSplit] = None,
+ features: Optional[Features] = None,
+ cache_dir: str = None,
+ keep_in_memory: bool = False,
+ streaming: bool = False,
+ num_proc: Optional[int] = None,
+ **kwargs,
+ ):
+ super().__init__(
+ path_or_paths,
+ split=split,
+ features=features,
+ cache_dir=cache_dir,
+ keep_in_memory=keep_in_memory,
+ streaming=streaming,
+ num_proc=num_proc,
+ **kwargs,
+ )
+ path_or_paths = path_or_paths if isinstance(path_or_paths, dict) else {self.split: path_or_paths}
+ self.builder = Csv(
+ cache_dir=cache_dir,
+ data_files=path_or_paths,
+ features=features,
+ **kwargs,
+ )
+
+ def read(self):
+ # Build iterable dataset
+ if self.streaming:
+ dataset = self.builder.as_streaming_dataset(split=self.split)
+ # Build regular (map-style) dataset
+ else:
+ download_config = None
+ download_mode = None
+ verification_mode = None
+ base_path = None
+
+ self.builder.download_and_prepare(
+ download_config=download_config,
+ download_mode=download_mode,
+ verification_mode=verification_mode,
+ base_path=base_path,
+ num_proc=self.num_proc,
+ )
+ dataset = self.builder.as_dataset(
+ split=self.split, verification_mode=verification_mode, in_memory=self.keep_in_memory
+ )
+ return dataset
+
+
+class CsvDatasetWriter:
+ def __init__(
+ self,
+ dataset: Dataset,
+ path_or_buf: Union[PathLike, BinaryIO],
+ batch_size: Optional[int] = None,
+ num_proc: Optional[int] = None,
+ storage_options: Optional[dict] = None,
+ **to_csv_kwargs,
+ ):
+ if num_proc is not None and num_proc <= 0:
+ raise ValueError(f"num_proc {num_proc} must be an integer > 0.")
+
+ self.dataset = dataset
+ self.path_or_buf = path_or_buf
+ self.batch_size = batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
+ self.num_proc = num_proc
+ self.encoding = "utf-8"
+ self.storage_options = storage_options or {}
+ self.to_csv_kwargs = to_csv_kwargs
+
+ def write(self) -> int:
+ _ = self.to_csv_kwargs.pop("path_or_buf", None)
+ header = self.to_csv_kwargs.pop("header", True)
+ index = self.to_csv_kwargs.pop("index", False)
+
+ if isinstance(self.path_or_buf, (str, bytes, os.PathLike)):
+ with fsspec.open(self.path_or_buf, "wb", **(self.storage_options or {})) as buffer:
+ written = self._write(file_obj=buffer, header=header, index=index, **self.to_csv_kwargs)
+ else:
+ written = self._write(file_obj=self.path_or_buf, header=header, index=index, **self.to_csv_kwargs)
+ return written
+
+ def _batch_csv(self, args):
+ offset, header, index, to_csv_kwargs = args
+
+ batch = query_table(
+ table=self.dataset.data,
+ key=slice(offset, offset + self.batch_size),
+ indices=self.dataset._indices,
+ )
+ csv_str = batch.to_pandas().to_csv(
+ path_or_buf=None, header=header if (offset == 0) else False, index=index, **to_csv_kwargs
+ )
+ return csv_str.encode(self.encoding)
+
+ def _write(self, file_obj: BinaryIO, header, index, **to_csv_kwargs) -> int:
+ """Writes the pyarrow table as CSV to a binary file handle.
+
+ Caller is responsible for opening and closing the handle.
+ """
+ written = 0
+
+ if self.num_proc is None or self.num_proc == 1:
+ for offset in hf_tqdm(
+ range(0, len(self.dataset), self.batch_size),
+ unit="ba",
+ desc="Creating CSV from Arrow format",
+ ):
+ csv_str = self._batch_csv((offset, header, index, to_csv_kwargs))
+ written += file_obj.write(csv_str)
+
+ else:
+ num_rows, batch_size = len(self.dataset), self.batch_size
+ with multiprocessing.Pool(self.num_proc) as pool:
+ for csv_str in hf_tqdm(
+ pool.imap(
+ self._batch_csv,
+ [(offset, header, index, to_csv_kwargs) for offset in range(0, num_rows, batch_size)],
+ ),
+ total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size,
+ unit="ba",
+ desc="Creating CSV from Arrow format",
+ ):
+ written += file_obj.write(csv_str)
+
+ return written
diff --git a/venv/lib/python3.10/site-packages/datasets/io/generator.py b/venv/lib/python3.10/site-packages/datasets/io/generator.py
new file mode 100644
index 0000000000000000000000000000000000000000..6c1eaee9b0f70e9e15dde7377eba885a32e999e7
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/io/generator.py
@@ -0,0 +1,64 @@
+from typing import Callable, Optional
+
+from .. import Features, NamedSplit, Split
+from ..packaged_modules.generator.generator import Generator
+from .abc import AbstractDatasetInputStream
+
+
+class GeneratorDatasetInputStream(AbstractDatasetInputStream):
+ def __init__(
+ self,
+ generator: Callable,
+ features: Optional[Features] = None,
+ cache_dir: str = None,
+ keep_in_memory: bool = False,
+ streaming: bool = False,
+ gen_kwargs: Optional[dict] = None,
+ num_proc: Optional[int] = None,
+ split: NamedSplit = Split.TRAIN,
+ fingerprint: Optional[str] = None,
+ **kwargs,
+ ):
+ super().__init__(
+ features=features,
+ cache_dir=cache_dir,
+ keep_in_memory=keep_in_memory,
+ streaming=streaming,
+ num_proc=num_proc,
+ **kwargs,
+ )
+ self.builder = Generator(
+ cache_dir=cache_dir,
+ features=features,
+ generator=generator,
+ gen_kwargs=gen_kwargs,
+ split=split,
+ config_id="default-fingerprint=" + fingerprint if fingerprint else None,
+ **kwargs,
+ )
+ self.fingerprint = fingerprint
+
+ def read(self):
+ # Build iterable dataset
+ if self.streaming:
+ dataset = self.builder.as_streaming_dataset(split=self.builder.config.split)
+ # Build regular (map-style) dataset
+ else:
+ download_config = None
+ download_mode = None
+ verification_mode = None
+ base_path = None
+
+ self.builder.download_and_prepare(
+ download_config=download_config,
+ download_mode=download_mode,
+ verification_mode=verification_mode,
+ base_path=base_path,
+ num_proc=self.num_proc,
+ )
+ dataset = self.builder.as_dataset(
+ split=self.builder.config.split, verification_mode=verification_mode, in_memory=self.keep_in_memory
+ )
+ if self.fingerprint:
+ dataset._fingerprint = self.fingerprint
+ return dataset
diff --git a/venv/lib/python3.10/site-packages/datasets/io/json.py b/venv/lib/python3.10/site-packages/datasets/io/json.py
new file mode 100644
index 0000000000000000000000000000000000000000..41abfa518ccb7b46c446b98eb249f4c95ed9a731
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/io/json.py
@@ -0,0 +1,174 @@
+import multiprocessing
+import os
+from typing import BinaryIO, Optional, Union
+
+import fsspec
+
+from .. import Dataset, Features, NamedSplit, config
+from ..formatting import query_table
+from ..packaged_modules.json.json import Json
+from ..utils import tqdm as hf_tqdm
+from ..utils.typing import NestedDataStructureLike, PathLike
+from .abc import AbstractDatasetReader
+
+
+class JsonDatasetReader(AbstractDatasetReader):
+ def __init__(
+ self,
+ path_or_paths: NestedDataStructureLike[PathLike],
+ split: Optional[NamedSplit] = None,
+ features: Optional[Features] = None,
+ cache_dir: str = None,
+ keep_in_memory: bool = False,
+ streaming: bool = False,
+ field: Optional[str] = None,
+ num_proc: Optional[int] = None,
+ **kwargs,
+ ):
+ super().__init__(
+ path_or_paths,
+ split=split,
+ features=features,
+ cache_dir=cache_dir,
+ keep_in_memory=keep_in_memory,
+ streaming=streaming,
+ num_proc=num_proc,
+ **kwargs,
+ )
+ self.field = field
+ path_or_paths = path_or_paths if isinstance(path_or_paths, dict) else {self.split: path_or_paths}
+ self.builder = Json(
+ cache_dir=cache_dir,
+ data_files=path_or_paths,
+ features=features,
+ field=field,
+ **kwargs,
+ )
+
+ def read(self):
+ # Build iterable dataset
+ if self.streaming:
+ dataset = self.builder.as_streaming_dataset(split=self.split)
+ # Build regular (map-style) dataset
+ else:
+ download_config = None
+ download_mode = None
+ verification_mode = None
+ base_path = None
+
+ self.builder.download_and_prepare(
+ download_config=download_config,
+ download_mode=download_mode,
+ verification_mode=verification_mode,
+ base_path=base_path,
+ num_proc=self.num_proc,
+ )
+ dataset = self.builder.as_dataset(
+ split=self.split, verification_mode=verification_mode, in_memory=self.keep_in_memory
+ )
+ return dataset
+
+
+class JsonDatasetWriter:
+ def __init__(
+ self,
+ dataset: Dataset,
+ path_or_buf: Union[PathLike, BinaryIO],
+ batch_size: Optional[int] = None,
+ num_proc: Optional[int] = None,
+ storage_options: Optional[dict] = None,
+ **to_json_kwargs,
+ ):
+ if num_proc is not None and num_proc <= 0:
+ raise ValueError(f"num_proc {num_proc} must be an integer > 0.")
+
+ self.dataset = dataset
+ self.path_or_buf = path_or_buf
+ self.batch_size = batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
+ self.num_proc = num_proc
+ self.encoding = "utf-8"
+ self.storage_options = storage_options or {}
+ self.to_json_kwargs = to_json_kwargs
+
+ def write(self) -> int:
+ _ = self.to_json_kwargs.pop("path_or_buf", None)
+ orient = self.to_json_kwargs.pop("orient", "records")
+ lines = self.to_json_kwargs.pop("lines", True if orient == "records" else False)
+ if "index" not in self.to_json_kwargs and orient in ["split", "table"]:
+ self.to_json_kwargs["index"] = False
+
+ # Determine the default compression value based on self.path_or_buf type
+ default_compression = "infer" if isinstance(self.path_or_buf, (str, bytes, os.PathLike)) else None
+ compression = self.to_json_kwargs.pop("compression", default_compression)
+
+ if compression not in [None, "infer", "gzip", "bz2", "xz"]:
+ raise NotImplementedError(f"`datasets` currently does not support {compression} compression")
+
+ if not lines and self.batch_size < self.dataset.num_rows:
+ raise NotImplementedError(
+ "Output JSON will not be formatted correctly when lines = False and batch_size < number of rows in the dataset. Use pandas.DataFrame.to_json() instead."
+ )
+
+ if isinstance(self.path_or_buf, (str, bytes, os.PathLike)):
+ with fsspec.open(
+ self.path_or_buf, "wb", compression=compression, **(self.storage_options or {})
+ ) as buffer:
+ written = self._write(file_obj=buffer, orient=orient, lines=lines, **self.to_json_kwargs)
+ else:
+ if compression:
+ raise NotImplementedError(
+ f"The compression parameter is not supported when writing to a buffer, but compression={compression}"
+ " was passed. Please provide a local path instead."
+ )
+ written = self._write(file_obj=self.path_or_buf, orient=orient, lines=lines, **self.to_json_kwargs)
+ return written
+
+ def _batch_json(self, args):
+ offset, orient, lines, to_json_kwargs = args
+
+ batch = query_table(
+ table=self.dataset.data,
+ key=slice(offset, offset + self.batch_size),
+ indices=self.dataset._indices,
+ )
+ json_str = batch.to_pandas().to_json(path_or_buf=None, orient=orient, lines=lines, **to_json_kwargs)
+ if not json_str.endswith("\n"):
+ json_str += "\n"
+ return json_str.encode(self.encoding)
+
+ def _write(
+ self,
+ file_obj: BinaryIO,
+ orient,
+ lines,
+ **to_json_kwargs,
+ ) -> int:
+ """Writes the pyarrow table as JSON lines to a binary file handle.
+
+ Caller is responsible for opening and closing the handle.
+ """
+ written = 0
+
+ if self.num_proc is None or self.num_proc == 1:
+ for offset in hf_tqdm(
+ range(0, len(self.dataset), self.batch_size),
+ unit="ba",
+ desc="Creating json from Arrow format",
+ ):
+ json_str = self._batch_json((offset, orient, lines, to_json_kwargs))
+ written += file_obj.write(json_str)
+ else:
+ num_rows, batch_size = len(self.dataset), self.batch_size
+ with multiprocessing.Pool(self.num_proc) as pool:
+ for json_str in hf_tqdm(
+ pool.imap(
+ self._batch_json,
+ [(offset, orient, lines, to_json_kwargs) for offset in range(0, num_rows, batch_size)],
+ ),
+ total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size,
+ unit="ba",
+ desc="Creating json from Arrow format",
+ ):
+ written += file_obj.write(json_str)
+
+ return written
diff --git a/venv/lib/python3.10/site-packages/datasets/io/parquet.py b/venv/lib/python3.10/site-packages/datasets/io/parquet.py
new file mode 100644
index 0000000000000000000000000000000000000000..b7b13b6b8a4229690ae8cf764ba2e3e1b4b7b039
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/io/parquet.py
@@ -0,0 +1,150 @@
+import json
+import os
+from typing import BinaryIO, Optional, Union
+
+import fsspec
+import pyarrow.parquet as pq
+
+from .. import Dataset, Features, NamedSplit, config
+from ..arrow_writer import get_writer_batch_size_from_data_size, get_writer_batch_size_from_features
+from ..formatting import query_table
+from ..packaged_modules import _PACKAGED_DATASETS_MODULES
+from ..packaged_modules.parquet.parquet import Parquet
+from ..utils import tqdm as hf_tqdm
+from ..utils.typing import NestedDataStructureLike, PathLike
+from .abc import AbstractDatasetReader
+
+
+class ParquetDatasetReader(AbstractDatasetReader):
+ def __init__(
+ self,
+ path_or_paths: NestedDataStructureLike[PathLike],
+ split: Optional[NamedSplit] = None,
+ features: Optional[Features] = None,
+ cache_dir: str = None,
+ keep_in_memory: bool = False,
+ streaming: bool = False,
+ num_proc: Optional[int] = None,
+ **kwargs,
+ ):
+ super().__init__(
+ path_or_paths,
+ split=split,
+ features=features,
+ cache_dir=cache_dir,
+ keep_in_memory=keep_in_memory,
+ streaming=streaming,
+ num_proc=num_proc,
+ **kwargs,
+ )
+ path_or_paths = path_or_paths if isinstance(path_or_paths, dict) else {self.split: path_or_paths}
+ hash = _PACKAGED_DATASETS_MODULES["parquet"][1]
+ self.builder = Parquet(
+ cache_dir=cache_dir,
+ data_files=path_or_paths,
+ features=features,
+ hash=hash,
+ **kwargs,
+ )
+
+ def read(self):
+ # Build iterable dataset
+ if self.streaming:
+ dataset = self.builder.as_streaming_dataset(split=self.split)
+ # Build regular (map-style) dataset
+ else:
+ download_config = None
+ download_mode = None
+ verification_mode = None
+ base_path = None
+
+ self.builder.download_and_prepare(
+ download_config=download_config,
+ download_mode=download_mode,
+ verification_mode=verification_mode,
+ base_path=base_path,
+ num_proc=self.num_proc,
+ )
+ dataset = self.builder.as_dataset(
+ split=self.split, verification_mode=verification_mode, in_memory=self.keep_in_memory
+ )
+ return dataset
+
+
+class ParquetDatasetWriter:
+ def __init__(
+ self,
+ dataset: Dataset,
+ path_or_buf: Union[PathLike, BinaryIO],
+ batch_size: Optional[int] = None,
+ storage_options: Optional[dict] = None,
+ use_content_defined_chunking: Union[bool, dict] = True,
+ write_page_index: bool = True,
+ **parquet_writer_kwargs,
+ ):
+ self.dataset = dataset
+ self.path_or_buf = path_or_buf
+ self.batch_size = (
+ batch_size
+ or get_writer_batch_size_from_features(dataset.features)
+ or get_writer_batch_size_from_data_size(len(dataset), dataset._estimate_nbytes())
+ )
+ self.storage_options = storage_options or {}
+ self.parquet_writer_kwargs = parquet_writer_kwargs
+ if use_content_defined_chunking is True:
+ use_content_defined_chunking = config.DEFAULT_CDC_OPTIONS
+ self.use_content_defined_chunking = use_content_defined_chunking
+ self.write_page_index = write_page_index
+
+ def write(self) -> int:
+ if isinstance(self.path_or_buf, (str, bytes, os.PathLike)):
+ with fsspec.open(self.path_or_buf, "wb", **(self.storage_options or {})) as buffer:
+ written = self._write(
+ file_obj=buffer,
+ batch_size=self.batch_size,
+ **self.parquet_writer_kwargs,
+ )
+ else:
+ written = self._write(
+ file_obj=self.path_or_buf,
+ batch_size=self.batch_size,
+ **self.parquet_writer_kwargs,
+ )
+ return written
+
+ def _write(self, file_obj: BinaryIO, batch_size: int, **parquet_writer_kwargs) -> int:
+ """Writes the pyarrow table as Parquet to a binary file handle.
+
+ Caller is responsible for opening and closing the handle.
+ """
+ written = 0
+ _ = parquet_writer_kwargs.pop("path_or_buf", None)
+ schema = self.dataset.features.arrow_schema
+
+ writer = pq.ParquetWriter(
+ file_obj,
+ schema=schema,
+ use_content_defined_chunking=self.use_content_defined_chunking,
+ write_page_index=self.write_page_index,
+ **parquet_writer_kwargs,
+ )
+
+ for offset in hf_tqdm(
+ range(0, len(self.dataset), batch_size),
+ unit="ba",
+ desc="Creating parquet from Arrow format",
+ ):
+ batch = query_table(
+ table=self.dataset._data,
+ key=slice(offset, offset + batch_size),
+ indices=self.dataset._indices,
+ )
+ writer.write_table(batch)
+ written += batch.nbytes
+
+ # TODO(kszucs): we may want to persist multiple parameters
+ if self.use_content_defined_chunking is not False:
+ writer.add_key_value_metadata({"content_defined_chunking": json.dumps(self.use_content_defined_chunking)})
+
+ writer.close()
+ return written
diff --git a/venv/lib/python3.10/site-packages/datasets/io/spark.py b/venv/lib/python3.10/site-packages/datasets/io/spark.py
new file mode 100644
index 0000000000000000000000000000000000000000..7562ba1fb5f77ed8f82374e3021fcb3a93b1da8d
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/io/spark.py
@@ -0,0 +1,57 @@
+from typing import Optional
+
+import pyspark
+
+from .. import Features, NamedSplit
+from ..download import DownloadMode
+from ..packaged_modules.spark.spark import Spark
+from .abc import AbstractDatasetReader
+
+
+class SparkDatasetReader(AbstractDatasetReader):
+ """A dataset reader that reads from a Spark DataFrame.
+
+ When caching, cache materialization is parallelized over Spark; an NFS that is accessible to the driver must be
+ provided. Streaming is not currently supported.
+ """
+
+ def __init__(
+ self,
+ df: pyspark.sql.DataFrame,
+ split: Optional[NamedSplit] = None,
+ features: Optional[Features] = None,
+ streaming: bool = True,
+ cache_dir: str = None,
+ keep_in_memory: bool = False,
+ working_dir: str = None,
+ load_from_cache_file: bool = True,
+ file_format: str = "arrow",
+ **kwargs,
+ ):
+ super().__init__(
+ split=split,
+ features=features,
+ cache_dir=cache_dir,
+ keep_in_memory=keep_in_memory,
+ streaming=streaming,
+ **kwargs,
+ )
+ self._load_from_cache_file = load_from_cache_file
+ self._file_format = file_format
+ self.builder = Spark(
+ df=df,
+ features=features,
+ cache_dir=cache_dir,
+ working_dir=working_dir,
+ **kwargs,
+ )
+
+ def read(self):
+ if self.streaming:
+ return self.builder.as_streaming_dataset(split=self.split)
+ download_mode = None if self._load_from_cache_file else DownloadMode.FORCE_REDOWNLOAD
+ self.builder.download_and_prepare(
+ download_mode=download_mode,
+ file_format=self._file_format,
+ )
+ return self.builder.as_dataset(split=self.split)
diff --git a/venv/lib/python3.10/site-packages/datasets/io/sql.py b/venv/lib/python3.10/site-packages/datasets/io/sql.py
new file mode 100644
index 0000000000000000000000000000000000000000..2331e3e6407fa81cd26569306ff2c7952dad4920
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/io/sql.py
@@ -0,0 +1,124 @@
+import multiprocessing
+from typing import TYPE_CHECKING, Optional, Union
+
+from .. import Dataset, Features, config
+from ..formatting import query_table
+from ..packaged_modules.sql.sql import Sql
+from ..utils import tqdm as hf_tqdm
+from .abc import AbstractDatasetInputStream
+
+
+if TYPE_CHECKING:
+ import sqlite3
+
+ import sqlalchemy
+
+
+class SqlDatasetReader(AbstractDatasetInputStream):
+ def __init__(
+ self,
+ sql: Union[str, "sqlalchemy.sql.Selectable"],
+ con: Union[str, "sqlalchemy.engine.Connection", "sqlalchemy.engine.Engine", "sqlite3.Connection"],
+ features: Optional[Features] = None,
+ cache_dir: str = None,
+ keep_in_memory: bool = False,
+ **kwargs,
+ ):
+ super().__init__(features=features, cache_dir=cache_dir, keep_in_memory=keep_in_memory, **kwargs)
+ self.builder = Sql(
+ cache_dir=cache_dir,
+ features=features,
+ sql=sql,
+ con=con,
+ **kwargs,
+ )
+
+ def read(self):
+ download_config = None
+ download_mode = None
+ verification_mode = None
+ base_path = None
+
+ self.builder.download_and_prepare(
+ download_config=download_config,
+ download_mode=download_mode,
+ verification_mode=verification_mode,
+ base_path=base_path,
+ )
+
+ # Build dataset for splits
+ dataset = self.builder.as_dataset(
+ split="train", verification_mode=verification_mode, in_memory=self.keep_in_memory
+ )
+ return dataset
+
+
+class SqlDatasetWriter:
+ def __init__(
+ self,
+ dataset: Dataset,
+ name: str,
+ con: Union[str, "sqlalchemy.engine.Connection", "sqlalchemy.engine.Engine", "sqlite3.Connection"],
+ batch_size: Optional[int] = None,
+ num_proc: Optional[int] = None,
+ **to_sql_kwargs,
+ ):
+ if num_proc is not None and num_proc <= 0:
+ raise ValueError(f"num_proc {num_proc} must be an integer > 0.")
+
+ self.dataset = dataset
+ self.name = name
+ self.con = con
+ self.batch_size = batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
+ self.num_proc = num_proc
+ self.to_sql_kwargs = to_sql_kwargs
+
+ def write(self) -> int:
+ _ = self.to_sql_kwargs.pop("sql", None)
+ _ = self.to_sql_kwargs.pop("con", None)
+ index = self.to_sql_kwargs.pop("index", False)
+
+ written = self._write(index=index, **self.to_sql_kwargs)
+ return written
+
+ def _batch_sql(self, args):
+ offset, index, to_sql_kwargs = args
+ to_sql_kwargs = {**to_sql_kwargs, "if_exists": "append"} if offset > 0 else to_sql_kwargs
+ batch = query_table(
+ table=self.dataset.data,
+ key=slice(offset, offset + self.batch_size),
+ indices=self.dataset._indices,
+ )
+ df = batch.to_pandas()
+ num_rows = df.to_sql(self.name, self.con, index=index, **to_sql_kwargs)
+ return num_rows or len(df)
+
+ def _write(self, index, **to_sql_kwargs) -> int:
+ """Writes the pyarrow table as SQL to a database.
+
+ Caller is responsible for opening and closing the SQL connection.
+ """
+ written = 0
+
+ if self.num_proc is None or self.num_proc == 1:
+ for offset in hf_tqdm(
+ range(0, len(self.dataset), self.batch_size),
+ unit="ba",
+ desc="Creating SQL from Arrow format",
+ ):
+ written += self._batch_sql((offset, index, to_sql_kwargs))
+ else:
+ num_rows, batch_size = len(self.dataset), self.batch_size
+ with multiprocessing.Pool(self.num_proc) as pool:
+ for num_rows in hf_tqdm(
+ pool.imap(
+ self._batch_sql,
+ [(offset, index, to_sql_kwargs) for offset in range(0, num_rows, batch_size)],
+ ),
+ total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size,
+ unit="ba",
+ desc="Creating SQL from Arrow format",
+ ):
+ written += num_rows
+
+ return written
diff --git a/venv/lib/python3.10/site-packages/datasets/io/text.py b/venv/lib/python3.10/site-packages/datasets/io/text.py
new file mode 100644
index 0000000000000000000000000000000000000000..58963f3c7ab1779d79226943e341200b782e1add
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/io/text.py
@@ -0,0 +1,60 @@
+from typing import Optional
+
+from .. import Features, NamedSplit
+from ..packaged_modules.text.text import Text
+from ..utils.typing import NestedDataStructureLike, PathLike
+from .abc import AbstractDatasetReader
+
+
+class TextDatasetReader(AbstractDatasetReader):
+ def __init__(
+ self,
+ path_or_paths: NestedDataStructureLike[PathLike],
+ split: Optional[NamedSplit] = None,
+ features: Optional[Features] = None,
+ cache_dir: str = None,
+ keep_in_memory: bool = False,
+ streaming: bool = False,
+ num_proc: Optional[int] = None,
+ **kwargs,
+ ):
+ super().__init__(
+ path_or_paths,
+ split=split,
+ features=features,
+ cache_dir=cache_dir,
+ keep_in_memory=keep_in_memory,
+ streaming=streaming,
+ num_proc=num_proc,
+ **kwargs,
+ )
+ path_or_paths = path_or_paths if isinstance(path_or_paths, dict) else {self.split: path_or_paths}
+ self.builder = Text(
+ cache_dir=cache_dir,
+ data_files=path_or_paths,
+ features=features,
+ **kwargs,
+ )
+
+ def read(self):
+ # Build iterable dataset
+ if self.streaming:
+ dataset = self.builder.as_streaming_dataset(split=self.split)
+ # Build regular (map-style) dataset
+ else:
+ download_config = None
+ download_mode = None
+ verification_mode = None
+ base_path = None
+
+ self.builder.download_and_prepare(
+ download_config=download_config,
+ download_mode=download_mode,
+ verification_mode=verification_mode,
+ base_path=base_path,
+ num_proc=self.num_proc,
+ )
+ dataset = self.builder.as_dataset(
+ split=self.split, verification_mode=verification_mode, in_memory=self.keep_in_memory
+ )
+ return dataset
diff --git a/venv/lib/python3.10/site-packages/datasets/iterable_dataset.py b/venv/lib/python3.10/site-packages/datasets/iterable_dataset.py
new file mode 100644
index 0000000000000000000000000000000000000000..9ac842d2c22847f169d8c29df5e024c40d709205
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/iterable_dataset.py
@@ -0,0 +1,4709 @@
+import asyncio
+import contextlib
+import copy
+import fnmatch
+import inspect
+import itertools
+import json
+import math
+import multiprocessing.pool
+import random
+import re
+import sys
+import time
+from collections import Counter
+from collections.abc import Iterable, Iterator
+from copy import deepcopy
+from dataclasses import dataclass
+from functools import partial
+from io import BytesIO
+from itertools import cycle, islice
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, BinaryIO, Callable, Optional, Union
+
+import fsspec.asyn
+import numpy as np
+import pandas as pd
+import pyarrow as pa
+import pyarrow.parquet as pq
+from huggingface_hub import (
+ CommitInfo,
+ CommitOperationAdd,
+ CommitOperationDelete,
+ DatasetCard,
+ DatasetCardData,
+ HfApi,
+ HfFileSystem,
+)
+from huggingface_hub.hf_api import RepoFile
+from huggingface_hub.utils import HfHubHTTPError, RepositoryNotFoundError
+from multiprocess import Pool
+
+from . import config
+from .arrow_dataset import PUSH_TO_HUB_WITHOUT_METADATA_CONFIGS_SPLIT_PATTERN_SHARDED, Dataset, DatasetInfoMixin
+from .data_files import sanitize_patterns
+from .features import Features
+from .features.features import (
+ FeatureType,
+ List,
+ Value,
+ _align_features,
+ _check_if_features_can_be_aligned,
+ _fix_for_backward_compatible_features,
+ _visit,
+ cast_to_python_objects,
+ require_decoding,
+)
+from .formatting import (
+ ArrowFormatter,
+ PythonFormatter,
+ TableFormatter,
+ TensorFormatter,
+ get_format_type_from_alias,
+ get_formatter,
+)
+from .info import DatasetInfo, DatasetInfosDict
+from .naming import _split_re
+from .splits import NamedSplit, Split, SplitDict, SplitInfo
+from .table import cast_table_to_features, embed_table_storage, read_schema_from_file, table_cast
+from .utils import tqdm as hf_tqdm
+from .utils.logging import get_logger
+from .utils.metadata import MetadataConfigs
+from .utils.py_utils import Literal, asdict, glob_pattern_to_regex, iflatmap_unordered, string_to_dict
+from .utils.sharding import _merge_gen_kwargs, _number_of_shards_in_gen_kwargs, _shuffle_gen_kwargs, _split_gen_kwargs
+from .utils.typing import PathLike
+
+
+if TYPE_CHECKING:
+ import sqlite3
+
+ import polars as pl
+ import sqlalchemy
+ import torch
+
+logger = get_logger(__name__)
+
+Key = Union[int, str]
+
+
+def identity_func(x):
+ return x
+
+
+def _rename_columns_fn(example: dict, column_mapping: dict[str, str]):
+ if any(col not in example for col in column_mapping):
+ raise ValueError(
+ f"Error when renaming {list(column_mapping)} to {list(column_mapping.values())}: columns {set(column_mapping) - set(example)} are not in the dataset."
+ )
+ if any(col in example for col in column_mapping.values()):
+ raise ValueError(
+ f"Error when renaming {list(column_mapping)} to {list(column_mapping.values())}: columns {set(example) - set(column_mapping.values())} are already in the dataset."
+ )
+ return {
+ new_column_name: example[original_column_name]
+ for original_column_name, new_column_name in column_mapping.items()
+ }
+
+
+def add_column_fn(example: dict, idx: int, name: str, column: list[dict]):
+ if name in example:
+ raise ValueError(f"Error when adding {name}: column {name} is already in the dataset.")
+ return {name: column[idx]}
+
+
+def _infer_features_from_batch(batch: dict[str, list], try_features: Optional[Features] = None) -> Features:
+ pa_table = pa.Table.from_pydict(batch)
+ if try_features is not None:
+ try:
+ pa_table = table_cast(pa_table, pa.schema(try_features.type))
+ except (TypeError, pa.ArrowInvalid, pa.ArrowNotImplementedError):
+ pass
+ return Features.from_arrow_schema(pa_table.schema)
+
+
+def _examples_to_batch(examples: list[dict[str, Any]]) -> dict[str, list]:
+ # we order the columns by order of appearance
+ # to do so, we use a dict as an ordered set
+ cols = {col: None for example in examples for col in example}
+ # when an example is missing a column, we set the value to None with .get()
+ arrays = [[example.get(col) for example in examples] for col in cols]
+ return dict(zip(cols, arrays))
+
+
+def _batch_to_examples(batch: dict[str, list]) -> Iterator[dict[str, Any]]:
+ """Convert a batch (dict of examples) to examples list"""
+ n_examples = 0 if len(batch) == 0 else len(batch[next(iter(batch))])
+ for i in range(n_examples):
+ yield {col: array[i] for col, array in batch.items()}
+
+
+def _convert_to_arrow(
+ iterable: Iterable[tuple[Key, dict]],
+ batch_size: int,
+ drop_last_batch: bool = False,
+) -> Iterator[tuple[Key, pa.Table]]:
+ """Convert and group examples in Arrow tables of size `batch_size`.
+
+ Args:
+ iterable (`Iterable[Tuple[Key, dict]]`):
+ An examples iterable containing tuples (example_key, example) of type (int/str, dict)
+ batch_size (`Optional[int]`):
+ Size of each sub-table to yield. If None or <= 0, yields the full table.
+ drop_last_batch (`bool`, defaults to `False`):
+ Drop the last batch if it is smaller than `batch_size`.
+ """
+ if batch_size is None or batch_size <= 0:
+ yield (
+ "all",
+ pa.Table.from_pylist(cast_to_python_objects([example for _, example in iterable], only_1d_for_numpy=True)),
+ )
+ return
+ iterator = iter(iterable)
+ for key, example in iterator:
+ iterator_batch = islice(iterator, batch_size - 1)
+ key_examples_list = [(key, example)] + list(iterator_batch)
+ if len(key_examples_list) < batch_size and drop_last_batch:
+ return
+ keys, examples = zip(*key_examples_list)
+ new_key = "_".join(str(key) for key in keys)
+ yield new_key, pa.Table.from_pylist(cast_to_python_objects(examples, only_1d_for_numpy=True))
+
+
+def shift_ex_examples_rngs(ex_iterable: "_BaseExamplesIterable", value: int) -> "_BaseExamplesIterable":
+ """We need to go through the ex_iterables recursively, create a new seed and return a new iterable, then set it to the containing ex_iterable."""
+
+ def set_seed_recursively(ex_iterable):
+ if hasattr(ex_iterable, "shift_rngs"):
+ ex_iterable = ex_iterable.shift_rngs(value)
+ if hasattr(ex_iterable, "ex_iterable"):
+ ex_iterable.ex_iterable = set_seed_recursively(ex_iterable.ex_iterable)
+ return ex_iterable
+
+ return set_seed_recursively(ex_iterable)
+
+
+class _BaseExamplesIterable:
+ """Base class for the examples iterable used by an IterableDataset"""
+
+ def __init__(self) -> None:
+ self._state_dict: Optional[Union[list, dict]] = None
+
+ def __iter__(self) -> Iterator[tuple[Key, dict]]:
+ """An examples iterable should yield tuples (example_key, example) of type (int/str, dict)"""
+ raise NotImplementedError(f"{type(self)} doesn't implement __iter__ yet")
+
+ @property
+ def iter_arrow(self) -> Optional[Callable[[], Iterator[tuple[Key, pa.Table]]]]:
+ return None
+
+ @property
+ def is_typed(self) -> bool:
+ return False
+
+ @property
+ def features(self) -> Optional[Features]:
+ return None
+
+ def shuffle_data_sources(self, generator: np.random.Generator) -> "_BaseExamplesIterable":
+ """
+ Either shuffle the shards/sources of the dataset, or propagate the shuffling to the underlying iterable.
+ If the order of the shards must stay fixed (when using .skip or .take for example), then this method returns self.
+ """
+ raise NotImplementedError(f"{type(self)} doesn't implement shuffle_data_sources yet")
+
+ def shard_data_sources(self, num_shards: int, index: int, contiguous=True) -> "_BaseExamplesIterable":
+ """Either keep only the requested shard, or propagate the request to the underlying iterable."""
+ raise NotImplementedError(f"{type(self)} doesn't implement shard_data_sources yet")
+
+ def split_shard_indices_by_worker(self, num_shards: int, index: int, contiguous=True) -> list[int]:
+ if contiguous:
+ div = self.num_shards // num_shards
+ mod = self.num_shards % num_shards
+ start = div * index + min(index, mod)
+ end = start + div + (1 if index < mod else 0)
+ return list(range(start, end))
+ else:
+ return list(range(index, self.num_shards, num_shards))
+
+ @property
+ def num_shards(self) -> int:
+ raise NotImplementedError(f"{type(self)} doesn't implement num_shards yet")
+
+ def _init_state_dict(self) -> dict:
+ raise NotImplementedError(f"{type(self)} doesn't implement _init_state_dict yet")
+
+ def load_state_dict(self, state_dict: dict) -> dict:
+ def _inner_load_state_dict(state, new_state):
+ if new_state is not None and isinstance(state, dict):
+ for key in new_state:
+ state[key] = _inner_load_state_dict(state[key], new_state[key])
+ return state
+ elif new_state is not None and isinstance(state, list):
+ for i in range(len(state)):
+ state[i] = _inner_load_state_dict(state[i], new_state[i])
+ return state
+ return new_state
+
+ return _inner_load_state_dict(self._state_dict, state_dict)
+
+ def state_dict(self) -> dict:
+ if self._state_dict:
+ return copy.deepcopy(self._state_dict)
+ raise RuntimeError("State dict is not initialized, please call ex_iterable._init_state_dict() first.")
+
+
+class ExamplesIterable(_BaseExamplesIterable):
+ def __init__(self, generate_examples_fn: Callable[..., tuple[Key, dict]], kwargs: dict):
+ super().__init__()
+ self.generate_examples_fn = generate_examples_fn
+ self.kwargs = kwargs
+
+ def _init_state_dict(self) -> dict:
+ self._state_dict = {"shard_idx": 0, "shard_example_idx": 0, "type": self.__class__.__name__}
+ return self._state_dict
+
+ def __iter__(self):
+ shard_idx_start = self._state_dict["shard_idx"] if self._state_dict else 0
+ for gen_kwags in islice(_split_gen_kwargs(self.kwargs, max_num_jobs=self.num_shards), shard_idx_start, None):
+ shard_example_idx_start = self._state_dict["shard_example_idx"] if self._state_dict else 0
+ for key_example in islice(self.generate_examples_fn(**gen_kwags), shard_example_idx_start, None):
+ if self._state_dict:
+ self._state_dict["shard_example_idx"] += 1
+ yield key_example
+ if self._state_dict:
+ self._state_dict["shard_idx"] += 1
+ self._state_dict["shard_example_idx"] = 0
+
+ def shuffle_data_sources(self, generator: np.random.Generator) -> "ExamplesIterable":
+ return ShuffledDataSourcesExamplesIterable(self.generate_examples_fn, self.kwargs, generator)
+
+ def shard_data_sources(self, num_shards: int, index: int, contiguous=True) -> "ExamplesIterable":
+ """Keep only the requested shard."""
+ gen_kwargs_list = _split_gen_kwargs(self.kwargs, max_num_jobs=self.num_shards)
+ shard_indices = self.split_shard_indices_by_worker(num_shards, index, contiguous=contiguous)
+ requested_gen_kwargs = _merge_gen_kwargs([gen_kwargs_list[i] for i in shard_indices])
+ return ExamplesIterable(self.generate_examples_fn, requested_gen_kwargs)
+
+ @property
+ def num_shards(self) -> int:
+ return _number_of_shards_in_gen_kwargs(self.kwargs)
+
+
+class ShuffledDataSourcesExamplesIterable(ExamplesIterable):
+ def __init__(
+ self, generate_examples_fn: Callable[..., tuple[Key, dict]], kwargs: dict, generator: np.random.Generator
+ ):
+ super().__init__(generate_examples_fn, kwargs)
+ self.generator = deepcopy(generator)
+
+ def shift_rngs(self, value: int) -> "_BaseExamplesIterable":
+ new_seed = self.generator.bit_generator.state["state"]["state"] + value
+ return ShuffledDataSourcesExamplesIterable(
+ self.generate_examples_fn,
+ self.kwargs,
+ np.random.default_rng(seed=new_seed),
+ )
+
+ def _init_state_dict(self) -> dict:
+ self._state_dict = {"shard_idx": 0, "shard_example_idx": 0, "type": self.__class__.__name__}
+ return self._state_dict
+
+ def __iter__(self):
+ """Shuffle the kwargs order to shuffle shards"""
+ rng = deepcopy(self.generator)
+ kwargs_with_shuffled_shards = _shuffle_gen_kwargs(rng, self.kwargs)
+ shard_idx_start = self._state_dict["shard_idx"] if self._state_dict else 0
+ for gen_kwags in islice(
+ _split_gen_kwargs(kwargs_with_shuffled_shards, max_num_jobs=self.num_shards), shard_idx_start, None
+ ):
+ shard_example_idx_start = self._state_dict["shard_example_idx"] if self._state_dict else 0
+ for key_example in islice(self.generate_examples_fn(**gen_kwags), shard_example_idx_start, None):
+ if self._state_dict:
+ self._state_dict["shard_example_idx"] += 1
+ yield key_example
+ if self._state_dict:
+ self._state_dict["shard_idx"] += 1
+ self._state_dict["shard_example_idx"] = 0
+
+ def shard_data_sources(self, num_shards: int, index: int, contiguous=True) -> "ExamplesIterable":
+ """Keep only the requested shard."""
+ rng = deepcopy(self.generator)
+ kwargs_with_shuffled_shards = _shuffle_gen_kwargs(rng, self.kwargs)
+ return ExamplesIterable(self.generate_examples_fn, kwargs_with_shuffled_shards).shard_data_sources(
+ num_shards, index, contiguous=contiguous
+ )
+
+
+class ArrowExamplesIterable(_BaseExamplesIterable):
+ def __init__(self, generate_tables_fn: Callable[..., tuple[Key, pa.Table]], kwargs: dict):
+ super().__init__()
+ self.generate_tables_fn = generate_tables_fn
+ self.kwargs = kwargs
+
+ @property
+ def iter_arrow(self):
+ return self._iter_arrow
+
+ def _init_state_dict(self) -> dict:
+ self._state_dict = {"shard_idx": 0, "shard_example_idx": 0, "type": self.__class__.__name__}
+ return self._state_dict
+
+ def __iter__(self):
+ formatter = PythonFormatter()
+ shard_idx_start = self._state_dict["shard_idx"] if self._state_dict else 0
+ for gen_kwags in islice(_split_gen_kwargs(self.kwargs, max_num_jobs=self.num_shards), shard_idx_start, None):
+ shard_example_idx_start = self._state_dict["shard_example_idx"] if self._state_dict else 0
+ shard_example_idx = 0
+ for key, pa_table in self.generate_tables_fn(**gen_kwags):
+ if shard_example_idx + len(pa_table) <= shard_example_idx_start:
+ shard_example_idx += len(pa_table)
+ continue
+ for pa_subtable in pa_table.to_reader(max_chunksize=config.ARROW_READER_BATCH_SIZE_IN_DATASET_ITER):
+ formatted_batch = formatter.format_batch(pa_subtable)
+ for example in _batch_to_examples(formatted_batch):
+ if shard_example_idx >= shard_example_idx_start:
+ if self._state_dict:
+ self._state_dict["shard_example_idx"] += 1
+ yield key, example
+ shard_example_idx += 1
+ if self._state_dict:
+ self._state_dict["shard_idx"] += 1
+ self._state_dict["shard_example_idx"] = 0
+
+ def _iter_arrow(self):
+ shard_idx_start = self._state_dict["shard_idx"] if self._state_dict else 0
+ for gen_kwags in islice(_split_gen_kwargs(self.kwargs, max_num_jobs=self.num_shards), shard_idx_start, None):
+ shard_example_idx_start = self._state_dict["shard_example_idx"] if self._state_dict else 0
+ shard_example_idx = 0
+ for key, pa_table in self.generate_tables_fn(**gen_kwags):
+ shard_example_idx += len(pa_table)
+ if shard_example_idx <= shard_example_idx_start:
+ continue
+ if self._state_dict:
+ self._state_dict["shard_example_idx"] += len(pa_table)
+ yield key, pa_table
+ if self._state_dict:
+ self._state_dict["shard_idx"] += 1
+ self._state_dict["shard_example_idx"] = 0
+
+ def shuffle_data_sources(self, generator: np.random.Generator) -> "ArrowExamplesIterable":
+ return ShuffledDataSourcesArrowExamplesIterable(self.generate_tables_fn, self.kwargs, generator)
+
+ def shard_data_sources(self, num_shards: int, index: int, contiguous=True) -> "ArrowExamplesIterable":
+ """Keep only the requested shard."""
+ gen_kwargs_list = _split_gen_kwargs(self.kwargs, max_num_jobs=self.num_shards)
+ shard_indices = self.split_shard_indices_by_worker(num_shards, index, contiguous=contiguous)
+ requested_gen_kwargs = _merge_gen_kwargs([gen_kwargs_list[i] for i in shard_indices])
+ return ArrowExamplesIterable(self.generate_tables_fn, requested_gen_kwargs)
+
+ @property
+ def num_shards(self) -> int:
+ return _number_of_shards_in_gen_kwargs(self.kwargs)
+
+
+class ShuffledDataSourcesArrowExamplesIterable(ArrowExamplesIterable):
+ def __init__(
+ self,
+ generate_tables_fn: Callable[..., tuple[Key, pa.Table]],
+ kwargs: dict,
+ generator: np.random.Generator,
+ ):
+ super().__init__(generate_tables_fn, kwargs)
+ self.generator = deepcopy(generator)
+
+ def shift_rngs(self, value: int) -> "_BaseExamplesIterable":
+ new_seed = self.generator.bit_generator.state["state"]["state"] + value
+ return ShuffledDataSourcesArrowExamplesIterable(
+ self.generate_examples_fn,
+ self.kwargs,
+ np.random.default_rng(seed=new_seed),
+ )
+
+ def _init_state_dict(self) -> dict:
+ self._state_dict = {"shard_idx": 0, "shard_example_idx": 0, "type": self.__class__.__name__}
+ return self._state_dict
+
+ def __iter__(self):
+ """Shuffle the kwargs order to shuffle shards"""
+ rng = deepcopy(self.generator)
+ kwargs_with_shuffled_shards = _shuffle_gen_kwargs(rng, self.kwargs)
+ formatter = PythonFormatter()
+ shard_idx_start = self._state_dict["shard_idx"] if self._state_dict else 0
+ for gen_kwags in islice(
+ _split_gen_kwargs(kwargs_with_shuffled_shards, max_num_jobs=self.num_shards), shard_idx_start, None
+ ):
+ shard_example_idx_start = self._state_dict["shard_example_idx"] if self._state_dict else 0
+ shard_example_idx = 0
+ for key, pa_table in self.generate_tables_fn(**gen_kwags):
+ if shard_example_idx + len(pa_table) <= shard_example_idx_start:
+ shard_example_idx += len(pa_table)
+ continue
+ for pa_subtable in pa_table.to_reader(max_chunksize=config.ARROW_READER_BATCH_SIZE_IN_DATASET_ITER):
+ formatted_batch = formatter.format_batch(pa_subtable)
+ for example in _batch_to_examples(formatted_batch):
+ if shard_example_idx >= shard_example_idx_start:
+ if self._state_dict:
+ self._state_dict["shard_example_idx"] += 1
+ yield key, example
+ shard_example_idx += 1
+ if self._state_dict:
+ self._state_dict["shard_idx"] += 1
+ self._state_dict["shard_example_idx"] = 0
+
+ def _iter_arrow(self):
+ rng = deepcopy(self.generator)
+ kwargs_with_shuffled_shards = _shuffle_gen_kwargs(rng, self.kwargs)
+ shard_idx_start = self._state_dict["shard_idx"] if self._state_dict else 0
+ for gen_kwags in islice(
+ _split_gen_kwargs(kwargs_with_shuffled_shards, max_num_jobs=self.num_shards), shard_idx_start, None
+ ):
+ shard_example_idx_start = self._state_dict["shard_example_idx"] if self._state_dict else 0
+ shard_example_idx = 0
+ for key, pa_table in self.generate_tables_fn(**gen_kwags):
+ shard_example_idx += len(pa_table)
+ if shard_example_idx <= shard_example_idx_start:
+ continue
+ if self._state_dict:
+ self._state_dict["shard_example_idx"] += len(pa_table)
+ yield key, pa_table
+ if self._state_dict:
+ self._state_dict["shard_idx"] += 1
+ self._state_dict["shard_example_idx"] = 0
+
+ def shard_data_sources(self, num_shards: int, index: int, contiguous=True) -> "ArrowExamplesIterable":
+ """Keep only the requested shard."""
+ rng = deepcopy(self.generator)
+ kwargs_with_shuffled_shards = _shuffle_gen_kwargs(rng, self.kwargs)
+ return ArrowExamplesIterable(self.generate_tables_fn, kwargs_with_shuffled_shards).shard_data_sources(
+ num_shards, index, contiguous=contiguous
+ )
+
+
+class RebatchedArrowExamplesIterable(_BaseExamplesIterable):
+ def __init__(self, ex_iterable: _BaseExamplesIterable, batch_size: Optional[int], drop_last_batch: bool = False):
+ super().__init__()
+ self.ex_iterable = ex_iterable
+ self.batch_size = batch_size
+ self.drop_last_batch = drop_last_batch
+
+ @property
+ def iter_arrow(self):
+ return self._iter_arrow
+
+ @property
+ def is_typed(self):
+ return self.ex_iterable.is_typed
+
+ @property
+ def features(self):
+ return self.ex_iterable.features
+
+ def _init_state_dict(self) -> dict:
+ self._state_dict = {
+ "examples_iterable": self.ex_iterable._init_state_dict(),
+ "previous_state": None,
+ "batch_idx": 0,
+ "num_chunks_since_previous_state": 0,
+ "cropped_chunk_length": 0,
+ "type": self.__class__.__name__,
+ }
+ return self._state_dict
+
+ def __iter__(self):
+ yield from self.ex_iterable
+
+ def _iter_arrow(self) -> Iterator[tuple[Key, pa.Table]]:
+ """Iterate over sub-tables of size `batch_size`."""
+ if self._state_dict and self._state_dict["previous_state"]:
+ self.ex_iterable.load_state_dict(self._state_dict["previous_state"])
+ if self.ex_iterable.iter_arrow:
+ iterator = self.ex_iterable.iter_arrow()
+ else:
+ iterator = _convert_to_arrow(self.ex_iterable, batch_size=1)
+ if self.batch_size is None or self.batch_size <= 0:
+ if self._state_dict and self._state_dict["batch_idx"] > 0:
+ return
+ all_pa_table = pa.concat_tables([pa_table for _, pa_table in iterator])
+ if self._state_dict:
+ self._state_dict["batch_idx"] = 1
+ yield "all", all_pa_table
+ return
+ keys_buffer = []
+ chunks_buffer = []
+ chunks_buffer_size = 0
+ num_chunks_to_skip = self._state_dict["num_chunks_since_previous_state"] if self._state_dict else 0
+ chunk_length_to_crop = self._state_dict["cropped_chunk_length"] if self._state_dict else 0
+ if self._state_dict:
+ previous_state = self.ex_iterable.state_dict()
+ self._state_dict["previous_state"] = previous_state
+ for key, pa_table in iterator:
+ for num_chunks_since_previous_state, chunk in enumerate(pa_table.to_reader(max_chunksize=self.batch_size)):
+ if num_chunks_to_skip > 1:
+ num_chunks_to_skip -= 1
+ continue
+ elif num_chunks_to_skip == 1 and chunk_length_to_crop == 0:
+ num_chunks_to_skip -= 1
+ continue
+ elif num_chunks_to_skip == 1 and chunk_length_to_crop > 0:
+ chunk = chunk.slice(chunk_length_to_crop, len(chunk) - chunk_length_to_crop)
+ num_chunks_to_skip = 0
+ chunk_length_to_crop = 0
+ if len(chunk) == 0:
+ continue
+
+ if chunks_buffer_size + len(chunk) < self.batch_size:
+ keys_buffer.append(key)
+ chunks_buffer.append(chunk)
+ chunks_buffer_size += len(chunk)
+ continue
+ elif chunks_buffer_size + len(chunk) == self.batch_size:
+ keys_buffer.append(key)
+ chunks_buffer.append(chunk)
+ new_key = "_".join(str(_key) for _key in keys_buffer)
+ if self._state_dict:
+ self._state_dict["batch_idx"] += 1
+ self._state_dict["num_chunks_since_previous_state"] += len(chunks_buffer)
+ self._state_dict["cropped_chunk_length"] = 0
+ yield new_key, pa.Table.from_batches(chunks_buffer)
+ keys_buffer = []
+ chunks_buffer = []
+ chunks_buffer_size = 0
+ if self._state_dict:
+ self._state_dict["previous_state"] = previous_state
+ self._state_dict["num_chunks_since_previous_state"] = num_chunks_since_previous_state + 1
+ else:
+ cropped_chunk_length = self.batch_size - chunks_buffer_size
+ keys_buffer.append(f"{key}[:{cropped_chunk_length}]")
+ chunks_buffer.append(chunk.slice(0, cropped_chunk_length))
+ new_key = "_".join(str(_key) for _key in keys_buffer)
+ if self._state_dict:
+ self._state_dict["batch_idx"] += 1
+ self._state_dict["num_chunks_since_previous_state"] += len(chunks_buffer)
+ self._state_dict["cropped_chunk_length"] = cropped_chunk_length
+ yield new_key, pa.Table.from_batches(chunks_buffer)
+ keys_buffer = [f"{key}[{cropped_chunk_length}:]"]
+ chunks_buffer = [chunk.slice(cropped_chunk_length, len(chunk) - cropped_chunk_length)]
+ chunks_buffer_size = len(chunk) - cropped_chunk_length
+ if self._state_dict:
+ self._state_dict["previous_state"] = previous_state
+ self._state_dict["num_chunks_since_previous_state"] = num_chunks_since_previous_state
+ if self._state_dict:
+ previous_state = self.ex_iterable.state_dict()
+ if not self.drop_last_batch and chunks_buffer:
+ new_key = "_".join(str(_key) for _key in keys_buffer)
+ if self._state_dict:
+ self._state_dict["previous_state"] = previous_state
+ self._state_dict["batch_idx"] += 1
+ self._state_dict["num_chunks_since_previous_state"] = 0
+ self._state_dict["cropped_chunk_length"] = 0
+ yield new_key, pa.Table.from_batches(chunks_buffer)
+
+ def shuffle_data_sources(self, generator: np.random.Generator) -> "RebatchedArrowExamplesIterable":
+ return RebatchedArrowExamplesIterable(
+ self.ex_iterable.shuffle_data_sources(generator), self.batch_size, self.drop_last_batch
+ )
+
+ def shard_data_sources(self, num_shards: int, index: int, contiguous=True) -> "RebatchedArrowExamplesIterable":
+ return RebatchedArrowExamplesIterable(
+ self.ex_iterable.shard_data_sources(num_shards, index, contiguous=contiguous),
+ self.batch_size,
+ self.drop_last_batch,
+ )
+
+ @property
+ def num_shards(self) -> int:
+ return self.ex_iterable.num_shards
+
+
+class SelectColumnsIterable(_BaseExamplesIterable):
+ def __init__(self, ex_iterable: _BaseExamplesIterable, column_names: list[str]):
+ super().__init__()
+ self.ex_iterable = ex_iterable
+ self.column_names = column_names
+
+ @property
+ def iter_arrow(self):
+ if self.ex_iterable.iter_arrow:
+ return self._iter_arrow
+
+ @property
+ def is_typed(self):
+ return self.ex_iterable.is_typed
+
+ @property
+ def features(self):
+ return self.ex_iterable.features
+
+ def _init_state_dict(self) -> dict:
+ self._state_dict = self.ex_iterable._init_state_dict()
+ return self._state_dict
+
+ def __iter__(self):
+ for idx, row in self.ex_iterable:
+ yield idx, {c: row[c] for c in self.column_names}
+
+ def _iter_arrow(self) -> Iterator[tuple[Key, pa.Table]]:
+ for idx, pa_table in self.ex_iterable.iter_arrow():
+ if len(pa_table) > 0: # empty tables have no schema
+ yield idx, pa_table.select(self.column_names)
+
+ def shuffle_data_sources(self, generator: np.random.Generator) -> "SelectColumnsIterable":
+ return SelectColumnsIterable(self.ex_iterable.shuffle_data_sources(generator), self.column_names)
+
+ def shard_data_sources(self, num_shards: int, index: int, contiguous=True) -> "SelectColumnsIterable":
+ return SelectColumnsIterable(
+ self.ex_iterable.shard_data_sources(num_shards, index, contiguous=contiguous), self.column_names
+ )
+
+ @property
+ def num_shards(self) -> int:
+ return self.ex_iterable.num_shards
+
+
+class StepExamplesIterable(_BaseExamplesIterable):
+ def __init__(self, ex_iterable: _BaseExamplesIterable, step: int, offset: int):
+ super().__init__()
+ self.ex_iterable = ex_iterable
+ self.step = step
+ self.offset = offset
+ # TODO(QL): implement iter_arrow
+
+ @property
+ def is_typed(self):
+ return self.ex_iterable.is_typed
+
+ @property
+ def features(self):
+ return self.ex_iterable.features
+
+ def _init_state_dict(self) -> dict:
+ self._state_dict = self.ex_iterable._init_state_dict()
+ return self._state_dict
+
+ def __iter__(self):
+ ex_iterator = iter(self.ex_iterable)
+ while True:
+ batch = list(islice(ex_iterator, self.step))
+ if len(batch) > self.offset:
+ yield batch[self.offset]
+ else:
+ break
+
+ def shuffle_data_sources(self, generator: np.random.Generator) -> "StepExamplesIterable":
+ return StepExamplesIterable(
+ self.ex_iterable.shuffle_data_sources(generator), step=self.step, offset=self.offset
+ )
+
+ def shard_data_sources(self, num_shards: int, index: int, contiguous=True) -> "StepExamplesIterable":
+ return StepExamplesIterable(
+ self.ex_iterable.shard_data_sources(num_shards, index, contiguous=contiguous),
+ step=self.step,
+ offset=self.offset,
+ )
+
+ @property
+ def num_shards(self) -> int:
+ return self.ex_iterable.num_shards
+
+
+class CyclingMultiSourcesExamplesIterable(_BaseExamplesIterable):
+ def __init__(
+ self,
+ ex_iterables: list[_BaseExamplesIterable],
+ stopping_strategy: Literal[
+ "first_exhausted", "all_exhausted", "all_exhausted_without_replacement"
+ ] = "first_exhausted",
+ ):
+ super().__init__()
+ self.ex_iterables = ex_iterables
+ self.stopping_strategy = stopping_strategy
+
+ # if undersampling ("first_exhausted"), we stop as soon as one dataset is exhausted
+ # if oversampling ("all_exhausted"), we stop as soons as every dataset is exhausted, i.e as soon as every samples of every dataset has been visited at least once
+ # if sampling without replacement ("all_exhausted_without_replacement"), we stop once all samples of every dataset has been visited exactly once.
+ self.bool_strategy_func = (
+ np.all if (stopping_strategy in ("all_exhausted", "all_exhausted_without_replacement")) else np.any
+ )
+
+ @property
+ def is_typed(self):
+ return self.ex_iterables[0].is_typed
+
+ @property
+ def features(self):
+ return self.ex_iterables[0].features
+
+ @property
+ def iter_arrow(self):
+ # Can iterate on arrow tables if all ex_iterables can iterate
+ return self._iter_arrow if all(ex_iterable.iter_arrow for ex_iterable in self.ex_iterables) else None
+
+ def _get_indices_iterator(self):
+ # this is an infinite iterator to keep track of which iterator we want to pick examples from
+ ex_iterable_idx = self._state_dict["ex_iterable_idx"] if self._state_dict else 0
+ for next_ex_iterable_idx in islice(cycle(range(len(self.ex_iterables))), ex_iterable_idx + 1, None):
+ if self._state_dict:
+ self._state_dict["ex_iterable_idx"] = next_ex_iterable_idx
+ yield ex_iterable_idx
+ ex_iterable_idx = next_ex_iterable_idx
+
+ def _init_state_dict(self) -> dict:
+ self._state_dict = {
+ "ex_iterable_idx": 0,
+ "ex_iterables": [ex_iterable._init_state_dict() for ex_iterable in self.ex_iterables],
+ "previous_states": [None] * len(self.ex_iterables),
+ "is_exhausted": [False] * len(self.ex_iterables),
+ "type": self.__class__.__name__,
+ }
+ return self._state_dict
+
+ def _iter_arrow(self):
+ # we use this to buffer one example of each iterator to know if an iterator is exhausted
+ nexts = [None] * len(self.ex_iterables)
+ # because of that, we need to rewind 1 example when reloading the state dict
+ if self._state_dict:
+ for i in range(len(self.ex_iterables)):
+ if self._state_dict["previous_states"][i] is not None:
+ self.ex_iterables[i].load_state_dict(self._state_dict["previous_states"][i])
+ iterators = [ex_iterable.iter_arrow() for ex_iterable in self.ex_iterables]
+
+ indices_iterator = self._get_indices_iterator()
+
+ is_exhausted = (
+ np.array(self._state_dict["is_exhausted"]) if self._state_dict else np.full(len(self.ex_iterables), False)
+ )
+ for i in indices_iterator:
+ # if the stopping criteria is met, break the main for loop
+ if self.bool_strategy_func(is_exhausted):
+ break
+ # Skip exhausted iterators if we sample without replacement
+ if is_exhausted[i] and self.stopping_strategy in ["all_exhausted_without_replacement"]:
+ continue
+ # let's pick one example from the iterator at index i
+ if nexts[i] is None:
+ nexts[i] = next(iterators[i], False)
+ result = nexts[i]
+ if self._state_dict:
+ self._state_dict["previous_states"][i] = deepcopy(self._state_dict["ex_iterables"][i])
+ nexts[i] = next(iterators[i], False)
+
+ # the iterator is exhausted
+ if nexts[i] is False:
+ is_exhausted[i] = True
+ if self._state_dict:
+ self._state_dict["is_exhausted"][i] = True
+ # we reset it in case the stopping crtieria isn't met yet and we sample with replacement
+ if self.stopping_strategy not in ["all_exhausted_without_replacement"]:
+ nexts[i] = None
+ if self._state_dict:
+ self._state_dict["ex_iterables"][i] = self.ex_iterables[i]._init_state_dict()
+ self._state_dict["previous_states"][i] = None
+ iterators[i] = self.ex_iterables[i]._iter_arrow()
+
+ if result is not False:
+ yield result
+
+ def __iter__(self):
+ # we use this to buffer one example of each iterator to know if an iterator is exhausted
+ nexts = [None] * len(self.ex_iterables)
+ # because of that, we need to rewind 1 example when reloading the state dict
+ if self._state_dict:
+ for i in range(len(self.ex_iterables)):
+ if self._state_dict["previous_states"][i] is not None:
+ self.ex_iterables[i].load_state_dict(self._state_dict["previous_states"][i])
+ iterators = [iter(ex_iterable) for ex_iterable in self.ex_iterables]
+
+ indices_iterator = self._get_indices_iterator()
+
+ is_exhausted = (
+ np.array(self._state_dict["is_exhausted"]) if self._state_dict else np.full(len(self.ex_iterables), False)
+ )
+ for i in indices_iterator:
+ # if the stopping criteria is met, break the main for loop
+ if self.bool_strategy_func(is_exhausted):
+ break
+ # let's pick one example from the iterator at index i
+ if is_exhausted[i] and self.stopping_strategy in ["all_exhausted_without_replacement"]:
+ continue
+ if nexts[i] is None:
+ nexts[i] = next(iterators[i], False)
+ result = nexts[i]
+ if self._state_dict:
+ self._state_dict["previous_states"][i] = deepcopy(self._state_dict["ex_iterables"][i])
+ nexts[i] = next(iterators[i], False)
+
+ # the iterator is exhausted
+ if nexts[i] is False:
+ is_exhausted[i] = True
+ if self._state_dict:
+ self._state_dict["is_exhausted"][i] = True
+ # we reset it in case the stopping crtieria isn't met yet
+ if self.stopping_strategy not in ["all_exhausted_without_replacement"]:
+ nexts[i] = None
+ if self._state_dict:
+ self._state_dict["ex_iterables"][i] = self.ex_iterables[i]._init_state_dict()
+ self._state_dict["previous_states"][i] = None
+ iterators[i] = iter(self.ex_iterables[i])
+ if result is not False:
+ yield result
+
+ def shuffle_data_sources(self, generator: np.random.Generator) -> "CyclingMultiSourcesExamplesIterable":
+ """Shuffle each underlying examples iterable."""
+ ex_iterables = [ex_iterable.shuffle_data_sources(generator) for ex_iterable in self.ex_iterables]
+ return CyclingMultiSourcesExamplesIterable(ex_iterables, self.stopping_strategy)
+
+ @property
+ def num_shards(self) -> int:
+ return min(ex_iterable.num_shards for ex_iterable in self.ex_iterables) if self.ex_iterables else 0
+
+ def shard_data_sources(
+ self, num_shards: int, index: int, contiguous=True
+ ) -> "CyclingMultiSourcesExamplesIterable":
+ """Either keep only the requested shard, or propagate the request to the underlying iterable."""
+ if num_shards < self.num_shards:
+ return CyclingMultiSourcesExamplesIterable(
+ [
+ iterable.shard_data_sources(num_shards, index, contiguous=contiguous)
+ for iterable in self.ex_iterables
+ ],
+ stopping_strategy=self.stopping_strategy,
+ )
+ elif index < self.num_shards:
+ return CyclingMultiSourcesExamplesIterable(
+ [
+ iterable.shard_data_sources(self.num_shards, index, contiguous=contiguous)
+ for iterable in self.ex_iterables
+ ],
+ stopping_strategy=self.stopping_strategy,
+ )
+ else:
+ return CyclingMultiSourcesExamplesIterable(
+ [],
+ stopping_strategy=self.stopping_strategy,
+ )
+
+
+class VerticallyConcatenatedMultiSourcesExamplesIterable(_BaseExamplesIterable):
+ """
+ VerticallyConcatenatedMultiSourcesExamplesIterable simply chains the input iterables.
+ It doesn't require the examples iterables to always yield the same columns.
+ Instead, this is handled by the `IterableDataset` class or `FormattedExamplesIterable`.
+
+ For information, `IterableDataset` merges the features of all the datasets to concatenate into one.
+ We use `IterableDataset._resolve_features` to obtain the features of all the datasets to concatenate.
+
+ Then for each example, `IterableDataset` and `FormattedExamplesIterable` automatically fill missing columns with None.
+ This is done with `_apply_feature_types_on_example`.
+ """
+
+ def __init__(self, ex_iterables: list[_BaseExamplesIterable]):
+ super().__init__()
+ self.ex_iterables = ex_iterables
+
+ @property
+ def is_typed(self):
+ return self.ex_iterables[0].is_typed
+
+ @property
+ def features(self):
+ return self.ex_iterables[0].features
+
+ @property
+ def iter_arrow(self):
+ if all(ex_iterable.iter_arrow is not None for ex_iterable in self.ex_iterables):
+ return self._iter_arrow
+
+ def _init_state_dict(self) -> dict:
+ self._state_dict = {
+ "ex_iterable_idx": 0,
+ "ex_iterables": [ex_iterable._init_state_dict() for ex_iterable in self.ex_iterables],
+ "type": self.__class__.__name__,
+ }
+ return self._state_dict
+
+ def __iter__(self):
+ ex_iterable_idx_start = self._state_dict["ex_iterable_idx"] if self._state_dict else 0
+ for ex_iterable in islice(self.ex_iterables, ex_iterable_idx_start, None):
+ yield from ex_iterable
+ if self._state_dict:
+ self._state_dict["ex_iterable_idx"] += 1
+
+ def _iter_arrow(self):
+ ex_iterable_idx_start = self._state_dict["ex_iterable_idx"] if self._state_dict else 0
+ for ex_iterable in islice(self.ex_iterables, ex_iterable_idx_start, None):
+ yield from ex_iterable.iter_arrow()
+ if self._state_dict:
+ self._state_dict["ex_iterable_idx"] += 1
+
+ def shuffle_data_sources(
+ self, generator: np.random.Generator
+ ) -> "VerticallyConcatenatedMultiSourcesExamplesIterable":
+ """Shuffle the list of examples iterable, as well as each underlying examples iterable."""
+ rng = deepcopy(generator)
+ ex_iterables = list(self.ex_iterables)
+ rng.shuffle(ex_iterables)
+ ex_iterables = [ex_iterable.shuffle_data_sources(generator) for ex_iterable in ex_iterables]
+ return VerticallyConcatenatedMultiSourcesExamplesIterable(ex_iterables)
+
+ @property
+ def num_shards(self) -> int:
+ return min(ex_iterable.num_shards for ex_iterable in self.ex_iterables)
+
+ def shard_data_sources(
+ self, num_shards: int, index: int, contiguous=True
+ ) -> "VerticallyConcatenatedMultiSourcesExamplesIterable":
+ """Either keep only the requested shard, or propagate the request to the underlying iterable."""
+ return VerticallyConcatenatedMultiSourcesExamplesIterable(
+ [iterable.shard_data_sources(num_shards, index, contiguous=contiguous) for iterable in self.ex_iterables]
+ )
+
+
+def _check_column_names(column_names: list[str]):
+ """Check the column names to make sure they don't contain duplicates."""
+ counter = Counter(column_names)
+ if not all(count == 1 for count in counter.values()):
+ duplicated_columns = [col for col in counter if counter[col] > 1]
+ raise ValueError(
+ f"The examples iterables can't have duplicated columns but columns {duplicated_columns} are duplicated."
+ )
+
+
+class HorizontallyConcatenatedMultiSourcesExamplesIterable(_BaseExamplesIterable):
+ """
+ HorizontallyConcatenatedMultiSourcesExamplesIterable merges examples together for the input list of iterables.
+ It also checks that there are no duplicate columns (otherwise we don't know which one to keep).
+ This check is done once when yielding the first example.
+
+ However it doesn't fill missing columns with None.
+ Instead, this is handled by the `IterableDataset` class or `FormattedExamplesIterable`.
+
+ For information, `IterableDataset` merges the features of all the datasets to concatenate into one.
+ We use `IterableDataset._resolve_features` to obtain the features of all the datasets to concatenate.
+
+ Then for each example, `IterableDataset` and `FormattedExamplesIterable` automatically fill missing columns with None.
+ This is done with `_apply_feature_types_on_example`.
+ """
+
+ def __init__(self, ex_iterables: list[_BaseExamplesIterable]):
+ super().__init__()
+ self.ex_iterables = ex_iterables
+ # TODO(QL): implement iter_arrow
+
+ @property
+ def is_typed(self):
+ return self.ex_iterables[0].is_typed
+
+ @property
+ def features(self):
+ return self.ex_iterables[0].features
+
+ def _init_state_dict(self) -> dict:
+ self._state_dict = {
+ "ex_iterables": [ex_iterable._init_state_dict() for ex_iterable in self.ex_iterables],
+ "type": self.__class__.__name__,
+ }
+ return self._state_dict
+
+ def __iter__(self):
+ ex_iterators = [iter(ex_iterable) for ex_iterable in self.ex_iterables]
+ for i in itertools.count():
+ keys = []
+ examples = []
+ for ex_iterator in list(ex_iterators):
+ try:
+ key, example = next(ex_iterator)
+ keys.append(key)
+ examples.append(example)
+ except StopIteration:
+ ex_iterators.remove(ex_iterator)
+ if ex_iterators:
+ if i == 0:
+ _check_column_names([column_name for example in examples for column_name in example])
+ new_example = {}
+ for example in examples:
+ new_example.update(example)
+ new_key = "_".join(str(key) for key in keys)
+ yield new_key, new_example
+ else:
+ break
+
+ def shuffle_data_sources(
+ self, generator: np.random.Generator
+ ) -> "HorizontallyConcatenatedMultiSourcesExamplesIterable":
+ """Doesn't shuffle the wrapped examples iterable since it would break the alignment between them."""
+ return self
+
+ @property
+ def num_shards(self) -> int:
+ return 1
+
+ def shard_data_sources(
+ self, num_shards: int, index: int, contiguous=True
+ ) -> "HorizontallyConcatenatedMultiSourcesExamplesIterable":
+ """Either keep only the requested shard, or propagate the request to the underlying iterable."""
+ return HorizontallyConcatenatedMultiSourcesExamplesIterable(
+ [iterable.shard_data_sources(num_shards, index, contiguous=contiguous) for iterable in self.ex_iterables]
+ )
+
+
+class RandomlyCyclingMultiSourcesExamplesIterable(CyclingMultiSourcesExamplesIterable):
+ def __init__(
+ self,
+ ex_iterables: list[_BaseExamplesIterable],
+ generator: np.random.Generator,
+ probabilities: Optional[list[float]] = None,
+ stopping_strategy: Literal[
+ "first_exhausted", "all_exhausted", "all_exhausted_without_replacement"
+ ] = "first_exhausted",
+ ):
+ super().__init__(ex_iterables, stopping_strategy)
+ self.generator = deepcopy(generator)
+ self.probabilities = probabilities
+
+ def shift_rngs(self, value: int) -> "_BaseExamplesIterable":
+ new_seed = self.generator.bit_generator.state["state"]["state"] + value
+ return RandomlyCyclingMultiSourcesExamplesIterable(
+ ex_iterables=self.ex_iterables,
+ generator=np.random.default_rng(seed=new_seed),
+ probabilities=self.probabilities,
+ stopping_strategy=self.stopping_strategy,
+ )
+
+ @property
+ def is_typed(self):
+ return self.ex_iterables[0].is_typed
+
+ @property
+ def features(self):
+ return self.ex_iterables[0].features
+
+ def _get_indices_iterator(self):
+ rng = deepcopy(self.generator)
+ num_sources = len(self.ex_iterables)
+ random_batch_size = 1000
+ # this is an infinite iterator that randomly samples the index of the source to pick examples from
+ index_offset = self._state_dict["bit_generator_index_offset"] if self._state_dict else 0
+ if self._state_dict:
+ rng.bit_generator.state = self._state_dict["bit_generator_state"]
+ if self.probabilities is None:
+ while True:
+ for i in islice(rng.integers(0, num_sources, size=random_batch_size), index_offset, None):
+ index_offset = (index_offset + 1) % random_batch_size
+ if self._state_dict:
+ self._state_dict["bit_generator_index_offset"] = index_offset
+ if index_offset == 0:
+ self._state_dict["bit_generator_state"] = rng.bit_generator.state
+ yield int(i)
+ else:
+ while True:
+ for i in islice(
+ rng.choice(num_sources, size=random_batch_size, p=self.probabilities), index_offset, None
+ ):
+ index_offset = (index_offset + 1) % random_batch_size
+ if self._state_dict:
+ self._state_dict["bit_generator_index_offset"] = index_offset
+ if index_offset == 0:
+ self._state_dict["bit_generator_state"] = rng.bit_generator.state
+ yield int(i)
+
+ def _init_state_dict(self) -> dict:
+ self._state_dict = {
+ "bit_generator_state": self.generator.bit_generator.state,
+ "bit_generator_index_offset": 0,
+ "ex_iterables": [ex_iterable._init_state_dict() for ex_iterable in self.ex_iterables],
+ "previous_states": [None] * len(self.ex_iterables),
+ "is_exhausted": [False] * len(self.ex_iterables),
+ "type": self.__class__.__name__,
+ }
+ return self._state_dict
+
+ def shuffle_data_sources(self, generator: np.random.Generator) -> "RandomlyCyclingMultiSourcesExamplesIterable":
+ """Shuffle the data sources of each wrapped examples iterable."""
+ ex_iterables = [ex_iterable.shuffle_data_sources(generator) for ex_iterable in self.ex_iterables]
+ return RandomlyCyclingMultiSourcesExamplesIterable(
+ ex_iterables,
+ generator=generator,
+ probabilities=self.probabilities,
+ stopping_strategy=self.stopping_strategy,
+ )
+
+ def shard_data_sources(
+ self, num_shards: int, index: int, contiguous=True
+ ) -> "RandomlyCyclingMultiSourcesExamplesIterable":
+ """Either keep only the requested shard, or propagate the request to the underlying iterable."""
+ if num_shards < self.num_shards:
+ return RandomlyCyclingMultiSourcesExamplesIterable(
+ [
+ iterable.shard_data_sources(num_shards, index, contiguous=contiguous)
+ for iterable in self.ex_iterables
+ ],
+ self.generator,
+ self.probabilities,
+ self.stopping_strategy,
+ )
+ elif index < self.num_shards:
+ return RandomlyCyclingMultiSourcesExamplesIterable(
+ [
+ iterable.shard_data_sources(self.num_shards, index, contiguous=contiguous)
+ for iterable in self.ex_iterables
+ ],
+ self.generator,
+ self.probabilities,
+ self.stopping_strategy,
+ )
+ else:
+ return RandomlyCyclingMultiSourcesExamplesIterable(
+ [],
+ self.generator,
+ self.probabilities,
+ self.stopping_strategy,
+ )
+
+
+def _table_output_to_arrow(output) -> pa.Table:
+ if isinstance(output, pa.Table):
+ return output
+ if isinstance(output, (pd.DataFrame, pd.Series)):
+ return pa.Table.from_pandas(output)
+ if config.POLARS_AVAILABLE and "polars" in sys.modules:
+ import polars as pl
+
+ if isinstance(output, (pl.DataFrame, pl.Series)):
+ return output.to_arrow()
+ return output
+
+
+class MappedExamplesIterable(_BaseExamplesIterable):
+ def __init__(
+ self,
+ ex_iterable: _BaseExamplesIterable,
+ function: Callable,
+ with_indices: bool = False,
+ input_columns: Optional[list[str]] = None,
+ batched: bool = False,
+ batch_size: Optional[int] = 1000,
+ drop_last_batch: bool = False,
+ remove_columns: Optional[list[str]] = None,
+ fn_kwargs: Optional[dict] = None,
+ formatting: Optional["FormattingConfig"] = None,
+ features: Optional[Features] = None,
+ max_num_running_async_map_functions_in_parallel: Optional[int] = None,
+ ):
+ super().__init__()
+ self.ex_iterable = ex_iterable
+ self.function = function
+ self.batched = batched
+ self.batch_size = batch_size
+ self.drop_last_batch = drop_last_batch
+ self.remove_columns = remove_columns
+ self.with_indices = with_indices
+ self.input_columns = input_columns
+ self.fn_kwargs = fn_kwargs or {}
+ self.formatting = formatting # required for iter_arrow
+ self._features = features
+ self.max_num_running_async_map_functions_in_parallel = (
+ max_num_running_async_map_functions_in_parallel or config.MAX_NUM_RUNNING_ASYNC_MAP_FUNCTIONS_IN_PARALLEL
+ )
+ # sanity checks
+ if formatting and formatting.is_table:
+ # batch_size should match for iter_arrow
+ if not isinstance(ex_iterable, RebatchedArrowExamplesIterable):
+ raise ValueError(
+ f"The {formatting.format_type.capitalize()}-formatted {type(self).__name__} has underlying iterable"
+ f"that is a {type(ex_iterable).__name__} instead of a RebatchedArrowExamplesIterable."
+ )
+ elif ex_iterable.batch_size != (batch_size if batched else 1):
+ raise ValueError(
+ f"The {formatting.format_type.capitalize()}-formatted {type(self).__name__} has batch_size={batch_size if batched else 1} which is"
+ f"different from {ex_iterable.batch_size=} from its underlying iterable."
+ )
+ # to enable graceful ends
+ self._owned_loops_and_tasks: list[tuple[asyncio.AbstractEventLoop, list[asyncio.Task]]] = []
+
+ @property
+ def iter_arrow(self):
+ if self.formatting and self.formatting.is_table:
+ return self._iter_arrow
+
+ @property
+ def is_typed(self):
+ return self.features is not None # user has extracted features
+
+ @property
+ def features(self):
+ return self._features
+
+ def _init_state_dict(self) -> dict:
+ self._state_dict = {
+ "examples_iterable": self.ex_iterable._init_state_dict(),
+ "previous_state": None,
+ "num_examples_since_previous_state": 0,
+ "previous_state_example_idx": 0,
+ "type": self.__class__.__name__,
+ }
+ return self._state_dict
+
+ def __iter__(self):
+ if self.formatting and self.formatting.is_table:
+ formatter = PythonFormatter()
+ for key, pa_table in self._iter_arrow(max_chunksize=1):
+ yield key, formatter.format_row(pa_table)
+ else:
+ yield from self._iter()
+
+ def _iter(self):
+ current_idx = self._state_dict["previous_state_example_idx"] if self._state_dict else 0
+ if self._state_dict and self._state_dict["previous_state"]:
+ self.ex_iterable.load_state_dict(self._state_dict["previous_state"])
+ num_examples_to_skip = self._state_dict["num_examples_since_previous_state"]
+ else:
+ num_examples_to_skip = 0
+ iterator = iter(self.ex_iterable)
+
+ # We use the same logic as in Dataset.map, but with less features/formatting
+ # since they're handled by FormattedExamplesIterable
+
+ if self.formatting:
+ formatter = get_formatter(self.formatting.format_type)
+ format_dict = formatter.recursive_tensorize if isinstance(formatter, TensorFormatter) else None
+ else:
+ format_dict = None
+
+ def iter_batched_inputs():
+ nonlocal current_idx
+ for key, example in iterator:
+ # If `batched`, first build the batch, if `batch_size` is None or <=0, then the batch is the whole dataset
+ iterator_batch = (
+ iterator
+ if self.batch_size is None or self.batch_size <= 0
+ else islice(iterator, self.batch_size - 1)
+ )
+ key_examples_list = [(key, example)] + list(iterator_batch)
+ keys, examples = zip(*key_examples_list)
+ # the new key is the concatenation of the examples keys from the batch
+ key = "_".join(str(key) for key in keys)
+ if (
+ self.drop_last_batch
+ and self.batch_size is not None
+ and self.batch_size > 0
+ and len(examples) < self.batch_size
+ ): # ignore last batch
+ return
+ batch = _examples_to_batch(examples)
+ # we need to format here in case we need to stack tensors together
+ batch = format_dict(batch) if format_dict else batch
+ indices = [current_idx + i for i in range(len(key_examples_list))]
+ current_idx += len(indices)
+ yield indices, (key, batch)
+
+ def iter_inputs():
+ nonlocal current_idx
+ for key, example in iterator:
+ # If not batched, we can apply the transform and yield the example directly
+ # first copy the example, since we might drop some keys
+ example = dict(example)
+ # no need to do formatting here
+ current_idx += 1
+ yield current_idx - 1, (key, example)
+
+ def validate_function_output(processed_inputs):
+ if self.batched and processed_inputs:
+ first_col = next(iter(processed_inputs))
+ bad_cols = [
+ col for col in processed_inputs if len(processed_inputs[col]) != len(processed_inputs[first_col])
+ ]
+ if bad_cols:
+ raise ValueError(
+ f"Column lengths mismatch: columns {bad_cols} have length {[len(processed_inputs[col]) for col in bad_cols]} "
+ f"while {first_col} has length {len(processed_inputs[first_col])}."
+ )
+
+ def prepare_inputs(key_example, indices):
+ key, example = key_example
+ fn_args = [example] if self.input_columns is None else [example[col] for col in self.input_columns]
+ additional_args = ()
+ if self.with_indices:
+ fn_args += (indices,)
+ inputs = dict(example)
+ return inputs, fn_args, additional_args, self.fn_kwargs
+
+ def prepare_outputs(key_example, inputs, processed_inputs):
+ validate_function_output(processed_inputs)
+ # this logic mimics the one in Dataset.map
+ if self.remove_columns:
+ for c in self.remove_columns:
+ if c in inputs:
+ del inputs[c]
+ if processed_inputs is key_example[1] and c in processed_inputs:
+ del processed_inputs[c]
+ transformed_inputs = {**inputs, **processed_inputs}
+ # no need to do features decoding here
+ return transformed_inputs
+
+ def apply_function(key_example, indices):
+ """Utility to apply the function on a selection of columns."""
+ inputs, fn_args, additional_args, fn_kwargs = prepare_inputs(key_example, indices)
+ processed_inputs = self.function(*fn_args, *additional_args, **fn_kwargs)
+ return prepare_outputs(key_example, inputs, processed_inputs)
+
+ async def async_apply_function(key_example, indices):
+ """Utility to apply the function on a selection of columns. Same code but async"""
+ inputs, fn_args, additional_args, fn_kwargs = prepare_inputs(key_example, indices)
+ processed_inputs = await self.function(*fn_args, *additional_args, **fn_kwargs)
+ return prepare_outputs(key_example, inputs, processed_inputs)
+
+ tasks: list[asyncio.Task] = []
+ if inspect.iscoroutinefunction(self.function):
+ try:
+ loop = asyncio.get_running_loop()
+ except RuntimeError:
+ loop = asyncio.new_event_loop()
+ self._owned_loops_and_tasks.append((loop, tasks))
+ else:
+ loop = None
+
+ def iter_outputs():
+ nonlocal tasks, loop
+ inputs_iterator = iter_batched_inputs() if self.batched else iter_inputs()
+ if inspect.iscoroutinefunction(self.function):
+ if self._state_dict:
+ previous_state = self.ex_iterable.state_dict()
+ self._state_dict["previous_state"] = previous_state
+ previous_state_task = None
+ previous_state_example_idx = self._state_dict["previous_state_example_idx"]
+ indices: Union[list[int], list[list[int]]] = []
+ for i, key_example in inputs_iterator:
+ indices.append(i)
+ tasks.append(loop.create_task(async_apply_function(key_example, i)))
+ # keep the total active tasks under a certain number
+ if len(tasks) >= self.max_num_running_async_map_functions_in_parallel:
+ done, pending = loop.run_until_complete(
+ asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)
+ )
+ while tasks and len(pending) >= self.max_num_running_async_map_functions_in_parallel:
+ done, pending = loop.run_until_complete(
+ asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)
+ )
+ if len(tasks) >= 10 * self.max_num_running_async_map_functions_in_parallel:
+ loop.run_until_complete(tasks[0])
+ # yield finished tasks
+ while tasks and tasks[0].done():
+ i, task = indices.pop(0), tasks.pop(0)
+ yield i, task.result()
+ if self._state_dict and task is previous_state_task:
+ self._state_dict["previous_state"] = previous_state
+ self._state_dict["num_examples_since_previous_state"] = 0
+ self._state_dict["previous_state_example_idx"] = previous_state_example_idx
+ previous_state, previous_state_task = None, None
+ # checkpoint
+ if self._state_dict and previous_state_task is None and tasks:
+ previous_state = self.ex_iterable.state_dict()
+ previous_state_task = tasks[-1]
+ previous_state_example_idx = current_idx
+ while tasks:
+ yield indices[0], loop.run_until_complete(tasks[0])
+ indices.pop(0), tasks.pop(0)
+ else:
+ if self._state_dict:
+ if self.batched:
+ self._state_dict["previous_state"] = self.ex_iterable.state_dict()
+ self._state_dict["num_examples_since_previous_state"] = 0
+ self._state_dict["previous_state_example_idx"] = current_idx
+ for i, key_example in inputs_iterator:
+ if self._state_dict:
+ if not self.batched:
+ self._state_dict["previous_state_example_idx"] = current_idx
+ yield i, apply_function(key_example, i)
+ if self._state_dict:
+ if self.batched:
+ self._state_dict["previous_state"] = self.ex_iterable.state_dict()
+ self._state_dict["num_examples_since_previous_state"] = 0
+ self._state_dict["previous_state_example_idx"] = current_idx
+
+ try:
+ outputs = iter_outputs()
+ if self.batched:
+ outputs = (
+ (key, transformed_example)
+ for key, transformed_batch in outputs
+ for transformed_example in _batch_to_examples(transformed_batch)
+ )
+ for key, transformed_example in outputs:
+ if self._state_dict and self._state_dict["previous_state"] is not None:
+ self._state_dict["num_examples_since_previous_state"] += 1
+ if num_examples_to_skip > 0:
+ num_examples_to_skip -= 1
+ continue
+ yield key, transformed_example
+ except (Exception, KeyboardInterrupt):
+ if loop:
+ logger.debug(f"Canceling {len(tasks)} async tasks.")
+ for task in tasks:
+ task.cancel(msg="KeyboardInterrupt")
+ try:
+ loop.run_until_complete(asyncio.gather(*tasks))
+ except (asyncio.CancelledError, ValueError):
+ logger.debug("Tasks canceled.")
+ raise
+
+ def _iter_arrow(self, max_chunksize: Optional[int] = None) -> Iterator[tuple[Key, pa.Table]]:
+ formatter: TableFormatter = get_formatter(self.formatting.format_type) if self.formatting else ArrowFormatter()
+ if self.ex_iterable.iter_arrow:
+ iterator = self.ex_iterable.iter_arrow()
+ else:
+ iterator = _convert_to_arrow(
+ self.ex_iterable,
+ batch_size=self.batch_size if self.batched else 1,
+ drop_last_batch=self.drop_last_batch,
+ )
+ if self._state_dict and self._state_dict["previous_state"]:
+ self.ex_iterable.load_state_dict(self._state_dict["previous_state"])
+ num_examples_to_skip = self._state_dict["num_examples_since_previous_state"]
+ else:
+ num_examples_to_skip = 0
+ if self._state_dict and max_chunksize is not None:
+ self._state_dict["previous_state"] = self.ex_iterable.state_dict()
+ self._state_dict["num_examples_since_previous_state"] = 0
+ current_idx = self._state_dict["previous_state_example_idx"] if self._state_dict else 0
+ for key, pa_table in iterator:
+ if (
+ self.batched
+ and self.batch_size is not None
+ and len(pa_table) < self.batch_size
+ and self.drop_last_batch
+ ):
+ return
+ # first build the batch
+ function_args = (
+ [formatter.format_batch(pa_table)]
+ if self.input_columns is None
+ else [pa_table[col] for col in self.input_columns]
+ )
+ if self.with_indices:
+ if self.batched:
+ function_args.append([current_idx + i for i in range(len(pa_table))])
+ else:
+ function_args.append(current_idx)
+ # then apply the transform
+ output = self.function(*function_args, **self.fn_kwargs)
+ output_table = _table_output_to_arrow(output)
+ if not isinstance(output_table, pa.Table):
+ raise TypeError(
+ f"Provided `function` which is applied to {formatter.table_type} returns a variable of type "
+ f"{type(output)}. Make sure provided `function` returns a {formatter.table_type} to update the dataset."
+ )
+ # we don't need to merge results for consistency with Dataset.map which merges iif both input and output are dicts
+ # then remove the unwanted columns
+ if self.remove_columns:
+ for column in self.remove_columns:
+ if column in output_table.column_names:
+ output_table = output_table.remove_column(output_table.column_names.index(column))
+ # return output
+ if max_chunksize is None:
+ current_idx += len(pa_table)
+ if self._state_dict:
+ self._state_dict["previous_state_example_idx"] += len(pa_table)
+ yield key, output_table
+ else:
+ for i, pa_subtable in enumerate(output_table.to_reader(max_chunksize=max_chunksize)):
+ current_idx += 1
+ if self._state_dict:
+ self._state_dict["num_examples_since_previous_state"] += 1
+ if num_examples_to_skip > 0:
+ num_examples_to_skip -= 1
+ continue
+ yield f"{key}_{i}", pa_subtable
+ if self._state_dict:
+ self._state_dict["previous_state"] = self.ex_iterable.state_dict()
+ self._state_dict["num_examples_since_previous_state"] = 0
+ self._state_dict["previous_state_example_idx"] += len(pa_table)
+
+ def shuffle_data_sources(self, generator: np.random.Generator) -> "MappedExamplesIterable":
+ """Shuffle the wrapped examples iterable."""
+ return MappedExamplesIterable(
+ self.ex_iterable.shuffle_data_sources(generator),
+ function=self.function,
+ with_indices=self.with_indices,
+ input_columns=self.input_columns,
+ batched=self.batched,
+ batch_size=self.batch_size,
+ drop_last_batch=self.drop_last_batch,
+ remove_columns=self.remove_columns,
+ fn_kwargs=self.fn_kwargs,
+ formatting=self.formatting,
+ features=self.features,
+ max_num_running_async_map_functions_in_parallel=self.max_num_running_async_map_functions_in_parallel,
+ )
+
+ def shard_data_sources(self, num_shards: int, index: int, contiguous=True) -> "MappedExamplesIterable":
+ """Keep only the requested shard."""
+ return MappedExamplesIterable(
+ self.ex_iterable.shard_data_sources(num_shards, index, contiguous=contiguous),
+ function=self.function,
+ with_indices=self.with_indices,
+ input_columns=self.input_columns,
+ batched=self.batched,
+ batch_size=self.batch_size,
+ drop_last_batch=self.drop_last_batch,
+ remove_columns=self.remove_columns,
+ fn_kwargs=self.fn_kwargs,
+ formatting=self.formatting,
+ features=self.features,
+ max_num_running_async_map_functions_in_parallel=self.max_num_running_async_map_functions_in_parallel,
+ )
+
+ @property
+ def num_shards(self) -> int:
+ return self.ex_iterable.num_shards
+
+
+def _add_mask(
+ input: Union[dict, pa.Table],
+ mask: Union[bool, list, pa.Array, pa.ChunkedArray, pa.BooleanScalar],
+ mask_column_name: str,
+):
+ if isinstance(input, pa.Table):
+ if not isinstance(mask, (list, pa.Array, pa.ChunkedArray)):
+ mask = pa.array([mask], type=pa.bool_())
+ return input.append_column(mask_column_name, mask)
+ else:
+ return {mask_column_name: mask}
+
+
+def add_mask(mask_function: Callable, input: Union[dict, pa.Table], *args, mask_column_name: str, **kwargs):
+ mask = mask_function(input, *args, **kwargs)
+ return _add_mask(input, mask, mask_column_name)
+
+
+async def async_add_mask(
+ mask_function: Callable, input: Union[dict, pa.Table], *args, mask_column_name: str, **kwargs
+):
+ mask = await mask_function(input, *args, **kwargs)
+ return _add_mask(input, mask, mask_column_name)
+
+
+class FilteredExamplesIterable(MappedExamplesIterable):
+ mask_column_name = "===MASK==="
+
+ def __init__(
+ self,
+ ex_iterable: _BaseExamplesIterable,
+ function: Callable,
+ with_indices: bool = False,
+ input_columns: Optional[list[str]] = None,
+ batched: bool = False,
+ batch_size: Optional[int] = 1000,
+ fn_kwargs: Optional[dict] = None,
+ formatting: Optional["FormattingConfig"] = None,
+ ):
+ self.mask_function = function
+ if ex_iterable.is_typed:
+ features = Features({**ex_iterable.features, self.mask_column_name: Value("bool")})
+ else:
+ features = None
+ super().__init__(
+ ex_iterable=ex_iterable,
+ function=partial(
+ async_add_mask if inspect.iscoroutinefunction(function) else add_mask,
+ function,
+ mask_column_name=self.mask_column_name,
+ ),
+ with_indices=with_indices,
+ input_columns=input_columns,
+ batched=batched,
+ batch_size=batch_size,
+ fn_kwargs=fn_kwargs,
+ formatting=formatting,
+ features=features,
+ )
+
+ def _iter(self):
+ for key, example in super()._iter():
+ example = dict(example)
+ if example.pop(self.mask_column_name):
+ yield key, example
+
+ def _iter_arrow(self, max_chunksize: Optional[int] = None):
+ for key, pa_table in super()._iter_arrow(max_chunksize=max_chunksize):
+ mask = pa_table[self.mask_column_name]
+ yield key, pa_table.drop(self.mask_column_name).filter(mask)
+
+ def shuffle_data_sources(self, seed: Optional[int]) -> "FilteredExamplesIterable":
+ """Shuffle the wrapped examples iterable."""
+ return FilteredExamplesIterable(
+ self.ex_iterable.shuffle_data_sources(seed),
+ function=self.mask_function,
+ with_indices=self.with_indices,
+ input_columns=self.input_columns,
+ batched=self.batched,
+ batch_size=self.batch_size,
+ fn_kwargs=self.fn_kwargs,
+ formatting=self.formatting,
+ )
+
+ def shard_data_sources(self, num_shards: int, index: int, contiguous=True) -> "FilteredExamplesIterable":
+ """Keep only the requested shard."""
+ return FilteredExamplesIterable(
+ self.ex_iterable.shard_data_sources(num_shards, index, contiguous=contiguous),
+ function=self.mask_function,
+ with_indices=self.with_indices,
+ input_columns=self.input_columns,
+ batched=self.batched,
+ batch_size=self.batch_size,
+ fn_kwargs=self.fn_kwargs,
+ formatting=self.formatting,
+ )
+
+ @property
+ def num_shards(self) -> int:
+ return self.ex_iterable.num_shards
+
+
+class BufferShuffledExamplesIterable(_BaseExamplesIterable):
+ def __init__(self, ex_iterable: _BaseExamplesIterable, buffer_size: int, generator: np.random.Generator):
+ super().__init__()
+ self.ex_iterable = ex_iterable
+ self.buffer_size = buffer_size
+ self.generator = generator
+
+ def shift_rngs(self, value: int) -> "_BaseExamplesIterable":
+ new_seed = self.generator.bit_generator.state["state"]["state"] + value
+ return BufferShuffledExamplesIterable(
+ ex_iterable=self.ex_iterable,
+ buffer_size=self.buffer_size,
+ generator=np.random.default_rng(seed=new_seed),
+ )
+
+ @property
+ def is_typed(self):
+ return self.ex_iterable.is_typed
+
+ @property
+ def features(self):
+ return self.ex_iterable.features
+
+ @property
+ def iter_arrow(self):
+ return self._iter_arrow if self.ex_iterable.iter_arrow else None
+
+ def _init_state_dict(self) -> dict:
+ self._state_dict = self.ex_iterable._init_state_dict()
+ self._original_state_dict = self.state_dict()
+ return self._state_dict
+
+ def load_state_dict(self, state_dict: dict) -> dict:
+ if self._state_dict:
+ if state_dict != self._original_state_dict:
+ logger.warning(
+ "Loading a state dict of a shuffle buffer of a dataset without the buffer content."
+ "The shuffle buffer will be refilled before starting to yield new examples."
+ )
+ return super().load_state_dict(state_dict)
+
+ @staticmethod
+ def _iter_random_indices(rng: np.random.Generator, buffer_size: int, random_batch_size=1000) -> Iterator[int]:
+ while True:
+ yield from (int(i) for i in rng.integers(0, buffer_size, size=random_batch_size))
+
+ def __iter__(self):
+ buffer_size = self.buffer_size
+ rng = deepcopy(self.generator)
+ indices_iterator = self._iter_random_indices(rng, buffer_size)
+ # this is the shuffle buffer that we keep in memory
+ mem_buffer = []
+ for x in self.ex_iterable:
+ if len(mem_buffer) == buffer_size: # if the buffer is full, pick and example from it
+ i = next(indices_iterator)
+ yield mem_buffer[i]
+ mem_buffer[i] = x # replace the picked example by a new one
+ else: # otherwise, keep filling the buffer
+ mem_buffer.append(x)
+ # when we run out of examples, we shuffle the remaining examples in the buffer and yield them
+ rng.shuffle(mem_buffer)
+ yield from mem_buffer
+
+ def _iter_arrow(self):
+ buffer_size = self.buffer_size
+ rng = deepcopy(self.generator)
+ indices_iterator = self._iter_random_indices(rng, buffer_size)
+ # this is the shuffle buffer that we keep in memory
+ mem_buffer = []
+ for key, pa_table in self.ex_iterable.iter_arrow():
+ if len(mem_buffer) == buffer_size: # if the buffer is full, pick and example from it
+ i = next(indices_iterator)
+ yield mem_buffer[i]
+ mem_buffer[i] = (key, pa_table) # replace the picked example by a new one
+ else: # otherwise, keep filling the buffer
+ mem_buffer.append((key, pa_table))
+ # when we run out of examples, we shuffle the remaining examples in the buffer and yield them
+ rng.shuffle(mem_buffer)
+ yield from mem_buffer
+
+ def shuffle_data_sources(self, generator: np.random.Generator) -> "BufferShuffledExamplesIterable":
+ """Shuffle the wrapped examples iterable as well as the shuffling buffer."""
+ return BufferShuffledExamplesIterable(
+ self.ex_iterable.shuffle_data_sources(generator), buffer_size=self.buffer_size, generator=generator
+ )
+
+ def shard_data_sources(self, num_shards: int, index: int, contiguous=True) -> "BufferShuffledExamplesIterable":
+ """Keep only the requested shard."""
+ return BufferShuffledExamplesIterable(
+ self.ex_iterable.shard_data_sources(num_shards, index, contiguous=contiguous),
+ buffer_size=self.buffer_size,
+ generator=self.generator,
+ )
+
+ @property
+ def num_shards(self) -> int:
+ return self.ex_iterable.num_shards
+
+
+class SkipExamplesIterable(_BaseExamplesIterable):
+ def __init__(
+ self,
+ ex_iterable: _BaseExamplesIterable,
+ n: int,
+ block_sources_order_when_shuffling: bool = True,
+ split_when_sharding: bool = True,
+ ):
+ super().__init__()
+ self.ex_iterable = ex_iterable
+ self.n = n
+ self.block_sources_order_when_shuffling = block_sources_order_when_shuffling
+ self.split_when_sharding = split_when_sharding
+ # TODO(QL): implement iter_arrow
+
+ @property
+ def is_typed(self):
+ return self.ex_iterable.is_typed
+
+ @property
+ def features(self):
+ return self.ex_iterable.features
+
+ def _init_state_dict(self) -> dict:
+ self._state_dict = {
+ "skipped": False,
+ "examples_iterable": self.ex_iterable._init_state_dict(),
+ "type": self.__class__.__name__,
+ }
+ return self._state_dict
+
+ def __iter__(self):
+ ex_iterable_idx_start = 0 if self._state_dict and self._state_dict["skipped"] else self.n
+ if self._state_dict:
+ self._state_dict["skipped"] = True
+ yield from islice(self.ex_iterable, ex_iterable_idx_start, None)
+
+ @staticmethod
+ def split_number(num, n):
+ quotient = num // n
+ remainder = num % n
+ result = [quotient] * n
+ for i in range(remainder):
+ result[i] += 1
+ return result
+
+ def shuffle_data_sources(self, generator: np.random.Generator) -> "SkipExamplesIterable":
+ """May not shuffle the wrapped examples iterable since it would skip examples from other shards instead."""
+ if self.block_sources_order_when_shuffling:
+ return self
+ else:
+ return SkipExamplesIterable(
+ self.ex_iterable.shuffle_data_sources(generator),
+ n=self.n,
+ block_sources_order_when_shuffling=self.block_sources_order_when_shuffling,
+ split_when_sharding=self.split_when_sharding,
+ )
+
+ def shard_data_sources(self, num_shards: int, index: int, contiguous=True) -> "SkipExamplesIterable":
+ """Keep only the requested shard."""
+ if self.split_when_sharding:
+ return SkipExamplesIterable(
+ self.ex_iterable.shard_data_sources(num_shards, index, contiguous=contiguous),
+ n=self.split_number(self.n, num_shards)[index],
+ block_sources_order_when_shuffling=self.block_sources_order_when_shuffling,
+ split_when_sharding=self.split_when_sharding,
+ )
+ else:
+ return self
+
+ @property
+ def num_shards(self) -> int:
+ return self.ex_iterable.num_shards
+
+
+class RepeatExamplesIterable(_BaseExamplesIterable):
+ """
+ Iterable that repeats the underlying iterable a given number of times.
+ """
+
+ def __init__(
+ self,
+ ex_iterable: _BaseExamplesIterable,
+ num_times: Optional[int],
+ ):
+ super().__init__()
+ self.ex_iterable = ex_iterable
+ self.num_times = num_times
+
+ def _init_state_dict(self) -> dict:
+ self._state_dict = {
+ "repeat_index": 0,
+ "examples_iterable": self.ex_iterable._init_state_dict(),
+ "type": self.__class__.__name__,
+ }
+ return self._state_dict
+
+ def __iter__(self):
+ repeat_index = self._state_dict["repeat_index"] if self._state_dict else 0
+ while True:
+ if self.num_times is not None and repeat_index >= max(self.num_times, 0):
+ break
+ yield from self.ex_iterable
+ repeat_index += 1
+ if self._state_dict:
+ self._state_dict["repeat_index"] = repeat_index
+ self._state_dict["examples_iterable"] = self.ex_iterable._init_state_dict()
+
+ def shuffle_data_sources(self, generator: np.random.Generator) -> "RepeatExamplesIterable":
+ """Shuffle the underlying iterable, then repeat."""
+ return RepeatExamplesIterable(self.ex_iterable.shuffle_data_sources(generator), num_times=self.num_times)
+
+ def shard_data_sources(self, num_shards: int, index: int, contiguous=True) -> "RepeatExamplesIterable":
+ """Shard, then repeat shards."""
+ return RepeatExamplesIterable(
+ self.ex_iterable.shard_data_sources(num_shards, index, contiguous=contiguous),
+ num_times=self.num_times,
+ )
+
+ @property
+ def num_shards(self) -> int:
+ return self.ex_iterable.num_shards
+
+
+class TakeExamplesIterable(_BaseExamplesIterable):
+ def __init__(
+ self,
+ ex_iterable: _BaseExamplesIterable,
+ n: int,
+ block_sources_order_when_shuffling: bool = True,
+ split_when_sharding: bool = True,
+ ):
+ super().__init__()
+ self.ex_iterable = ex_iterable
+ self.n = n
+ self.block_sources_order_when_shuffling = block_sources_order_when_shuffling
+ self.split_when_sharding = split_when_sharding
+ # TODO(QL): implement iter_arrow
+
+ @property
+ def is_typed(self):
+ return self.ex_iterable.is_typed
+
+ @property
+ def features(self):
+ return self.ex_iterable.features
+
+ def _init_state_dict(self) -> dict:
+ self._state_dict = {
+ "num_taken": 0,
+ "examples_iterable": self.ex_iterable._init_state_dict(),
+ "type": self.__class__.__name__,
+ }
+ return self._state_dict
+
+ def __iter__(self):
+ ex_iterable_num_taken = self._state_dict["num_taken"] if self._state_dict else 0
+ for key_example in islice(self.ex_iterable, self.n - ex_iterable_num_taken):
+ if self._state_dict:
+ self._state_dict["num_taken"] += 1
+ yield key_example
+
+ @staticmethod
+ def split_number(num, n):
+ quotient = num // n
+ remainder = num % n
+ result = [quotient] * n
+ for i in range(remainder):
+ result[i] += 1
+ return result
+
+ def shuffle_data_sources(self, generator: np.random.Generator) -> "TakeExamplesIterable":
+ """May not shuffle the wrapped examples iterable since it would take examples from other shards instead."""
+ if self.block_sources_order_when_shuffling:
+ return self
+ else:
+ return TakeExamplesIterable(
+ self.ex_iterable.shuffle_data_sources(generator),
+ n=self.n,
+ block_sources_order_when_shuffling=self.block_sources_order_when_shuffling,
+ split_when_sharding=self.split_when_sharding,
+ )
+
+ def shard_data_sources(self, num_shards: int, index: int, contiguous=True) -> "TakeExamplesIterable":
+ """Keep only the requested shard."""
+ if self.split_when_sharding:
+ return TakeExamplesIterable(
+ self.ex_iterable.shard_data_sources(num_shards, index, contiguous=contiguous),
+ n=self.split_number(self.n, num_shards)[index],
+ block_sources_order_when_shuffling=self.block_sources_order_when_shuffling,
+ split_when_sharding=self.split_when_sharding,
+ )
+ else:
+ return TakeExamplesIterable(
+ self.ex_iterable.shard_data_sources(num_shards, index, contiguous=contiguous),
+ n=self.n,
+ block_sources_order_when_shuffling=self.block_sources_order_when_shuffling,
+ split_when_sharding=self.split_when_sharding,
+ )
+
+ @property
+ def num_shards(self) -> int:
+ return self.ex_iterable.num_shards
+
+
+def _apply_feature_types_on_example(
+ example: dict, features: Features, token_per_repo_id: dict[str, Union[str, bool, None]]
+) -> dict:
+ example = dict(example)
+ # add missing columns
+ for column_name in features:
+ if column_name not in example:
+ example[column_name] = None
+ # we encode the example for ClassLabel feature types for example
+ encoded_example = features.encode_example(example)
+ # Decode example for Audio feature, e.g.
+ decoded_example = features.decode_example(encoded_example, token_per_repo_id=token_per_repo_id)
+ return decoded_example
+
+
+def _apply_feature_types_on_batch(
+ batch: dict, features: Features, token_per_repo_id: dict[str, Union[str, bool, None]]
+) -> dict:
+ batch = dict(batch)
+ # add missing columns
+ n_examples = len(batch[next(iter(batch))])
+ for column_name in features:
+ if column_name not in batch:
+ batch[column_name] = [None] * n_examples
+ # we encode the batch for ClassLabel feature types for example
+ encoded_batch = features.encode_batch(batch)
+ # Decode batch for Audio feature, e.g.
+ decoded_batch = features.decode_batch(encoded_batch, token_per_repo_id=token_per_repo_id)
+ return decoded_batch
+
+
+@dataclass
+class FormattingConfig:
+ format_type: Optional[str]
+
+ @property
+ def is_table(self) -> bool:
+ return isinstance(get_formatter(self.format_type), TableFormatter)
+
+ @property
+ def is_tensor(self) -> bool:
+ return isinstance(get_formatter(self.format_type), TensorFormatter)
+
+
+class FormattedExamplesIterable(_BaseExamplesIterable):
+ def __init__(
+ self,
+ ex_iterable: _BaseExamplesIterable,
+ formatting: Optional[FormattingConfig],
+ features: Optional[Features],
+ token_per_repo_id: dict[str, Union[str, bool, None]],
+ ):
+ super().__init__()
+ self.ex_iterable = ex_iterable
+ self._features = features
+ self.formatting = formatting
+ self.token_per_repo_id = token_per_repo_id
+
+ @property
+ def iter_arrow(self):
+ if self.ex_iterable.iter_arrow and (not self.formatting or self.formatting.is_table):
+ return self._iter_arrow
+
+ @property
+ def is_typed(self):
+ return self.ex_iterable.is_typed or self._features is not None
+
+ @property
+ def features(self):
+ return self._features
+
+ def _init_state_dict(self) -> dict:
+ self._state_dict = self.ex_iterable._init_state_dict()
+ return self._state_dict
+
+ def __iter__(self):
+ if not self.formatting or self.formatting.is_table:
+ formatter = PythonFormatter(
+ features=self._features if not self.ex_iterable.is_typed else None,
+ token_per_repo_id=self.token_per_repo_id,
+ )
+ else:
+ formatter = get_formatter(
+ self.formatting.format_type,
+ features=self._features if not self.ex_iterable.is_typed else None,
+ token_per_repo_id=self.token_per_repo_id,
+ )
+ if self.ex_iterable.iter_arrow:
+ # feature casting (inc column addition) handled within self._iter_arrow()
+ for key, pa_table in self._iter_arrow():
+ batch = formatter.format_batch(pa_table)
+ for example in _batch_to_examples(batch):
+ yield key, example
+ else:
+ format_dict = (
+ formatter.recursive_tensorize
+ if isinstance(formatter, TensorFormatter)
+ else None # cast in case features is None
+ )
+ for key, example in self.ex_iterable:
+ # don't apply feature types if already applied by ex_iterable (e.g. in case of chained with_format)
+ if self.features and not self.ex_iterable.is_typed:
+ example = _apply_feature_types_on_example(
+ example, self.features, token_per_repo_id=self.token_per_repo_id
+ )
+ if format_dict:
+ example = format_dict(example)
+ yield key, example
+
+ def _iter_arrow(self) -> Iterator[tuple[Key, pa.Table]]:
+ if not self.features:
+ yield from self.ex_iterable._iter_arrow()
+ for key, pa_table in self.ex_iterable._iter_arrow():
+ columns = set(pa_table.column_names)
+ schema = self.features.arrow_schema
+ # add missing columns
+ for column_name in self.features:
+ if column_name not in columns:
+ col = pa.NullArray.from_buffers(pa.null(), len(pa_table), [None])
+ pa_table = pa_table.append_column(column_name, col)
+ if pa_table.schema != schema:
+ pa_table = cast_table_to_features(pa_table, self.features)
+ yield key, pa_table
+
+ def shuffle_data_sources(self, generator: np.random.Generator) -> "FormattedExamplesIterable":
+ """Shuffle the wrapped examples iterable."""
+ return FormattedExamplesIterable(
+ self.ex_iterable.shuffle_data_sources(generator),
+ features=self.features,
+ token_per_repo_id=self.token_per_repo_id,
+ formatting=self.formatting,
+ )
+
+ def shard_data_sources(self, num_shards: int, index: int, contiguous=True) -> "FormattedExamplesIterable":
+ """Keep only the requested shard."""
+ return FormattedExamplesIterable(
+ self.ex_iterable.shard_data_sources(num_shards, index, contiguous=contiguous),
+ features=self.features,
+ token_per_repo_id=self.token_per_repo_id,
+ formatting=self.formatting,
+ )
+
+ @property
+ def num_shards(self) -> int:
+ return self.ex_iterable.num_shards
+
+
+@dataclass
+class ShufflingConfig:
+ generator: np.random.Generator
+ _original_seed: Optional[int] = None
+
+
+@dataclass
+class DistributedConfig:
+ rank: int
+ world_size: int
+
+
+def _maybe_add_torch_iterable_dataset_parent_class(cls):
+ """Add torch.utils.data.IterableDataset as a parent class if 'torch' is available"""
+ if config.TORCH_AVAILABLE:
+ import torch.utils.data
+
+ if torch.utils.data.IterableDataset not in cls.__bases__:
+ cls.__bases__ += (torch.utils.data.IterableDataset,)
+
+
+def _maybe_share_with_torch_persistent_workers(value: Union[int, "torch.Tensor"]) -> Union[int, "torch.Tensor"]:
+ if config.TORCH_AVAILABLE:
+ import torch
+
+ if isinstance(value, torch.Tensor):
+ return value.share_memory_()
+ else:
+ return torch.tensor(value).share_memory_()
+ else:
+ return value
+
+
+class IterableColumn:
+ """
+ An iterable for a specific column of an [`IterableDataset`].
+
+ Example:
+
+ Iterate on the texts of the "text" column of a dataset:
+
+ ```python
+ for text in dataset["text"]:
+ ...
+ ```
+
+ It also works with nested columns:
+
+ ```python
+ for source in dataset["metadata"]["source"]:
+ ...
+ ```
+ """
+
+ def __init__(self, source: Union["IterableDataset", "IterableColumn"], column_name: str):
+ self.source = source
+ self.column_name = column_name
+
+ def __iter__(self) -> Iterator[Any]:
+ for example in self.source:
+ yield example[self.column_name]
+
+ def __getitem__(self, column_name: str) -> "IterableColumn":
+ return IterableColumn(self, column_name)
+
+
+class IterableDataset(DatasetInfoMixin):
+ """A Dataset backed by an iterable."""
+
+ def __init__(
+ self,
+ ex_iterable: _BaseExamplesIterable,
+ info: Optional[DatasetInfo] = None,
+ split: Optional[NamedSplit] = None,
+ formatting: Optional[FormattingConfig] = None,
+ shuffling: Optional[ShufflingConfig] = None,
+ distributed: Optional[DistributedConfig] = None,
+ token_per_repo_id: Optional[dict[str, Union[str, bool, None]]] = None,
+ ):
+ if distributed and distributed.world_size > 1 and shuffling and shuffling._original_seed is None:
+ raise RuntimeError(
+ "The dataset doesn't have a fixed random seed across nodes to shuffle and split the list of dataset shards by node. "
+ "Please pass e.g. `seed=42` in `.shuffle()` to make all the nodes use the same seed. "
+ )
+
+ info = info.copy() if info is not None else DatasetInfo()
+ DatasetInfoMixin.__init__(self, info=info, split=split)
+
+ self._ex_iterable = copy.copy(ex_iterable)
+ self._formatting = formatting
+ self._shuffling = shuffling
+ self._distributed = distributed
+ self._token_per_repo_id: dict[str, Union[str, bool, None]] = token_per_repo_id or {}
+ self._epoch: Union[int, "torch.Tensor"] = _maybe_share_with_torch_persistent_workers(0)
+ self._starting_state_dict: Optional[dict] = None
+ self.__hffs_cache = HfFileSystem._cache # keep the cache on pickling (e.g. for dataloader workers)
+ self._prepare_ex_iterable_for_iteration() # set state_dict
+ _maybe_add_torch_iterable_dataset_parent_class(self.__class__) # subclass of torch IterableDataset
+
+ @property
+ def num_columns(self) -> Optional[int]:
+ """Number of columns in the dataset.
+ This can be None if the dataset has unknown features (e.g. after a map() operation).
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> ds.num_columns
+ 2
+ ```
+ """
+ return None if self.features is None else len(self.features)
+
+ @property
+ def column_names(self) -> Optional[list[str]]:
+ """Names of the columns in the dataset.
+ This can be None if the dataset has unknown features (e.g. after a map() operation).
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation", streaming=True)
+ >>> ds.column_names
+ ['text', 'label']
+ ```
+ """
+ return None if self.features is None else list(self.features)
+
+ def state_dict(self) -> dict:
+ """Get the current state_dict of the dataset.
+ It corresponds to the state at the latest example it yielded.
+
+ Resuming returns exactly where the checkpoint was saved except in two cases:
+
+ 1. examples from shuffle buffers are lost when resuming and the buffers are refilled with new data
+ 2. combinations of `.with_format(arrow)` and batched `.map()` may skip one batch.
+
+ Returns:
+ `dict`
+
+ Example:
+
+ ```py
+ >>> from datasets import Dataset, concatenate_datasets
+ >>> ds = Dataset.from_dict({"a": range(6)}).to_iterable_dataset(num_shards=3)
+ >>> for idx, example in enumerate(ds):
+ ... print(example)
+ ... if idx == 2:
+ ... state_dict = ds.state_dict()
+ ... print("checkpoint")
+ ... break
+ >>> ds.load_state_dict(state_dict)
+ >>> print(f"restart from checkpoint")
+ >>> for example in ds:
+ ... print(example)
+ ```
+
+ which returns:
+ ```
+ {'a': 0}
+ {'a': 1}
+ {'a': 2}
+ checkpoint
+ restart from checkpoint
+ {'a': 3}
+ {'a': 4}
+ {'a': 5}
+ ```
+
+ ```py
+ >>> from torchdata.stateful_dataloader import StatefulDataLoader
+ >>> ds = load_dataset("deepmind/code_contests", streaming=True, split="train")
+ >>> dataloader = StatefulDataLoader(ds, batch_size=32, num_workers=4)
+ >>> # checkpoint
+ >>> state_dict = dataloader.state_dict() # uses ds.state_dict() under the hood
+ >>> # resume from checkpoint
+ >>> dataloader.load_state_dict(state_dict) # uses ds.load_state_dict() under the hood
+ ```
+ """
+ return copy.deepcopy(self._state_dict)
+
+ def load_state_dict(self, state_dict: dict) -> None:
+ """Load the state_dict of the dataset.
+ The iteration will restart at the next example from when the state was saved.
+
+ Resuming returns exactly where the checkpoint was saved except in two cases:
+
+ 1. examples from shuffle buffers are lost when resuming and the buffers are refilled with new data
+ 2. combinations of `.with_format(arrow)` and batched `.map()` may skip one batch.
+
+ Example:
+
+ ```py
+ >>> from datasets import Dataset, concatenate_datasets
+ >>> ds = Dataset.from_dict({"a": range(6)}).to_iterable_dataset(num_shards=3)
+ >>> for idx, example in enumerate(ds):
+ ... print(example)
+ ... if idx == 2:
+ ... state_dict = ds.state_dict()
+ ... print("checkpoint")
+ ... break
+ >>> ds.load_state_dict(state_dict)
+ >>> print(f"restart from checkpoint")
+ >>> for example in ds:
+ ... print(example)
+ ```
+
+ which returns:
+ ```
+ {'a': 0}
+ {'a': 1}
+ {'a': 2}
+ checkpoint
+ restart from checkpoint
+ {'a': 3}
+ {'a': 4}
+ {'a': 5}
+ ```
+
+ ```py
+ >>> from torchdata.stateful_dataloader import StatefulDataLoader
+ >>> ds = load_dataset("deepmind/code_contests", streaming=True, split="train")
+ >>> dataloader = StatefulDataLoader(ds, batch_size=32, num_workers=4)
+ >>> # checkpoint
+ >>> state_dict = dataloader.state_dict() # uses ds.state_dict() under the hood
+ >>> # resume from checkpoint
+ >>> dataloader.load_state_dict(state_dict) # uses ds.load_state_dict() under the hood
+ ```
+ """
+ self._starting_state_dict = state_dict
+
+ def __repr__(self):
+ return f"IterableDataset({{\n features: {list(self._info.features.keys()) if self._info.features is not None else 'Unknown'},\n num_shards: {self.num_shards}\n}})"
+
+ def __getstate__(self):
+ return self.__dict__
+
+ def __setstate__(self, d):
+ self.__dict__ = d
+ # Re-add torch shared memory, since shared memory is not always kept when pickling
+ self._epoch = _maybe_share_with_torch_persistent_workers(self._epoch)
+ # Re-add the cache to keep on pickling (e.g. for dataloader workers)
+ self.__hffs_cache = HfFileSystem._cache
+ # Re-add torch iterable dataset as a parent class, since dynamically added parent classes are not kept when pickling
+ _maybe_add_torch_iterable_dataset_parent_class(self.__class__)
+
+ def _head(self, n=5):
+ return next(iter(self.iter(batch_size=n)))
+
+ @property
+ def epoch(self) -> int:
+ return int(self._epoch)
+
+ def _effective_generator(self):
+ if self._shuffling and self.epoch == 0:
+ return self._shuffling.generator
+ elif self._shuffling:
+ # Create effective seed using self.epoch (we subtract in order to avoir overflow in long_scalars)
+ effective_seed = deepcopy(self._shuffling.generator).integers(0, 1 << 63) - self.epoch
+ effective_seed = (1 << 63) + effective_seed if effective_seed < 0 else effective_seed
+ return np.random.default_rng(effective_seed)
+ else:
+ raise ValueError("This dataset is not shuffled")
+
+ @property
+ def num_shards(self) -> int:
+ if self._distributed and self._ex_iterable.num_shards % self._distributed.world_size == 0:
+ return self._ex_iterable.num_shards // self._distributed.world_size
+ return self._ex_iterable.num_shards
+
+ @property
+ def n_shards(self) -> int: # backward compatibility
+ return self.num_shards
+
+ def _iter_pytorch(self):
+ ex_iterable = self._prepare_ex_iterable_for_iteration()
+ # Fix for fsspec when using multiprocess to avoid hanging in the ML training loop. (only required for fsspec >= 0.9.0)
+ # See https://github.com/fsspec/gcsfs/issues/379
+ fsspec.asyn.reset_lock()
+ # check if there aren't too many workers
+ import torch.utils.data
+
+ worker_info = torch.utils.data.get_worker_info()
+ if self._is_main_process() and ex_iterable.num_shards < worker_info.num_workers:
+ logger.warning(
+ f"Too many dataloader workers: {worker_info.num_workers} (max is dataset.num_shards={ex_iterable.num_shards}). "
+ f"Stopping {worker_info.num_workers - ex_iterable.num_shards} dataloader workers."
+ )
+ logger.info(
+ f"To parallelize data loading, we give each process some shards (or data sources) to process. "
+ f"Therefore it's unnecessary to have a number of workers greater than dataset.num_shards={ex_iterable.num_shards}. "
+ f"To enable more parallelism, please split the dataset in more files than {ex_iterable.num_shards}."
+ )
+ # split workload
+ _log_prefix = f"node#{self._distributed.rank} " if self._distributed else ""
+ shards_indices = ex_iterable.split_shard_indices_by_worker(
+ num_shards=worker_info.num_workers, index=worker_info.id, contiguous=False
+ )
+ if shards_indices:
+ logger.debug(
+ f"{_log_prefix}dataloader worker#{worker_info.id}, ': Starting to iterate over {len(shards_indices)}/{ex_iterable.num_shards} shards."
+ )
+ ex_iterable = ex_iterable.shard_data_sources(
+ num_shards=worker_info.num_workers, index=worker_info.id, contiguous=False
+ )
+ ex_iterable = shift_ex_examples_rngs(ex_iterable=ex_iterable, value=worker_info.id)
+ self._state_dict = {
+ "examples_iterable": ex_iterable._init_state_dict(),
+ "epoch": self.epoch,
+ }
+ if self._starting_state_dict and self.epoch == self._starting_state_dict["epoch"]:
+ ex_iterable.load_state_dict(self._starting_state_dict["examples_iterable"])
+
+ if self._formatting and (ex_iterable.iter_arrow or self._formatting.is_table):
+ formatter = get_formatter(self._formatting.format_type, features=self.features)
+ if ex_iterable.iter_arrow:
+ iterator = ex_iterable.iter_arrow()
+ else:
+ iterator = _convert_to_arrow(ex_iterable, batch_size=1)
+ for key, pa_table in iterator:
+ yield formatter.format_row(pa_table)
+ return
+ else:
+ for key, example in ex_iterable:
+ # no need to format thanks to FormattedExamplesIterable
+ yield example
+ logger.debug(
+ f"{_log_prefix}dataloader worker#{worker_info.id}, ': Finished iterating over {len(shards_indices)}/{ex_iterable.num_shards} shards."
+ )
+ else:
+ logger.debug(
+ f"{_log_prefix}dataloader worker#{worker_info.id}, ': Stopping... Number of dataset shards < num_workers ({ex_iterable.num_shards}<{worker_info.num_workers})."
+ )
+
+ def _is_main_process(self):
+ if self._distributed and self._distributed.rank > 0:
+ return False
+ if "torch" in sys.modules:
+ import torch.utils.data
+
+ worker_info = torch.utils.data.get_worker_info()
+ if worker_info is not None and worker_info.id > 0:
+ return False
+ return True
+
+ def _prepare_ex_iterable_for_iteration(
+ self, batch_size: int = 1, drop_last_batch: bool = False
+ ) -> _BaseExamplesIterable:
+ ex_iterable = self._ex_iterable
+ if (
+ self._formatting
+ and (ex_iterable.iter_arrow or self._formatting.is_table)
+ or (self.features and ex_iterable.features != self.features)
+ ):
+ ex_iterable = RebatchedArrowExamplesIterable(
+ ex_iterable, batch_size=batch_size, drop_last_batch=drop_last_batch
+ )
+ if self._shuffling:
+ ex_iterable = ex_iterable.shuffle_data_sources(self._effective_generator())
+ else:
+ ex_iterable = ex_iterable
+
+ if self._distributed:
+ rank = self._distributed.rank
+ world_size = self._distributed.world_size
+ if ex_iterable.num_shards % world_size == 0:
+ if self._is_main_process():
+ num_shards_per_node = ex_iterable.num_shards // world_size
+ plural = "s" if num_shards_per_node > 1 else ""
+ logger.info(
+ f"Assigning {num_shards_per_node} shard{plural} (or data source{plural}) of the dataset to each node."
+ )
+ ex_iterable = ex_iterable.shard_data_sources(num_shards=world_size, index=rank, contiguous=False)
+ else:
+ if self._is_main_process():
+ logger.info(
+ f"Assigning 1 out of {world_size} examples of the dataset to each node. The others are skipped during the iteration."
+ )
+ logger.info(
+ f"It is more optimized to distribute the dataset shards (or data sources) across nodes. "
+ f"You can do that by using a dataset with number of shards that is a factor of world_size={world_size}. "
+ f"The current dataset has {ex_iterable.num_shards} which is not a factor of {world_size}"
+ )
+ ex_iterable = StepExamplesIterable(ex_iterable, step=world_size, offset=rank)
+
+ if self._formatting or (self.features and ex_iterable.features != self.features):
+ ex_iterable = FormattedExamplesIterable(
+ ex_iterable,
+ formatting=self._formatting,
+ features=self.features,
+ token_per_repo_id=self._token_per_repo_id,
+ )
+
+ self._state_dict = {
+ "examples_iterable": ex_iterable._init_state_dict(),
+ "epoch": self.epoch,
+ }
+ if self._starting_state_dict and self.epoch == self._starting_state_dict["epoch"]:
+ ex_iterable.load_state_dict(self._starting_state_dict["examples_iterable"])
+ return ex_iterable
+
+ def __iter__(self):
+ if "torch" in sys.modules:
+ import torch.utils.data
+
+ worker_info = torch.utils.data.get_worker_info()
+ if isinstance(self, torch.utils.data.IterableDataset) and worker_info is not None:
+ # We're a torch.utils.data.IterableDataset in a PyTorch worker process
+ yield from self._iter_pytorch()
+ return
+
+ ex_iterable = self._prepare_ex_iterable_for_iteration()
+ if self._formatting and (ex_iterable.iter_arrow or self._formatting.is_table):
+ formatter = get_formatter(self._formatting.format_type, features=self.features)
+ if ex_iterable.iter_arrow:
+ iterator = ex_iterable.iter_arrow()
+ else:
+ iterator = _convert_to_arrow(ex_iterable, batch_size=1)
+ for key, pa_table in iterator:
+ yield formatter.format_row(pa_table)
+ return
+
+ for key, example in ex_iterable:
+ # no need to format thanks to FormattedExamplesIterable
+ yield example
+
+ def iter(self, batch_size: int, drop_last_batch: bool = False):
+ """Iterate through the batches of size `batch_size`.
+
+ Args:
+ batch_size (:obj:`int`): size of each batch to yield.
+ drop_last_batch (:obj:`bool`, default `False`): Whether a last batch smaller than the batch_size should be
+ dropped
+ """
+
+ if self._formatting:
+ formatter = get_formatter(self._formatting.format_type, features=self.features)
+ format_dict = formatter.recursive_tensorize if isinstance(formatter, TensorFormatter) else None
+ else:
+ format_dict = None
+
+ ex_iterable = self._prepare_ex_iterable_for_iteration(batch_size=batch_size, drop_last_batch=drop_last_batch)
+ if self._formatting and (ex_iterable.iter_arrow or self._formatting.is_table):
+ if ex_iterable.iter_arrow:
+ iterator = ex_iterable.iter_arrow()
+ else:
+ iterator = _convert_to_arrow(ex_iterable, batch_size=batch_size, drop_last_batch=drop_last_batch)
+ for key, pa_table in iterator:
+ yield formatter.format_batch(pa_table)
+ return
+
+ iterator = iter(ex_iterable)
+ for key, example in iterator:
+ # If batched, first build the batch
+ examples = [example] + [example for key, example in islice(iterator, batch_size - 1)]
+ if drop_last_batch and len(examples) < batch_size: # ignore last batch
+ return
+ batch = _examples_to_batch(examples)
+ # we need to format here in case we need to stack tensors together
+ yield format_dict(batch) if format_dict else batch
+
+ def __getitem__(self, column_name: str) -> IterableColumn:
+ return IterableColumn(self, column_name)
+
+ @staticmethod
+ def from_generator(
+ generator: Callable,
+ features: Optional[Features] = None,
+ gen_kwargs: Optional[dict] = None,
+ split: NamedSplit = Split.TRAIN,
+ ) -> "IterableDataset":
+ """Create an Iterable Dataset from a generator.
+
+ Args:
+ generator (`Callable`):
+ A generator function that `yields` examples.
+ features (`Features`, *optional*):
+ Dataset features.
+ gen_kwargs(`dict`, *optional*):
+ Keyword arguments to be passed to the `generator` callable.
+ You can define a sharded iterable dataset by passing the list of shards in `gen_kwargs`.
+ This can be used to improve shuffling and when iterating over the dataset with multiple workers.
+ split ([`NamedSplit`], defaults to `Split.TRAIN`):
+ Split name to be assigned to the dataset.
+
+
+ Returns:
+ `IterableDataset`
+
+ Example:
+
+ ```py
+ >>> def gen():
+ ... yield {"text": "Good", "label": 0}
+ ... yield {"text": "Bad", "label": 1}
+ ...
+ >>> ds = IterableDataset.from_generator(gen)
+ ```
+
+ ```py
+ >>> def gen(shards):
+ ... for shard in shards:
+ ... with open(shard) as f:
+ ... for line in f:
+ ... yield {"line": line}
+ ...
+ >>> shards = [f"data{i}.txt" for i in range(32)]
+ >>> ds = IterableDataset.from_generator(gen, gen_kwargs={"shards": shards})
+ >>> ds = ds.shuffle(seed=42, buffer_size=10_000) # shuffles the shards order + uses a shuffle buffer
+ >>> from torch.utils.data import DataLoader
+ >>> dataloader = DataLoader(ds.with_format("torch"), num_workers=4) # give each worker a subset of 32/4=8 shards
+ ```
+ """
+ from .io.generator import GeneratorDatasetInputStream
+
+ return GeneratorDatasetInputStream(
+ generator=generator, features=features, gen_kwargs=gen_kwargs, streaming=True, split=split
+ ).read()
+
+ @staticmethod
+ def from_spark(
+ df: "pyspark.sql.DataFrame",
+ split: Optional[NamedSplit] = None,
+ features: Optional[Features] = None,
+ **kwargs,
+ ) -> "IterableDataset":
+ """Create an IterableDataset from Spark DataFrame. The dataset is streamed to the driver in batches.
+
+ Args:
+ df (`pyspark.sql.DataFrame`):
+ The DataFrame containing the desired data.
+ split (`NamedSplit`, *optional*):
+ Split name to be assigned to the dataset.
+ features (`Features`, *optional*):
+ Dataset features.
+
+ Returns:
+ [`IterableDataset`]
+
+ Example:
+
+ ```py
+ >>> df = spark.createDataFrame(
+ >>> data=[[1, "Elia"], [2, "Teo"], [3, "Fang"]],
+ >>> columns=["id", "name"],
+ >>> )
+ >>> ds = IterableDataset.from_spark(df)
+ ```
+ """
+ from .io.spark import SparkDatasetReader
+
+ if sys.platform == "win32":
+ raise OSError("IterableDataset.from_spark is not currently supported on Windows")
+
+ return SparkDatasetReader(
+ df,
+ split=split,
+ features=features,
+ streaming=True,
+ **kwargs,
+ ).read()
+
+ @staticmethod
+ def from_file(filename: str) -> "IterableDataset":
+ """Instantiate a IterableDataset from Arrow table at filename.
+
+ Args:
+ filename (`str`):
+ File name of the dataset.
+
+ Returns:
+ [`IterableDataset`]
+ """
+ pa_table_schema = read_schema_from_file(filename)
+ inferred_features = Features.from_arrow_schema(pa_table_schema)
+ ex_iterable = ArrowExamplesIterable(Dataset._generate_tables_from_cache_file, kwargs={"filename": filename})
+ return IterableDataset(ex_iterable=ex_iterable, info=DatasetInfo(features=inferred_features))
+
+ def with_format(
+ self,
+ type: Optional[str] = None,
+ ) -> "IterableDataset":
+ """
+ Return a dataset with the specified format.
+
+ Args:
+
+ type (`str`, *optional*):
+ Either output type selected in `[None, 'numpy', 'torch', 'tensorflow', 'jax', 'arrow', 'pandas', 'polars']`.
+ `None` means it returns python objects (default).
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> from transformers import AutoTokenizer
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation", streaming=True)
+ >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
+ >>> ds = ds.map(lambda x: tokenizer(x['text'], truncation=True, padding=True), batched=True)
+ >>> ds = ds.with_format("torch")
+ >>> next(iter(ds))
+ {'text': 'compassionately explores the seemingly irreconcilable situation between conservative christian parents and their estranged gay and lesbian children .',
+ 'label': tensor(1),
+ 'input_ids': tensor([ 101, 18027, 16310, 16001, 1103, 9321, 178, 11604, 7235, 6617,
+ 1742, 2165, 2820, 1206, 6588, 22572, 12937, 1811, 2153, 1105,
+ 1147, 12890, 19587, 6463, 1105, 15026, 1482, 119, 102, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0]),
+ 'token_type_ids': tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]),
+ 'attention_mask': tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])}
+ ```
+ """
+ type = get_format_type_from_alias(type)
+ # TODO(QL): add format_kwargs
+ # TODO(QL): add format_columns and return_all_columns
+ # TODO(QL): add pandas format
+ return IterableDataset(
+ ex_iterable=self._ex_iterable,
+ info=self._info.copy(),
+ split=self._split,
+ formatting=FormattingConfig(format_type=type),
+ shuffling=copy.deepcopy(self._shuffling),
+ distributed=copy.deepcopy(self._distributed),
+ token_per_repo_id=self._token_per_repo_id,
+ )
+
+ def map(
+ self,
+ function: Optional[Callable] = None,
+ with_indices: bool = False,
+ input_columns: Optional[Union[str, list[str]]] = None,
+ batched: bool = False,
+ batch_size: Optional[int] = 1000,
+ drop_last_batch: bool = False,
+ remove_columns: Optional[Union[str, list[str]]] = None,
+ features: Optional[Features] = None,
+ fn_kwargs: Optional[dict] = None,
+ ) -> "IterableDataset":
+ """
+ Apply a function to all the examples in the iterable dataset (individually or in batches) and update them.
+ If your function returns a column that already exists, then it overwrites it.
+ The function is applied on-the-fly on the examples when iterating over the dataset.
+
+ You can specify whether the function should be batched or not with the `batched` parameter:
+
+ - If batched is `False`, then the function takes 1 example in and should return 1 example.
+ An example is a dictionary, e.g. `{"text": "Hello there !"}`.
+ - If batched is `True` and `batch_size` is 1, then the function takes a batch of 1 example as input and can return a batch with 1 or more examples.
+ A batch is a dictionary, e.g. a batch of 1 example is {"text": ["Hello there !"]}.
+ - If batched is `True` and `batch_size` is `n` > 1, then the function takes a batch of `n` examples as input and can return a batch with `n` examples, or with an arbitrary number of examples.
+ Note that the last batch may have less than `n` examples.
+ A batch is a dictionary, e.g. a batch of `n` examples is `{"text": ["Hello there !"] * n}`.
+
+ If the function is asynchronous, then `map` will run your function in parallel, with up to one thousand simulatenous calls.
+ It is recommended to use a `asyncio.Semaphore` in your function if you want to set a maximum number of operations that can run at the same time.
+
+ Args:
+ function (`Callable`, *optional*, defaults to `None`):
+ Function applied on-the-fly on the examples when you iterate on the dataset.
+ It must have one of the following signatures:
+
+ - `function(example: Dict[str, Any]) -> Dict[str, Any]` if `batched=False` and `with_indices=False`
+ - `function(example: Dict[str, Any], idx: int) -> Dict[str, Any]` if `batched=False` and `with_indices=True`
+ - `function(batch: Dict[str, List]) -> Dict[str, List]` if `batched=True` and `with_indices=False`
+ - `function(batch: Dict[str, List], indices: List[int]) -> Dict[str, List]` if `batched=True` and `with_indices=True`
+
+ For advanced usage, the function can also return a `pyarrow.Table`.
+ If the function is asynchronous, then `map` will run your function in parallel.
+ Moreover if your function returns nothing (`None`), then `map` will run your function and return the dataset unchanged.
+ If no function is provided, default to identity function: `lambda x: x`.
+ with_indices (`bool`, defaults to `False`):
+ Provide example indices to `function`. Note that in this case the signature of `function` should be `def function(example, idx[, rank]): ...`.
+ input_columns (`Optional[Union[str, List[str]]]`, defaults to `None`):
+ The columns to be passed into `function`
+ as positional arguments. If `None`, a dict mapping to all formatted columns is passed as one argument.
+ batched (`bool`, defaults to `False`):
+ Provide batch of examples to `function`.
+ batch_size (`int`, *optional*, defaults to `1000`):
+ Number of examples per batch provided to `function` if `batched=True`.
+ `batch_size <= 0` or `batch_size == None` then provide the full dataset as a single batch to `function`.
+ drop_last_batch (`bool`, defaults to `False`):
+ Whether a last batch smaller than the batch_size should be
+ dropped instead of being processed by the function.
+ remove_columns (`[List[str]]`, *optional*, defaults to `None`):
+ Remove a selection of columns while doing the mapping.
+ Columns will be removed before updating the examples with the output of `function`, i.e. if `function` is adding
+ columns with names in `remove_columns`, these columns will be kept.
+ features (`[Features]`, *optional*, defaults to `None`):
+ Feature types of the resulting dataset.
+ fn_kwargs (`Dict`, *optional*, default `None`):
+ Keyword arguments to be passed to `function`.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train", streaming=True)
+ >>> def add_prefix(example):
+ ... example["text"] = "Review: " + example["text"]
+ ... return example
+ >>> ds = ds.map(add_prefix)
+ >>> list(ds.take(3))
+ [{'label': 1,
+ 'text': 'Review: the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'},
+ {'label': 1,
+ 'text': 'Review: the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson\'s expanded vision of j . r . r . tolkien\'s middle-earth .'},
+ {'label': 1, 'text': 'Review: effective but too-tepid biopic'}]
+ ```
+ """
+ if isinstance(input_columns, str):
+ input_columns = [input_columns]
+ if isinstance(remove_columns, str):
+ remove_columns = [remove_columns]
+ if function is None:
+ function = identity_func
+ if fn_kwargs is None:
+ fn_kwargs = {}
+ if features is not None:
+ features = _fix_for_backward_compatible_features(features)
+
+ ex_iterable = self._ex_iterable
+ # no need to apply features if ex_iterable is typed and if there was no cast_column()
+ input_features = (
+ None
+ if (ex_iterable.is_typed and (self._info.features is None or self._info.features == ex_iterable.features))
+ else self._info.features
+ )
+
+ if self._formatting and self._formatting.is_table:
+ # apply formatting before iter_arrow to keep map examples iterable happy
+ ex_iterable = FormattedExamplesIterable(
+ ex_iterable,
+ formatting=copy.deepcopy(self._formatting),
+ features=input_features,
+ token_per_repo_id=self._token_per_repo_id,
+ )
+ ex_iterable = RebatchedArrowExamplesIterable(
+ ex_iterable, batch_size=batch_size if batched else 1, drop_last_batch=drop_last_batch
+ )
+ else:
+ if self._formatting and self._ex_iterable.iter_arrow:
+ ex_iterable = RebatchedArrowExamplesIterable(
+ self._ex_iterable, batch_size=batch_size if batched else 1, drop_last_batch=drop_last_batch
+ )
+ if self._formatting or input_features:
+ # apply formatting after iter_arrow to avoid re-encoding the examples
+ ex_iterable = FormattedExamplesIterable(
+ ex_iterable,
+ formatting=copy.deepcopy(self._formatting),
+ features=input_features,
+ token_per_repo_id=self._token_per_repo_id,
+ )
+
+ ex_iterable = MappedExamplesIterable(
+ ex_iterable,
+ function=function,
+ with_indices=with_indices,
+ input_columns=input_columns,
+ batched=batched,
+ batch_size=batch_size,
+ drop_last_batch=drop_last_batch,
+ remove_columns=remove_columns,
+ fn_kwargs=fn_kwargs,
+ formatting=self._formatting,
+ features=features,
+ )
+ info = self.info.copy()
+ info.features = features
+ return IterableDataset(
+ ex_iterable=ex_iterable,
+ info=info,
+ split=self._split,
+ formatting=self._formatting,
+ shuffling=copy.deepcopy(self._shuffling),
+ distributed=copy.deepcopy(self._distributed),
+ token_per_repo_id=self._token_per_repo_id,
+ )
+
+ def filter(
+ self,
+ function: Optional[Callable] = None,
+ with_indices=False,
+ input_columns: Optional[Union[str, list[str]]] = None,
+ batched: bool = False,
+ batch_size: Optional[int] = 1000,
+ fn_kwargs: Optional[dict] = None,
+ ) -> "IterableDataset":
+ """Apply a filter function to all the elements so that the dataset only includes examples according to the filter function.
+ The filtering is done on-the-fly when iterating over the dataset.
+
+ If the function is asynchronous, then `filter` will run your function in parallel, with up to one thousand simulatenous calls (configurable).
+ It is recommended to use a `asyncio.Semaphore` in your function if you want to set a maximum number of operations that can run at the same time.
+
+ Args:
+ function (`Callable`):
+ Callable with one of the following signatures:
+
+ - `function(example: Dict[str, Any]) -> bool` if `with_indices=False, batched=False`
+ - `function(example: Dict[str, Any], indices: int) -> bool` if `with_indices=True, batched=False`
+ - `function(example: Dict[str, List]) -> List[bool]` if `with_indices=False, batched=True`
+ - `function(example: Dict[str, List], indices: List[int]) -> List[bool]` if `with_indices=True, batched=True`
+
+ If the function is asynchronous, then `filter` will run your function in parallel.
+ If no function is provided, defaults to an always True function: `lambda x: True`.
+ with_indices (`bool`, defaults to `False`):
+ Provide example indices to `function`. Note that in this case the signature of `function` should be `def function(example, idx): ...`.
+ input_columns (`str` or `List[str]`, *optional*):
+ The columns to be passed into `function` as
+ positional arguments. If `None`, a dict mapping to all formatted columns is passed as one argument.
+ batched (`bool`, defaults to `False`):
+ Provide batch of examples to `function`.
+ batch_size (`int`, *optional*, default `1000`):
+ Number of examples per batch provided to `function` if `batched=True`.
+ fn_kwargs (`Dict`, *optional*, default `None`):
+ Keyword arguments to be passed to `function`.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train", streaming=True)
+ >>> ds = ds.filter(lambda x: x["label"] == 0)
+ >>> list(ds.take(3))
+ [{'label': 0, 'movie_review': 'simplistic , silly and tedious .'},
+ {'label': 0,
+ 'movie_review': "it's so laddish and juvenile , only teenage boys could possibly find it funny ."},
+ {'label': 0,
+ 'movie_review': 'exploitative and largely devoid of the depth or sophistication that would make watching such a graphic treatment of the crimes bearable .'}]
+ ```
+ """
+ if isinstance(input_columns, str):
+ input_columns = [input_columns]
+
+ # We need the examples to be decoded for certain feature types like Image or Audio,
+ # format and type before filtering
+ ex_iterable = self._ex_iterable
+ if self._info.features or self._formatting:
+ ex_iterable = FormattedExamplesIterable(
+ ex_iterable,
+ formatting=self._formatting,
+ features=None if ex_iterable.is_typed else self._info.features,
+ token_per_repo_id=self._token_per_repo_id,
+ )
+
+ ex_iterable = FilteredExamplesIterable(
+ ex_iterable,
+ function=function,
+ with_indices=with_indices,
+ input_columns=input_columns,
+ batched=batched,
+ batch_size=batch_size,
+ fn_kwargs=fn_kwargs,
+ formatting=self._formatting,
+ )
+ return IterableDataset(
+ ex_iterable=ex_iterable,
+ info=self._info,
+ split=self._split,
+ formatting=self._formatting,
+ shuffling=copy.deepcopy(self._shuffling),
+ distributed=copy.deepcopy(self._distributed),
+ token_per_repo_id=self._token_per_repo_id,
+ )
+
+ def shuffle(
+ self, seed=None, generator: Optional[np.random.Generator] = None, buffer_size: int = 1000
+ ) -> "IterableDataset":
+ """
+ Randomly shuffles the elements of this dataset.
+
+ This dataset fills a buffer with `buffer_size` elements, then randomly samples elements from this buffer,
+ replacing the selected elements with new elements. For perfect shuffling, a buffer size greater than or
+ equal to the full size of the dataset is required.
+
+ For instance, if your dataset contains 10,000 elements but `buffer_size` is set to 1000, then `shuffle` will
+ initially select a random element from only the first 1000 elements in the buffer. Once an element is
+ selected, its space in the buffer is replaced by the next (i.e. 1,001-st) element,
+ maintaining the 1000 element buffer.
+
+ If the dataset is made of several shards, it also does shuffle the order of the shards.
+ However if the order has been fixed by using [`~datasets.IterableDataset.skip`] or [`~datasets.IterableDataset.take`]
+ then the order of the shards is kept unchanged.
+
+ Args:
+ seed (`int`, *optional*, defaults to `None`):
+ Random seed that will be used to shuffle the dataset.
+ It is used to sample from the shuffle buffer and also to shuffle the data shards.
+ generator (`numpy.random.Generator`, *optional*):
+ Numpy random Generator to use to compute the permutation of the dataset rows.
+ If `generator=None` (default), uses `np.random.default_rng` (the default BitGenerator (PCG64) of NumPy).
+ buffer_size (`int`, defaults to `1000`):
+ Size of the buffer.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train", streaming=True)
+ >>> list(ds.take(3))
+ [{'label': 1,
+ 'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'},
+ {'label': 1,
+ 'text': 'the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson\'s expanded vision of j . r . r . tolkien\'s middle-earth .'},
+ {'label': 1, 'text': 'effective but too-tepid biopic'}]
+ >>> shuffled_ds = ds.shuffle(seed=42)
+ >>> list(shuffled_ds.take(3))
+ [{'label': 1,
+ 'text': "a sports movie with action that's exciting on the field and a story you care about off it ."},
+ {'label': 1,
+ 'text': 'at its best , the good girl is a refreshingly adult take on adultery . . .'},
+ {'label': 1,
+ 'text': "sam jones became a very lucky filmmaker the day wilco got dropped from their record label , proving that one man's ruin may be another's fortune ."}]
+ ```
+ """
+ if generator is None:
+ generator = np.random.default_rng(seed)
+ else:
+ generator = deepcopy(generator)
+ shuffling = ShufflingConfig(generator=generator, _original_seed=seed)
+ return IterableDataset(
+ BufferShuffledExamplesIterable(
+ RebatchedArrowExamplesIterable(self._ex_iterable, batch_size=1)
+ if self._ex_iterable.iter_arrow
+ else self._ex_iterable,
+ buffer_size=buffer_size,
+ generator=generator,
+ ),
+ info=self._info.copy(),
+ split=self._split,
+ formatting=self._formatting,
+ shuffling=shuffling,
+ distributed=copy.deepcopy(self._distributed),
+ token_per_repo_id=self._token_per_repo_id,
+ )
+
+ def set_epoch(self, epoch: int):
+ self._epoch += epoch - self._epoch # update torch value in shared memory in-place
+
+ def skip(self, n: int) -> "IterableDataset":
+ """
+ Create a new [`IterableDataset`] that skips the first `n` elements.
+
+ Args:
+ n (`int`):
+ Number of elements to skip.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train", streaming=True)
+ >>> list(ds.take(3))
+ [{'label': 1,
+ 'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'},
+ {'label': 1,
+ 'text': 'the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson\'s expanded vision of j . r . r . tolkien\'s middle-earth .'},
+ {'label': 1, 'text': 'effective but too-tepid biopic'}]
+ >>> ds = ds.skip(1)
+ >>> list(ds.take(3))
+ [{'label': 1,
+ 'text': 'the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson\'s expanded vision of j . r . r . tolkien\'s middle-earth .'},
+ {'label': 1, 'text': 'effective but too-tepid biopic'},
+ {'label': 1,
+ 'text': 'if you sometimes like to go to the movies to have fun , wasabi is a good place to start .'}]
+ ```
+ """
+ ex_iterable = SkipExamplesIterable(
+ self._ex_iterable,
+ n,
+ block_sources_order_when_shuffling=self._shuffling is None,
+ split_when_sharding=self._distributed is None,
+ )
+ return IterableDataset(
+ ex_iterable=ex_iterable,
+ info=self._info.copy(),
+ split=self._split,
+ formatting=self._formatting,
+ shuffling=copy.deepcopy(self._shuffling),
+ distributed=copy.deepcopy(self._distributed),
+ token_per_repo_id=self._token_per_repo_id,
+ )
+
+ def repeat(self, num_times: Optional[int]) -> "IterableDataset":
+ """
+ Create a new [`IterableDataset`] that repeats the underlying dataset `num_times` times.
+
+ N.B. The effect of calling shuffle after repeat depends significantly on buffer size.
+ With buffer_size 1, duplicate data is never seen in the same iteration, even after shuffling:
+ ds.repeat(n).shuffle(seed=42, buffer_size=1) is equivalent to ds.shuffle(seed=42, buffer_size=1).repeat(n),
+ and only shuffles shard orders within each iteration.
+ With buffer size >= (num samples in the dataset * num_times), we get full shuffling of the repeated data, i.e. we can observe duplicates in
+ the same iteration.
+
+ Args:
+ num_times (`int`) or (`None`):
+ Number of times to repeat the dataset. If `None`, the dataset will be repeated indefinitely.
+
+ Example:
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train")
+ >>> ds = ds.take(2).repeat(2)
+ >>> list(ds)
+ [{'label': 1,
+ 'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'},
+ {'label': 1,
+ 'text': 'the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson\'s expanded vision of j . r . r . tolkien\'s middle-earth .'},
+ {'label': 1, 'text': 'effective but too-tepid biopic'},
+ {'label': 1,
+ 'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'},
+ {'label': 1,
+ 'text': 'the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson\'s expanded vision of j . r . r . tolkien\'s middle-earth .'},
+ {'label': 1, 'text': 'effective but too-tepid biopic'}]
+ ```
+ """
+ return IterableDataset(
+ ex_iterable=RepeatExamplesIterable(self._ex_iterable, num_times=num_times),
+ info=self._info,
+ split=self._split,
+ formatting=self._formatting,
+ shuffling=copy.deepcopy(self._shuffling),
+ distributed=copy.deepcopy(self._distributed),
+ token_per_repo_id=self._token_per_repo_id,
+ )
+
+ def take(self, n: int) -> "IterableDataset":
+ """
+ Create a new [`IterableDataset`] with only the first `n` elements.
+
+ Args:
+ n (`int`):
+ Number of elements to take.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train", streaming=True)
+ >>> small_ds = ds.take(2)
+ >>> list(small_ds)
+ [{'label': 1,
+ 'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'},
+ {'label': 1,
+ 'text': 'the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson\'s expanded vision of j . r . r . tolkien\'s middle-earth .'}]
+ ```
+ """
+ ex_iterable = TakeExamplesIterable(
+ self._ex_iterable,
+ n,
+ block_sources_order_when_shuffling=self._shuffling is None,
+ split_when_sharding=self._distributed is None,
+ )
+ return IterableDataset(
+ ex_iterable=ex_iterable,
+ info=self._info.copy(),
+ split=self._split,
+ formatting=self._formatting,
+ shuffling=copy.deepcopy(self._shuffling),
+ distributed=copy.deepcopy(self._distributed),
+ token_per_repo_id=self._token_per_repo_id,
+ )
+
+ def shard(
+ self,
+ num_shards: int,
+ index: int,
+ contiguous: bool = True,
+ ) -> "IterableDataset":
+ """Return the `index`-nth shard from dataset split into `num_shards` pieces.
+
+ This shards deterministically. `dataset.shard(n, i)` splits the dataset into contiguous chunks,
+ so it can be easily concatenated back together after processing. If `dataset.num_shards % n == l`, then the
+ first `l` datasets each have `(dataset.num_shards // n) + 1` shards, and the remaining datasets have `(dataset.num_shards // n)` shards.
+ `datasets.concatenate_datasets([dset.shard(n, i) for i in range(n)])` returns a dataset with the same order as the original.
+ In particular, `dataset.shard(dataset.num_shards, i)` returns a dataset with 1 shard.
+
+ Note: n should be less or equal to the number of shards in the dataset `dataset.num_shards`.
+
+ On the other hand, `dataset.shard(n, i, contiguous=False)` contains all the shards of the dataset whose index mod `n = i`.
+
+ Be sure to shard before using any randomizing operator (such as `shuffle`).
+ It is best if the shard operator is used early in the dataset pipeline.
+
+ Args:
+ num_shards (`int`):
+ How many shards to split the dataset into.
+ index (`int`):
+ Which shard to select and return.
+ contiguous: (`bool`, defaults to `True`):
+ Whether to select contiguous blocks of indices for shards.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("amazon_polarity", split="train", streaming=True)
+ >>> ds
+ Dataset({
+ features: ['label', 'title', 'content'],
+ num_shards: 4
+ })
+ >>> ds.shard(num_shards=2, index=0)
+ Dataset({
+ features: ['label', 'title', 'content'],
+ num_shards: 2
+ })
+ ```
+ """
+ ex_iterable = self._ex_iterable.shard_data_sources(num_shards=num_shards, index=index, contiguous=contiguous)
+ return IterableDataset(
+ ex_iterable=ex_iterable,
+ info=self._info.copy(),
+ split=self._split,
+ formatting=self._formatting,
+ shuffling=copy.deepcopy(self._shuffling),
+ distributed=copy.deepcopy(self._distributed),
+ token_per_repo_id=self._token_per_repo_id,
+ )
+
+ def add_column(self, name: str, column: Union[list, np.array]) -> "IterableDataset":
+ """Add column to Dataset.
+
+ Args:
+ name (str): Column name.
+ column (list or np.array): Column data to be added.
+
+ Returns:
+ `IterableDataset`
+ """
+ return self.map(partial(add_column_fn, name=name, column=column), with_indices=True)
+
+ def rename_column(self, original_column_name: str, new_column_name: str) -> "IterableDataset":
+ """
+ Rename a column in the dataset, and move the features associated to the original column under the new column
+ name.
+
+ Args:
+ original_column_name (`str`):
+ Name of the column to rename.
+ new_column_name (`str`):
+ New name for the column.
+
+ Returns:
+ `IterableDataset`: A copy of the dataset with a renamed column.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train", streaming=True)
+ >>> next(iter(ds))
+ {'label': 1,
+ 'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'}
+ >>> ds = ds.rename_column("text", "movie_review")
+ >>> next(iter(ds))
+ {'label': 1,
+ 'movie_review': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'}
+ ```
+ """
+ return self.rename_columns({original_column_name: new_column_name})
+
+ def rename_columns(self, column_mapping: dict[str, str]) -> "IterableDataset":
+ """
+ Rename several columns in the dataset, and move the features associated to the original columns under
+ the new column names.
+
+ Args:
+ column_mapping (`Dict[str, str]`): A mapping of columns to rename to their new names
+
+ Returns:
+ `IterableDataset`: A copy of the dataset with renamed columns
+ """
+
+ original_features = self._info.features.copy() if self._info.features else None
+ ds_iterable = self.map(
+ partial(_rename_columns_fn, column_mapping=column_mapping), remove_columns=list(column_mapping)
+ )
+ if original_features is not None:
+ ds_iterable._info.features = Features(
+ {
+ column_mapping[col] if col in column_mapping.keys() else col: feature
+ for col, feature in original_features.items()
+ }
+ )
+ return ds_iterable
+
+ def remove_columns(self, column_names: Union[str, list[str]]) -> "IterableDataset":
+ """
+ Remove one or several column(s) in the dataset and the features associated to them.
+ The removal is done on-the-fly on the examples when iterating over the dataset.
+
+
+ Args:
+ column_names (`Union[str, List[str]]`):
+ Name of the column(s) to remove.
+
+ Returns:
+ `IterableDataset`: A copy of the dataset object without the columns to remove.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train", streaming=True)
+ >>> next(iter(ds))
+ {'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .', 'label': 1}
+ >>> ds = ds.remove_columns("label")
+ >>> next(iter(ds))
+ {'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'}
+ ```
+ """
+ original_features = self._info.features.copy() if self._info.features else None
+ ds_iterable = self.map(remove_columns=column_names)
+ if original_features is not None:
+ ds_iterable._info.features = original_features.copy()
+ for col, _ in original_features.items():
+ if col in column_names:
+ del ds_iterable._info.features[col]
+
+ return ds_iterable
+
+ def select_columns(self, column_names: Union[str, list[str]]) -> "IterableDataset":
+ """Select one or several column(s) in the dataset and the features
+ associated to them. The selection is done on-the-fly on the examples
+ when iterating over the dataset.
+
+
+ Args:
+ column_names (`Union[str, List[str]]`):
+ Name of the column(s) to select.
+
+ Returns:
+ `IterableDataset`: A copy of the dataset object with selected columns.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train", streaming=True)
+ >>> next(iter(ds))
+ {'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .', 'label': 1}
+ >>> ds = ds.select_columns("text")
+ >>> next(iter(ds))
+ {'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'}
+ ```
+ """
+ if isinstance(column_names, str):
+ column_names = [column_names]
+
+ if self._info:
+ info = copy.deepcopy(self._info)
+ if self._info.features is not None:
+ missing_columns = set(column_names) - set(self._info.features.keys())
+ if missing_columns:
+ raise ValueError(
+ f"Column name {list(missing_columns)} not in the "
+ "dataset. Columns in the dataset: "
+ f"{list(self._info.features.keys())}."
+ )
+ info.features = Features({c: info.features[c] for c in column_names})
+
+ ex_iterable = SelectColumnsIterable(self._ex_iterable, column_names)
+ return IterableDataset(
+ ex_iterable=ex_iterable,
+ info=info,
+ split=self._split,
+ formatting=self._formatting,
+ shuffling=self._shuffling,
+ distributed=self._distributed,
+ token_per_repo_id=self._token_per_repo_id,
+ )
+
+ def cast_column(self, column: str, feature: FeatureType) -> "IterableDataset":
+ """Cast column to feature for decoding.
+
+ Args:
+ column (`str`):
+ Column name.
+ feature (`Feature`):
+ Target feature.
+
+ Returns:
+ `IterableDataset`
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset, Audio
+ >>> ds = load_dataset("PolyAI/minds14", name="en-US", split="train", streaming=True)
+ >>> ds.features
+ {'audio': Audio(sampling_rate=8000, mono=True, decode=True, id=None),
+ 'english_transcription': Value('string'),
+ 'intent_class': ClassLabel(num_classes=14, names=['abroad', 'address', 'app_error', 'atm_limit', 'balance', 'business_loan', 'card_issues', 'cash_deposit', 'direct_debit', 'freeze', 'high_value_payment', 'joint_account', 'latest_transactions', 'pay_bill']),
+ 'lang_id': ClassLabel(num_classes=14, names=['cs-CZ', 'de-DE', 'en-AU', 'en-GB', 'en-US', 'es-ES', 'fr-FR', 'it-IT', 'ko-KR', 'nl-NL', 'pl-PL', 'pt-PT', 'ru-RU', 'zh-CN']),
+ 'path': Value('string'),
+ 'transcription': Value('string')}
+ >>> ds = ds.cast_column("audio", Audio(sampling_rate=16000))
+ >>> ds.features
+ {'audio': Audio(sampling_rate=16000, mono=True, decode=True, id=None),
+ 'english_transcription': Value('string'),
+ 'intent_class': ClassLabel(num_classes=14, names=['abroad', 'address', 'app_error', 'atm_limit', 'balance', 'business_loan', 'card_issues', 'cash_deposit', 'direct_debit', 'freeze', 'high_value_payment', 'joint_account', 'latest_transactions', 'pay_bill']),
+ 'lang_id': ClassLabel(num_classes=14, names=['cs-CZ', 'de-DE', 'en-AU', 'en-GB', 'en-US', 'es-ES', 'fr-FR', 'it-IT', 'ko-KR', 'nl-NL', 'pl-PL', 'pt-PT', 'ru-RU', 'zh-CN']),
+ 'path': Value('string'),
+ 'transcription': Value('string')}
+ ```
+ """
+ feature = _fix_for_backward_compatible_features(feature)
+ info = self._info.copy()
+ info.features[column] = feature
+ return IterableDataset(
+ ex_iterable=self._ex_iterable,
+ info=info,
+ split=self._split,
+ formatting=self._formatting,
+ shuffling=copy.deepcopy(self._shuffling),
+ distributed=copy.deepcopy(self._distributed),
+ token_per_repo_id=self._token_per_repo_id,
+ )
+
+ def cast(
+ self,
+ features: Features,
+ ) -> "IterableDataset":
+ """
+ Cast the dataset to a new set of features.
+
+ Args:
+ features ([`Features`]):
+ New features to cast the dataset to.
+ The name of the fields in the features must match the current column names.
+ The type of the data must also be convertible from one type to the other.
+ For non-trivial conversion, e.g. `string` <-> `ClassLabel` you should use [`~Dataset.map`] to update the Dataset.
+
+ Returns:
+ `IterableDataset`: A copy of the dataset with casted features.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset, ClassLabel, Value
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train", streaming=True)
+ >>> ds.features
+ {'label': ClassLabel(names=['neg', 'pos']),
+ 'text': Value('string')}
+ >>> new_features = ds.features.copy()
+ >>> new_features["label"] = ClassLabel(names=["bad", "good"])
+ >>> new_features["text"] = Value("large_string")
+ >>> ds = ds.cast(new_features)
+ >>> ds.features
+ {'label': ClassLabel(names=['bad', 'good']),
+ 'text': Value('large_string')}
+ ```
+ """
+ features = _fix_for_backward_compatible_features(features)
+ info = self._info.copy()
+ info.features = features
+ return IterableDataset(
+ ex_iterable=self._ex_iterable,
+ info=info,
+ split=self._split,
+ formatting=self._formatting,
+ shuffling=copy.deepcopy(self._shuffling),
+ distributed=copy.deepcopy(self._distributed),
+ token_per_repo_id=self._token_per_repo_id,
+ )
+
+ def decode(self, enable: bool = True, num_threads: int = 0) -> "IterableDataset":
+ """
+ Enable or disable the dataset features decoding for audio, image, video.
+
+ When enabled (default), media types are decoded:
+
+ * audio -> dict of "array" and "sampling_rate" and "path"
+ * image -> PIL.Image
+ * video -> torchvision.io.VideoReader
+
+ You can enable multithreading using `num_threads`. This is especially useful to speed up remote
+ data streaming. However it can be slower than `num_threads=0` for local data on fast disks.
+
+ Disabling decoding is useful if you want to iterate on the paths or bytes of the media files
+ without actually decoding their content. To disable decoding you can use `.decode(False)`, which
+ is equivalent to calling `.cast()` or `.cast_column()` with all the Audio, Image and Video types
+ set to `decode=False`.
+
+ Args:
+ enable (`bool`, defaults to `True`):
+ Enable or disable features decoding.
+ num_threads (`int`, defaults to `0`):
+ Enable multithreading for features decoding.
+
+ Returns:
+ `IterableDataset`: A copy of the dataset with casted features.
+
+ Examples:
+
+ Disable decoding:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("sshh12/planet-textures", split="train", streaming=True)
+ >>> next(iter(ds))
+ {'image': ,
+ 'text': 'A distant celestial object with an icy crust, displaying a light blue shade, covered with round pits and rugged terrains.'}
+ >>> ds = ds.decode(False)
+ >>> ds.features
+ {'image': Image(mode=None, decode=False, id=None),
+ 'text': Value('string')}
+ >>> next(iter(ds))
+ {
+ 'image': {
+ 'path': 'hf://datasets/sshh12/planet-textures@69dc4cef7a5c4b2cfe387727ec8ea73d4bff7302/train/textures/0000.png',
+ 'bytes': None
+ },
+ 'text': 'A distant celestial object with an icy crust, displaying a light blue shade, covered with round pits and rugged terrains.'
+ }
+ ```
+
+ Speed up streaming with multithreading:
+
+ ```py
+ >>> import os
+ >>> from datasets import load_dataset
+ >>> from tqdm import tqdm
+ >>> ds = load_dataset("sshh12/planet-textures", split="train", streaming=True)
+ >>> num_threads = min(32, (os.cpu_count() or 1) + 4)
+ >>> ds = ds.decode(num_threads=num_threads)
+ >>> for _ in tqdm(ds): # 20 times faster !
+ ... ...
+ ```
+ """
+ if not self.features:
+ raise ValueError(
+ "Features decoding is only available for datasets with known features, but features are Unknown. "
+ "Please set the datasets features with `ds = ds.cast(features)`."
+ )
+ ds = self
+
+ def set_decoding(decode: bool, feature):
+ if hasattr(feature, "decode"):
+ feature.decode = decode
+
+ if enable and num_threads > 0:
+ disabled_decoding_features = self.features.copy()
+ enabled_decoding_features = self.features.copy()
+
+ _visit(disabled_decoding_features, partial(set_decoding, False))
+ _visit(enabled_decoding_features, partial(set_decoding, True))
+ ds = ds.cast(disabled_decoding_features)
+ pool = multiprocessing.pool.ThreadPool(num_threads)
+ func = partial(_apply_async, pool, enabled_decoding_features.decode_example)
+ ds = ds.map(func, features=enabled_decoding_features)
+ assert isinstance(ds._ex_iterable, MappedExamplesIterable)
+ ds._ex_iterable.max_num_running_async_map_functions_in_parallel = 2 * num_threads
+ else:
+ features = ds.features.copy()
+ _visit(features, partial(set_decoding, enable))
+ ds = ds.cast(features)
+ return ds
+
+ def _step(self, step: int, offset: int) -> "IterableDataset":
+ ex_iterable = StepExamplesIterable(self._ex_iterable, step=step, offset=offset)
+ return IterableDataset(
+ ex_iterable=ex_iterable,
+ info=self._info.copy(),
+ split=self._split,
+ formatting=self._formatting,
+ shuffling=copy.deepcopy(self._shuffling),
+ distributed=copy.deepcopy(self._distributed),
+ token_per_repo_id=self._token_per_repo_id,
+ )
+
+ def _resolve_features(self):
+ if self.features is not None:
+ return self
+ elif self._ex_iterable.is_typed:
+ features = self._ex_iterable.features
+ else:
+ features = _infer_features_from_batch(self.with_format(None)._head())
+ info = self.info.copy()
+ info.features = features
+ return IterableDataset(
+ ex_iterable=self._ex_iterable,
+ info=info,
+ split=self._split,
+ formatting=self._formatting,
+ shuffling=copy.deepcopy(self._shuffling),
+ distributed=copy.deepcopy(self._distributed),
+ token_per_repo_id=self._token_per_repo_id,
+ )
+
+ def batch(self, batch_size: int, drop_last_batch: bool = False) -> "IterableDataset":
+ """
+ Group samples from the dataset into batches.
+
+ Args:
+ batch_size (`int`): The number of samples in each batch.
+ drop_last_batch (`bool`, defaults to `False`): Whether to drop the last incomplete batch.
+
+ Example:
+ ```py
+ >>> ds = load_dataset("some_dataset", streaming=True)
+ >>> batched_ds = ds.batch(batch_size=32)
+ ```
+ """
+
+ if self.features:
+ features = Features({col: List(feature) for col, feature in self.features.items()})
+ else:
+ features = None
+ return self.map(
+ _batch_fn, batched=True, batch_size=batch_size, drop_last_batch=drop_last_batch, features=features
+ )
+
+ def to_dict(self, batch_size: Optional[int] = None, batched: bool = False) -> Union[dict, Iterator[dict]]:
+ """Returns the dataset as a Python dict. Can also return a generator for large datasets.
+
+ Args:
+ batch_size (`int`, *optional*): The size (number of rows) of the batches if `batched` is `True`.
+ Defaults to `datasets.config.DEFAULT_MAX_BATCH_SIZE`.
+
+ Returns:
+ `dict` or `Iterator[dict]`
+
+ Example:
+
+ ```py
+ >>> ds.to_dict()
+ ```
+ """
+ if batched:
+ for table in self.with_format("arrow").iter(batch_size=batch_size):
+ yield Dataset(table, fingerprint="unset").to_dict()
+ else:
+ table = pa.concat_tables(list(self.with_format("arrow").iter(batch_size=1000)))
+ return Dataset(table, fingerprint="unset").to_dict()
+
+ def to_list(self) -> list:
+ """Returns the dataset as a Python list.
+
+ Returns:
+ `list`
+
+ Example:
+
+ ```py
+ >>> ds.to_list()
+ ```
+ """
+ table = pa.concat_tables(list(self.with_format("arrow").iter(batch_size=1000)))
+ return Dataset(table, fingerprint="unset").to_list()
+
+ def to_pandas(
+ self, batch_size: Optional[int] = None, batched: bool = False
+ ) -> Union[pd.DataFrame, Iterator[pd.DataFrame]]:
+ """Returns the dataset as a `pandas.DataFrame`. Can also return a generator for large datasets.
+
+ Args:
+ batch_size (`int`, *optional*):
+ The size (number of rows) of the batches if `batched` is `True`.
+ Defaults to `datasets.config.DEFAULT_MAX_BATCH_SIZE`.
+ batched (`bool`):
+ Set to `True` to return a generator that yields the dataset as batches
+ of `batch_size` rows. Defaults to `False` (returns the whole datasets once).
+
+ Returns:
+ `pandas.DataFrame` or `Iterator[pandas.DataFrame]`
+
+ Example:
+
+ ```py
+ >>> ds.to_pandas()
+ ```
+ """
+ if batched:
+ for table in self.with_format("arrow").iter(batch_size=batch_size):
+ yield Dataset(table, fingerprint="unset").to_pandas()
+ else:
+ table = pa.concat_tables(list(self.with_format("arrow").iter(batch_size=1000)))
+ return Dataset(table, fingerprint="unset").to_pandas()
+
+ def to_polars(
+ self,
+ batch_size: Optional[int] = None,
+ batched: bool = False,
+ schema_overrides: Optional[dict] = None,
+ rechunk: bool = True,
+ ) -> Union["pl.DataFrame", Iterator["pl.DataFrame"]]:
+ """Returns the dataset as a `polars.DataFrame`. Can also return a generator for large datasets.
+
+ Args:
+ batch_size (`int`, *optional*):
+ The size (number of rows) of the batches if `batched` is `True`.
+ Defaults to `datasets.config.DEFAULT_MAX_BATCH_SIZE`.
+ batched (`bool`):
+ Set to `True` to return a generator that yields the dataset as batches
+ of `batch_size` rows. Defaults to `False` (returns the whole datasets once).
+ schema_overrides (`dict`, *optional*):
+ Support type specification or override of one or more columns; note that
+ any dtypes inferred from the schema param will be overridden.
+ rechunk (`bool`):
+ Make sure that all data is in contiguous memory. Defaults to `True`.
+ Returns:
+ `polars.DataFrame` or `Iterator[polars.DataFrame]`
+
+ Example:
+
+ ```py
+ >>> ds.to_polars()
+ ```
+ """
+ if batched:
+ for table in self.with_format("arrow").iter(batch_size=batch_size):
+ yield Dataset(table, fingerprint="unset").to_polars(schema_overrides=schema_overrides, rechunk=rechunk)
+ else:
+ table = pa.concat_tables(list(self.with_format("arrow").iter(batch_size=1000)))
+ return Dataset(table, fingerprint="unset").to_polars(schema_overrides=schema_overrides, rechunk=rechunk)
+
+ def to_csv(
+ self,
+ path_or_buf: Union[PathLike, BinaryIO],
+ batch_size: Optional[int] = None,
+ storage_options: Optional[dict] = None,
+ **to_csv_kwargs,
+ ) -> int:
+ """Exports the dataset to csv.
+
+ This iterates on the dataset and loads it completely in memory before writing it.
+
+ Args:
+ path_or_buf (`PathLike` or `FileOrBuffer`):
+ Either a path to a file (e.g. `file.csv`), a remote URI (e.g. `hf://datasets/username/my_dataset_name/data.csv`),
+ or a BinaryIO, where the dataset will be saved to in the specified format.
+ batch_size (`int`, *optional*):
+ Size of the batch to load in memory and write at once.
+ Defaults to `datasets.config.DEFAULT_MAX_BATCH_SIZE`.
+ storage_options (`dict`, *optional*):
+ Key/value pairs to be passed on to the file-system backend, if any.
+ **to_csv_kwargs (additional keyword arguments):
+ Parameters to pass to pandas's [`pandas.DataFrame.to_csv`](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.to_csv.html).
+ The parameter `index` defaults to `False` if not specified.
+ If you would like to write the index, pass `index=True` and also set a name for the index column by
+ passing `index_label`.
+
+ Returns:
+ `int`: The number of characters or bytes written.
+
+ Example:
+
+ ```py
+ >>> ds.to_csv("path/to/dataset/directory")
+ ```
+ """
+ table = pa.concat_tables(list(self.with_format("arrow").iter(batch_size=1000)))
+ return Dataset(table, fingerprint="unset").to_csv(
+ path_or_buf,
+ batch_size=batch_size,
+ storage_options=storage_options,
+ **to_csv_kwargs,
+ )
+
+ def to_json(
+ self,
+ path_or_buf: Union[PathLike, BinaryIO],
+ batch_size: Optional[int] = None,
+ storage_options: Optional[dict] = None,
+ **to_json_kwargs,
+ ) -> int:
+ """Export the dataset to JSON Lines or JSON.
+
+ This iterates on the dataset and loads it completely in memory before writing it.
+
+ The default output format is [JSON Lines](https://jsonlines.org/).
+ To export to [JSON](https://www.json.org), pass `lines=False` argument and the desired `orient`.
+
+ Args:
+ path_or_buf (`PathLike` or `FileOrBuffer`):
+ Either a path to a file (e.g. `file.json`), a remote URI (e.g. `hf://datasets/username/my_dataset_name/data.json`),
+ or a BinaryIO, where the dataset will be saved to in the specified format.
+ batch_size (`int`, *optional*):
+ Size of the batch to load in memory and write at once.
+ Defaults to `datasets.config.DEFAULT_MAX_BATCH_SIZE`.
+ storage_options (`dict`, *optional*):
+ Key/value pairs to be passed on to the file-system backend, if any.
+ **to_json_kwargs (additional keyword arguments):
+ Parameters to pass to pandas's [`pandas.DataFrame.to_json`](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.to_json.html).
+ Default arguments are `lines=True` and `orient="records".
+ The parameter `index` defaults to `False` if `orient` is `"split"` or `"table"`.
+ If you would like to write the index, pass `index=True`.
+
+ Returns:
+ `int`: The number of characters or bytes written.
+
+ Example:
+
+ ```py
+ >>> ds.to_json("path/to/dataset/directory/filename.jsonl")
+ ```
+
+ ```py
+ >>> num_shards = dataset.num_shards
+ >>> for index in range(num_shards):
+ ... shard = dataset.shard(index, num_shards)
+ ... shard.to_json(f"path/of/my/dataset/data-{index:05d}.jsonl")
+ ```
+
+ """
+ table = pa.concat_tables(list(self.with_format("arrow").iter(batch_size=1000)))
+ return Dataset(table, fingerprint="unset").to_json(
+ path_or_buf,
+ batch_size=batch_size,
+ storage_options=storage_options,
+ **to_json_kwargs,
+ )
+
+ def to_sql(
+ self,
+ name: str,
+ con: Union[str, "sqlalchemy.engine.Connection", "sqlalchemy.engine.Engine", "sqlite3.Connection"],
+ batch_size: Optional[int] = None,
+ **sql_writer_kwargs,
+ ) -> int:
+ """Exports the dataset to a SQL database.
+
+ Args:
+ name (`str`):
+ Name of SQL table.
+ con (`str` or `sqlite3.Connection` or `sqlalchemy.engine.Connection` or `sqlalchemy.engine.Connection`):
+ A [URI string](https://docs.sqlalchemy.org/en/13/core/engines.html#database-urls) or a SQLite3/SQLAlchemy connection object used to write to a database.
+ batch_size (`int`, *optional*):
+ Size of the batch to load in memory and write at once.
+ Defaults to `datasets.config.DEFAULT_MAX_BATCH_SIZE`.
+ **sql_writer_kwargs (additional keyword arguments):
+ Parameters to pass to pandas's [`pandas.DataFrame.to_sql`](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.to_sql.html).
+ The parameter `index` defaults to `False` if not specified.
+ If you would like to write the index, pass `index=True` and also set a name for the index column by
+ passing `index_label`.
+
+
+ Returns:
+ `int`: The number of records written.
+
+ Example:
+
+ ```py
+ >>> # con provided as a connection URI string
+ >>> ds.to_sql("data", "sqlite:///my_own_db.sql")
+ >>> # con provided as a sqlite3 connection object
+ >>> import sqlite3
+ >>> con = sqlite3.connect("my_own_db.sql")
+ >>> with con:
+ ... ds.to_sql("data", con)
+ ```
+ """
+ table = pa.concat_tables(list(self.with_format("arrow").iter(batch_size=1000)))
+ return Dataset(table, fingerprint="unset").to_sql(name, con, batch_size=batch_size, **sql_writer_kwargs)
+
+ def to_parquet(
+ self,
+ path_or_buf: Union[PathLike, BinaryIO],
+ batch_size: Optional[int] = None,
+ storage_options: Optional[dict] = None,
+ **parquet_writer_kwargs,
+ ) -> int:
+ """Exports the dataset to parquet
+
+ Args:
+ path_or_buf (`PathLike` or `FileOrBuffer`):
+ Either a path to a file (e.g. `file.parquet`), a remote URI (e.g. `hf://datasets/username/my_dataset_name/data.parquet`),
+ or a BinaryIO, where the dataset will be saved to in the specified format.
+ batch_size (`int`, *optional*):
+ Size of the batch to load in memory and write at once.
+ Defaults to `datasets.config.DEFAULT_MAX_BATCH_SIZE`.
+ storage_options (`dict`, *optional*):
+ Key/value pairs to be passed on to the file-system backend, if any.
+
+
+ **parquet_writer_kwargs (additional keyword arguments):
+ Parameters to pass to PyArrow's `pyarrow.parquet.ParquetWriter`.
+
+ Returns:
+ `int`: The number of characters or bytes written.
+
+ Example:
+
+ ```py
+ >>> ds.to_parquet("path/to/dataset/directory")
+ ```
+
+ ```py
+ >>> num_shards = dataset.num_shards
+ >>> for index in range(num_shards):
+ ... shard = dataset.shard(index, num_shards)
+ ... shard.to_parquet(f"path/of/my/dataset/data-{index:05d}.parquet")
+ ```
+
+ """
+ from .arrow_writer import get_arrow_writer_batch_size_from_features
+
+ batch_size = get_arrow_writer_batch_size_from_features(self.features) or config.DEFAULT_MAX_BATCH_SIZE
+ table = pa.concat_tables(list(self.with_format("arrow").iter(batch_size=batch_size)))
+ return Dataset(table, fingerprint="unset").to_parquet(
+ path_or_buf, storage_options=storage_options, **parquet_writer_kwargs
+ )
+
+ def _push_parquet_shards_to_hub_single(
+ self,
+ job_id: int,
+ num_jobs: int,
+ repo_id: str,
+ data_dir: str,
+ split: str,
+ token: Optional[str],
+ revision: Optional[str],
+ create_pr: Optional[bool],
+ # max_shard_size: Optional[Union[int, str]] = None, # TODO(QL): add arg
+ num_shards: int,
+ embed_external_files: bool,
+ ) -> Iterable[tuple[list[CommitOperationAdd], int, int]]:
+ """Pushes the dataset shards as Parquet files to the hub.
+
+ Returns:
+ additions (`List[CommitOperation]`): list of the `CommitOperationAdd` of the uploaded shards
+ uploaded_size (`int`): number of uploaded bytes to the repository
+ dataset_nbytes (`int`): approximate size in bytes of the uploaded dataset after uncompression
+ """
+
+ div = num_shards // num_jobs
+ mod = num_shards % num_jobs
+ start = div * job_id + min(job_id, mod)
+ end = start + div + (1 if job_id < mod else 0)
+
+ index_shards = (
+ (start + i, self.shard(num_shards=end - start, index=i, contiguous=True)) for i in range(end - start)
+ )
+
+ api = HfApi(endpoint=config.HF_ENDPOINT, token=token)
+
+ uploaded_size = 0
+ dataset_nbytes = 0
+ num_examples = 0
+ additions: list[CommitOperationAdd] = []
+ for index, shard in index_shards:
+ if embed_external_files:
+ from .arrow_writer import get_arrow_writer_batch_size_from_features
+
+ shard = shard.with_format("arrow")
+ shard = shard.map(
+ partial(embed_table_storage, token_per_repo_id=self._token_per_repo_id),
+ batched=True,
+ batch_size=get_arrow_writer_batch_size_from_features(shard.features),
+ )
+ shard_path_in_repo = f"{data_dir}/{split}-{index:05d}-of-{num_shards:05d}.parquet"
+ buffer = BytesIO()
+ shard.to_parquet(buffer)
+ parquet_metadata = pq.read_metadata(buffer)
+ num_examples += parquet_metadata.num_rows
+ dataset_nbytes += sum(
+ parquet_metadata.row_group(i).total_byte_size for i in range(parquet_metadata.num_row_groups)
+ )
+ parquet_content = buffer.getvalue()
+ uploaded_size += len(parquet_content)
+ del buffer
+ shard_addition = CommitOperationAdd(path_in_repo=shard_path_in_repo, path_or_fileobj=parquet_content)
+ api.preupload_lfs_files(
+ repo_id=repo_id,
+ additions=[shard_addition],
+ repo_type="dataset",
+ revision=revision,
+ create_pr=create_pr,
+ )
+ additions.append(shard_addition)
+ yield job_id, False, 1
+
+ yield job_id, True, (additions, dataset_nbytes, num_examples)
+
+ def _push_parquet_shards_to_hub(
+ self,
+ repo_id: str,
+ data_dir: str,
+ split: str,
+ token: Optional[str],
+ revision: Optional[str],
+ create_pr: Optional[bool],
+ # max_shard_size: Optional[Union[int, str]], # TODO(QL): add arg
+ num_shards: Optional[int],
+ embed_external_files: bool,
+ num_proc: Optional[int],
+ ) -> tuple[list[CommitOperationAdd], int, int, int]:
+ """Pushes the dataset shards as Parquet files to the hub.
+
+ Returns:
+ additions (`List[CommitOperation]`): list of the `CommitOperationAdd` of the uploaded shards
+ uploaded_size (`int`): number of uploaded bytes to the repository
+ dataset_nbytes (`int`): approximate size in bytes of the uploaded dataset after uncompression
+ num_examples (`int`): number of examples of the uploaded dataset
+ """
+
+ # Find decodable columns, because if there are any, we need to:
+ # embed the bytes from the files in the shards
+ decodable_columns = (
+ [k for k, v in self._info.features.items() if require_decoding(v, ignore_decode_attribute=True)]
+ if embed_external_files
+ else []
+ )
+ embed_external_files = embed_external_files and bool(decodable_columns)
+
+ if num_shards is None:
+ # TODO(QL): this can depend on max_shard_size later
+ num_shards = self.num_shards
+
+ additions: list[CommitOperationAdd] = []
+ dataset_nbytes = num_examples = 0
+
+ num_jobs = num_proc or 1
+ kwargs_iterable = [
+ {
+ "self": self.shard(num_shards=num_jobs, index=job_id, contiguous=True),
+ "job_id": job_id,
+ "num_jobs": num_jobs,
+ "repo_id": repo_id,
+ "data_dir": data_dir,
+ "split": split,
+ "token": token,
+ "revision": revision,
+ "create_pr": create_pr,
+ "num_shards": num_shards,
+ "embed_external_files": embed_external_files,
+ }
+ for job_id in range(num_jobs)
+ ]
+ desc = "Uploading the dataset shards"
+ desc += f" (num_proc={num_proc})" if num_proc is not None and num_proc >= 1 else ""
+ pbar = hf_tqdm(
+ unit=" shards",
+ total=num_shards,
+ desc=desc,
+ )
+ with contextlib.nullcontext() if num_proc is None or num_proc < 1 else Pool(num_proc) as pool:
+ update_stream = (
+ IterableDataset._push_parquet_shards_to_hub_single(**kwargs_iterable[0])
+ if pool is None
+ else iflatmap_unordered(
+ pool,
+ IterableDataset._push_parquet_shards_to_hub_single,
+ kwargs_iterable=kwargs_iterable,
+ )
+ )
+ for job_id, done, content in update_stream:
+ if not done:
+ pbar.update(content)
+ else:
+ additions += content[0]
+ dataset_nbytes += content[1]
+ num_examples += content[2]
+ if pool is not None:
+ pool.close()
+ pool.join()
+
+ uploaded_size = sum(addition.upload_info.size for addition in additions)
+ return additions, uploaded_size, dataset_nbytes, num_examples
+
+ def push_to_hub(
+ self,
+ repo_id: str,
+ config_name: str = "default",
+ set_default: Optional[bool] = None,
+ split: Optional[str] = None,
+ data_dir: Optional[str] = None,
+ commit_message: Optional[str] = None,
+ commit_description: Optional[str] = None,
+ private: Optional[bool] = None,
+ token: Optional[str] = None,
+ revision: Optional[str] = None,
+ create_pr: Optional[bool] = False,
+ # max_shard_size: Optional[Union[int, str]] = None, # TODO(QL): add arg
+ num_shards: Optional[int] = None,
+ embed_external_files: bool = True,
+ num_proc: Optional[int] = None,
+ ) -> CommitInfo:
+ """Pushes the dataset to the hub as a Parquet dataset.
+ The dataset is pushed using HTTP requests and does not need to have neither git or git-lfs installed.
+
+ The resulting Parquet files are self-contained by default. If your dataset contains [`Image`], [`Audio`] or [`Video`]
+ data, the Parquet files will store the bytes of your images or audio files.
+ You can disable this by setting `embed_external_files` to `False`.
+
+ Args:
+ repo_id (`str`):
+ The ID of the repository to push to in the following format: `/` or
+ `/`. Also accepts ``, which will default to the namespace
+ of the logged-in user.
+ config_name (`str`, defaults to "default"):
+ The configuration name (or subset) of a dataset. Defaults to "default".
+ set_default (`bool`, *optional*):
+ Whether to set this configuration as the default one. Otherwise, the default configuration is the one
+ named "default".
+ split (`str`, *optional*):
+ The name of the split that will be given to that dataset. Defaults to `self.split`.
+ data_dir (`str`, *optional*):
+ Directory name that will contain the uploaded data files. Defaults to the `config_name` if different
+ from "default", else "data".
+ commit_message (`str`, *optional*):
+ Message to commit while pushing. Will default to `"Upload dataset"`.
+ commit_description (`str`, *optional*):
+ Description of the commit that will be created.
+ Additionally, description of the PR if a PR is created (`create_pr` is True).
+ private (`bool`, *optional*):
+ Whether to make the repo private. If `None` (default), the repo will be public unless the
+ organization's default is private. This value is ignored if the repo already exists.
+ token (`str`, *optional*):
+ An optional authentication token for the Hugging Face Hub. If no token is passed, will default
+ to the token saved locally when logging in with `huggingface-cli login`. Will raise an error
+ if no token is passed and the user is not logged-in.
+ revision (`str`, *optional*):
+ Branch to push the uploaded files to. Defaults to the `"main"` branch.
+ create_pr (`bool`, *optional*, defaults to `False`):
+ Whether to create a PR with the uploaded files or directly commit.
+ num_shards (`int`, *optional*):
+ Number of shards to write. Equals to this dataset's `.num_shards` by default.
+ embed_external_files (`bool`, defaults to `True`):
+ Whether to embed file bytes in the shards.
+ In particular, this will do the following before the push for the fields of type:
+
+ - [`Audio`] and [`Image`]: remove local path information and embed file content in the Parquet files.
+ num_proc (`int`, *optional*, defaults to `None`):
+ Number of processes when preparing and uploading the dataset.
+ This is helpful if the dataset is made of many samples and transformations.
+ Multiprocessing is disabled by default.
+
+ Return:
+ huggingface_hub.CommitInfo
+
+ Example:
+
+ ```python
+ >>> dataset.push_to_hub("/")
+ >>> dataset_dict.push_to_hub("/", private=True)
+ >>> dataset.push_to_hub("/", num_shards=1024)
+ ```
+
+ If your dataset has multiple splits (e.g. train/validation/test):
+
+ ```python
+ >>> train_dataset.push_to_hub("/", split="train")
+ >>> val_dataset.push_to_hub("/", split="validation")
+ >>> # later
+ >>> dataset = load_dataset("/")
+ >>> train_dataset = dataset["train"]
+ >>> val_dataset = dataset["validation"]
+ ```
+
+ If you want to add a new configuration (or subset) to a dataset (e.g. if the dataset has multiple tasks/versions/languages):
+
+ ```python
+ >>> english_dataset.push_to_hub("/", "en")
+ >>> french_dataset.push_to_hub("/", "fr")
+ >>> # later
+ >>> english_dataset = load_dataset("/", "en")
+ >>> french_dataset = load_dataset("/", "fr")
+ ```
+ """
+ if "Video(" in str(self.features):
+ raise NotImplementedError(
+ "push_to_hub is not implemented for video datasets, instead you should upload the video files "
+ "using e.g. the huggingface_hub library and optionally upload a metadata.csv or metadata.jsonl "
+ "file containing other information like video captions, features or labels. More information "
+ "at https://huggingface.co/docs/datasets/main/en/video_load#videofolder"
+ )
+ if num_proc is not None and num_proc > self.num_shards:
+ logger.warning(
+ f"Too many num_proc: {num_proc} (max is dataset.num_shards={self.num_shards}). "
+ f"Stopping {num_proc - self.num_shards} processes."
+ )
+ logger.info(
+ f"To parallelize data loading, we give each process some shards (or data sources) to process. "
+ f"Therefore it's unnecessary to have a number of processes greater than dataset.num_shards={self.num_shards}. "
+ f"To enable more parallelism, please split the dataset in more files than {self.num_shards}."
+ )
+ num_proc = self.num_shards
+
+ if config_name == "data":
+ raise ValueError("`config_name` cannot be 'data'. Please, choose another name for configuration.")
+
+ # if max_shard_size is not None and num_shards is not None:
+ # raise ValueError(
+ # "Failed to push_to_hub: please specify either max_shard_size or num_shards, but not both."
+ # )
+
+ if split is None:
+ split = str(self.split) if self.split is not None else "train"
+
+ if not re.match(_split_re, split):
+ raise ValueError(f"Split name should match '{_split_re}' but got '{split}'.")
+
+ api = HfApi(endpoint=config.HF_ENDPOINT, token=token)
+
+ try:
+ repo_id = api.repo_info(repo_id, repo_type="dataset").id
+ except RepositoryNotFoundError:
+ repo_url = api.create_repo(
+ repo_id,
+ repo_type="dataset",
+ private=private,
+ exist_ok=True,
+ )
+ repo_id = repo_url.repo_id
+
+ if revision is not None and not revision.startswith("refs/pr/"):
+ # We do not call create_branch for a PR reference: 400 Bad Request
+ api.create_branch(repo_id, branch=revision, token=token, repo_type="dataset", exist_ok=True)
+
+ if not data_dir:
+ data_dir = config_name if config_name != "default" else "data" # for backward compatibility
+
+ additions, uploaded_size, dataset_nbytes, num_examples = self._push_parquet_shards_to_hub(
+ repo_id=repo_id,
+ data_dir=data_dir,
+ split=split,
+ token=token,
+ revision=revision,
+ # max_shard_size=max_shard_size, # TODO(QL): add arg
+ num_shards=num_shards,
+ create_pr=create_pr,
+ embed_external_files=embed_external_files,
+ num_proc=num_proc,
+ )
+
+ def get_deletions_and_dataset_card() -> tuple[str, list[CommitOperationDelete], str, Optional[str]]:
+ parent_commit = api.repo_info(repo_id, repo_type="dataset", revision=revision).sha
+
+ # Check if the repo already has a README.md and/or a dataset_infos.json to update them with the new split info (size and pattern)
+ # and delete old split shards (if they exist)
+ repo_with_dataset_card, repo_with_dataset_infos = False, False
+ deletions: list[CommitOperationDelete] = []
+ deleted_size = 0
+ repo_splits: list[str] = [] # use a list to keep the order of the splits
+ repo_files_to_add = [addition.path_in_repo for addition in additions]
+ for repo_file in api.list_repo_tree(
+ repo_id=repo_id, revision=parent_commit, repo_type="dataset", token=token, recursive=True
+ ):
+ if not isinstance(repo_file, RepoFile):
+ continue
+ if repo_file.rfilename == config.REPOCARD_FILENAME:
+ repo_with_dataset_card = True
+ elif repo_file.rfilename == config.DATASETDICT_INFOS_FILENAME:
+ repo_with_dataset_infos = True
+ elif (
+ repo_file.rfilename.startswith(f"{data_dir}/{split}-")
+ and repo_file.rfilename not in repo_files_to_add
+ ):
+ deletions.append(CommitOperationDelete(path_in_repo=repo_file.rfilename))
+ deleted_size += repo_file.size
+ elif fnmatch.fnmatch(
+ repo_file.rfilename,
+ PUSH_TO_HUB_WITHOUT_METADATA_CONFIGS_SPLIT_PATTERN_SHARDED.replace("{split}", "*"),
+ ):
+ pattern = glob_pattern_to_regex(PUSH_TO_HUB_WITHOUT_METADATA_CONFIGS_SPLIT_PATTERN_SHARDED)
+ split_pattern_fields = string_to_dict(repo_file.rfilename, pattern)
+ assert split_pattern_fields is not None
+ repo_split = split_pattern_fields["split"]
+ if repo_split not in repo_splits:
+ repo_splits.append(repo_split)
+
+ organization, dataset_name = repo_id.split("/") if "/" in repo_id else (None, repo_id)
+ info_to_dump = self.info.copy()
+ info_to_dump.download_checksums = None
+ info_to_dump.download_size = uploaded_size
+ info_to_dump.dataset_size = dataset_nbytes
+ info_to_dump.size_in_bytes = uploaded_size + dataset_nbytes
+ info_to_dump.config_name = config_name
+ info_to_dump.splits = SplitDict(
+ {
+ split: SplitInfo(
+ split, num_bytes=dataset_nbytes, num_examples=num_examples, dataset_name=dataset_name
+ )
+ }
+ )
+ # get the info from the README to update them
+ if repo_with_dataset_card:
+ dataset_card_path = api.hf_hub_download(
+ repo_id, config.REPOCARD_FILENAME, repo_type="dataset", revision=parent_commit
+ )
+ dataset_card = DatasetCard.load(Path(dataset_card_path))
+ dataset_card_data = dataset_card.data
+ metadata_configs = MetadataConfigs.from_dataset_card_data(dataset_card_data)
+ dataset_infos: DatasetInfosDict = DatasetInfosDict.from_dataset_card_data(dataset_card_data)
+ if dataset_infos and config_name in dataset_infos:
+ repo_info = dataset_infos[config_name]
+ else:
+ repo_info = None
+ # get the deprecated dataset_infos.json to update them
+ elif repo_with_dataset_infos:
+ dataset_card = None
+ dataset_card_data = DatasetCardData()
+ metadata_configs = MetadataConfigs()
+ dataset_infos_path = api.hf_hub_download(
+ repo_id, config.DATASETDICT_INFOS_FILENAME, repo_type="dataset", revision=parent_commit
+ )
+ with open(dataset_infos_path, encoding="utf-8") as f:
+ dataset_infos: dict = json.load(f)
+ dataset_info = dataset_infos.get(config_name, None) if dataset_infos else None
+ repo_info = DatasetInfo.from_dict(dataset_info) if dataset_info else None
+ else:
+ dataset_card = None
+ dataset_card_data = DatasetCardData()
+ metadata_configs = MetadataConfigs()
+ repo_info = None
+ # update the total info to dump from existing info
+ if repo_info is not None:
+ logger.info("Updating downloaded metadata with the new split.")
+ if repo_info.splits and list(repo_info.splits) != [split]:
+ if self._info.features != repo_info.features:
+ raise ValueError(
+ f"Features of the new split don't match the features of the existing splits on the hub: {self._info.features} != {repo_info.features}"
+ )
+
+ if split in repo_info.splits:
+ repo_info.download_size -= deleted_size
+ repo_info.dataset_size -= repo_info.splits.get(split, SplitInfo()).num_bytes or 0
+
+ repo_info.download_checksums = None
+ repo_info.download_size = (repo_info.download_size or 0) + uploaded_size
+ repo_info.dataset_size = (repo_info.dataset_size or 0) + dataset_nbytes
+ repo_info.size_in_bytes = repo_info.download_size + repo_info.dataset_size
+ repo_info.splits.pop(split, None)
+ repo_info.splits[split] = SplitInfo(
+ split, num_bytes=dataset_nbytes, num_examples=len(self), dataset_name=dataset_name
+ )
+ info_to_dump = repo_info
+ # create the metadata configs if it was uploaded with push_to_hub before metadata configs existed
+ if not metadata_configs and repo_splits:
+ default_metadata_configs_to_dump = {
+ "data_files": [{"split": split, "path": f"data/{split}-*"} for split in repo_splits]
+ }
+ MetadataConfigs({"default": default_metadata_configs_to_dump}).to_dataset_card_data(dataset_card_data)
+ # update the metadata configs
+ if config_name in metadata_configs:
+ metadata_config = metadata_configs[config_name]
+ if "data_files" in metadata_config:
+ data_files_to_dump = sanitize_patterns(metadata_config["data_files"])
+ else:
+ data_files_to_dump = {}
+ # add the new split
+ data_files_to_dump[split] = [f"{data_dir}/{split}-*"]
+ metadata_config_to_dump = {
+ "data_files": [
+ {
+ "split": _split,
+ "path": _pattern[0] if len(_pattern) == 1 else _pattern,
+ }
+ for _split, _pattern in data_files_to_dump.items()
+ ]
+ }
+ else:
+ metadata_config_to_dump = {"data_files": [{"split": split, "path": f"{data_dir}/{split}-*"}]}
+ configs_to_dump = {config_name: metadata_config_to_dump}
+ if set_default and config_name != "default":
+ if metadata_configs:
+ current_default_config_name = metadata_configs.get_default_config_name()
+ if current_default_config_name == "default":
+ raise ValueError(
+ "There exists a configuration named 'default'. To set a different configuration as default, "
+ "rename the 'default' one first."
+ )
+ if current_default_config_name:
+ _ = metadata_configs[current_default_config_name].pop("default")
+ configs_to_dump[current_default_config_name] = metadata_configs[current_default_config_name]
+ metadata_config_to_dump["default"] = True
+ # push to the deprecated dataset_infos.json
+ if repo_with_dataset_infos:
+ dataset_infos_path = api.hf_hub_download(
+ repo_id, config.DATASETDICT_INFOS_FILENAME, repo_type="dataset", revision=parent_commit
+ )
+ with open(dataset_infos_path, encoding="utf-8") as f:
+ dataset_infos: dict = json.load(f)
+ dataset_infos[config_name] = asdict(info_to_dump)
+ new_dataset_infos = json.dumps(dataset_infos, indent=4)
+ else:
+ new_dataset_infos = None
+ # push to README
+ DatasetInfosDict({config_name: info_to_dump}).to_dataset_card_data(dataset_card_data)
+ MetadataConfigs(configs_to_dump).to_dataset_card_data(dataset_card_data)
+ new_dataset_card = (
+ DatasetCard(f"---\n{dataset_card_data}\n---\n") if dataset_card is None else dataset_card
+ )
+ return parent_commit, deletions, new_dataset_card, new_dataset_infos
+
+ commit_message = commit_message if commit_message is not None else "Upload dataset"
+ if len(additions) > config.UPLOADS_MAX_NUMBER_PER_COMMIT:
+ logger.info(
+ f"Number of files to upload is larger than {config.UPLOADS_MAX_NUMBER_PER_COMMIT}. Splitting the push into multiple commits."
+ )
+ num_commits = math.ceil(len(additions) / config.UPLOADS_MAX_NUMBER_PER_COMMIT)
+ for i in range(0, num_commits):
+ operations = additions[
+ i * config.UPLOADS_MAX_NUMBER_PER_COMMIT : (i + 1) * config.UPLOADS_MAX_NUMBER_PER_COMMIT
+ ]
+ for retry, sleep_time in enumerate(itertools.chain(range(10), itertools.repeat(30)), start=1):
+ # We need to retry if another commit happens at the same time
+ sleep_time *= 1 + random.random()
+ try:
+ commit_info = api.create_commit(
+ repo_id,
+ operations=operations,
+ commit_message=commit_message + f" (part {i:05d}-of-{num_commits:05d})",
+ commit_description=commit_description,
+ repo_type="dataset",
+ revision=revision,
+ create_pr=create_pr,
+ )
+ except HfHubHTTPError as err:
+ if (
+ err.__context__
+ and isinstance(err.__context__, HfHubHTTPError)
+ and err.__context__.response.status_code == 409
+ ):
+ # 409 is Conflict (another commit is in progress)
+ time.sleep(sleep_time)
+ logger.info(
+ f"Retrying intermediate commit for {repo_id}, {config_name} ({retry}/n with status_code {err.__context__.response.status_code})"
+ )
+ continue
+ else:
+ raise
+ break
+ logger.info(
+ f"Commit #{i + 1} completed"
+ + (f" (still {num_commits - i - 1} to go)" if num_commits - i - 1 else "")
+ + "."
+ )
+ last_commit_additions = []
+ else:
+ last_commit_additions = additions
+
+ for retry, sleep_time in enumerate(itertools.chain(range(10), itertools.repeat(30)), start=1):
+ # We need to retry if there was a commit in between in case it touched the dataset card data
+ sleep_time *= 1 + random.random()
+ parent_commit, deletions, dataset_card, dataset_infos = get_deletions_and_dataset_card()
+ dataset_card_additions = []
+ if dataset_infos:
+ dataset_card_additions.append(
+ CommitOperationAdd(
+ path_in_repo=config.DATASETDICT_INFOS_FILENAME,
+ path_or_fileobj=dataset_infos.encode("utf-8"),
+ )
+ )
+ dataset_card_additions.append(
+ CommitOperationAdd(path_in_repo=config.REPOCARD_FILENAME, path_or_fileobj=str(dataset_card).encode())
+ )
+ try:
+ commit_info = api.create_commit(
+ repo_id,
+ operations=last_commit_additions + dataset_card_additions + deletions,
+ commit_message=commit_message,
+ commit_description=commit_description,
+ repo_type="dataset",
+ revision=revision,
+ create_pr=create_pr,
+ parent_commit=parent_commit,
+ )
+ except HfHubHTTPError as err:
+ if (
+ err.__context__
+ and isinstance(err.__context__, HfHubHTTPError)
+ and err.__context__.response.status_code in (412, 409)
+ ):
+ # 412 is Precondition failed (parent_commit isn't satisfied)
+ # 409 is Conflict (another commit is in progress)
+ time.sleep(sleep_time)
+ logger.info(
+ f"Retrying commit for {repo_id}, {config_name} ({retry}/n with status_code {err.__context__.response.status_code})"
+ )
+ continue
+ else:
+ raise
+ break
+
+ return commit_info
+
+
+def _concatenate_iterable_datasets(
+ dsets: list[IterableDataset],
+ info: Optional[DatasetInfo] = None,
+ split: Optional[NamedSplit] = None,
+ axis: int = 0,
+) -> IterableDataset:
+ """
+ Converts a list of `IterableDataset` with the same schema into a single `IterableDataset`.
+ Missing data are filled with None values.
+
+
+
+ Args:
+ dsets (`List[datasets.IterableDataset]`): List of Datasets to concatenate.
+ info (`DatasetInfo`, optional): Dataset information, like description, citation, etc.
+ split (`NamedSplit`, optional): Name of the dataset split.
+ axis (``{0, 1}``, default ``0``, meaning over rows):
+ Axis to concatenate over, where ``0`` means over rows (vertically) and ``1`` means over columns
+ (horizontally).
+
+ *New in version 1.6.0*
+
+ Example:
+
+ ```py
+ >>> ds3 = _concatenate_iterable_datasets([ds1, ds2])
+ ```
+ """
+ dsets = [d._resolve_features() for d in dsets]
+
+ # Perform checks (and a potentional cast if axis=0)
+ if axis == 0:
+ _check_if_features_can_be_aligned([dset.features for dset in dsets])
+ else:
+ _check_column_names([col_name for dset in dsets for col_name in dset.features])
+
+ # Check format is consistent; if so, will set format for concatenated dataset
+ if all(dset._formatting is None for dset in dsets):
+ formatting = None
+ elif any(dset._formatting is None for dset in dsets):
+ formatting = None
+ logger.info(
+ "Some of the datasets have disparate format or format not set. Resetting the format of the concatenated dataset."
+ )
+ else:
+ format_type_set = {dset._formatting.format_type for dset in dsets}
+ if len(format_type_set) == 1:
+ format_type = format_type_set.pop()
+ formatting = FormattingConfig(format_type=format_type)
+ else:
+ formatting = None
+ logger.info(
+ "Some of the datasets have disparate format or format not set. Resetting the format of the concatenated dataset."
+ )
+
+ # TODO: improve this to account for a mix of ClassLabel and Value for example
+ # right now it would keep the type of the first dataset in the list
+ features = Features(
+ {k: v for features in _align_features([dset.features for dset in dsets]) for k, v in features.items()}
+ )
+
+ ex_iterables = [copy.deepcopy(d._ex_iterable) for d in dsets]
+ if axis == 0:
+ ex_iterable = VerticallyConcatenatedMultiSourcesExamplesIterable(ex_iterables)
+ else:
+ ex_iterable = HorizontallyConcatenatedMultiSourcesExamplesIterable(ex_iterables)
+ # Set new info - we update the features
+ # setting the features also ensures to fill missing columns with None
+ if info is None:
+ info = DatasetInfo.from_merge([d.info for d in dsets])
+ else:
+ info = info.copy()
+ info.features = features
+ # Get all the auth tokens per repository - in case the datasets come from different private repositories
+ token_per_repo_id = {repo_id: token for dataset in dsets for repo_id, token in dataset._token_per_repo_id.items()}
+ # Return new daset
+ return IterableDataset(
+ ex_iterable=ex_iterable,
+ info=info,
+ split=split,
+ token_per_repo_id=token_per_repo_id,
+ formatting=formatting,
+ )
+
+
+def _interleave_iterable_datasets(
+ datasets: list[IterableDataset],
+ probabilities: Optional[list[float]] = None,
+ seed: Optional[int] = None,
+ info: Optional[DatasetInfo] = None,
+ split: Optional[NamedSplit] = None,
+ stopping_strategy: Literal[
+ "first_exhausted", "all_exhausted", "all_exhausted_without_replacement"
+ ] = "first_exhausted",
+) -> IterableDataset:
+ """
+ Interleave several iterable datasets (sources) into a single iterable dataset.
+ The new iterable dataset alternates between the sources to yield examples.
+ If `probabilities = None` (default) the iterable dataset will cycles through the sources in order for each next example in the iteration.
+ If `probabilities` is not `None, the iterable dataset will sample a random source according to the provided probabilities for each next examples in the iteration.
+
+
+
+ Args:
+ datasets (`List[IterableDataset]`): list of datasets to interleave
+ probabilities (`List[float]`, optional, default None): If specified, the new iterable dataset samples
+ examples from one source at a time according to these probabilities.
+ seed (`int`, optional, default None): The random seed used to choose a source for each example.
+ stopping_strategy (`str`, defaults to `first_exhausted`):
+ Two strategies are proposed right now.
+ By default, `first_exhausted` is an undersampling strategy, i.e the dataset construction is stopped as soon as one dataset has ran out of samples.
+ If the strategy is `all_exhausted`, we use an oversampling strategy, i.e the dataset construction is stopped as soon as every samples of every dataset has been added at least once.
+ Note that if the strategy is `all_exhausted`, the interleaved dataset size can get enormous:
+ - with no probabilities, the resulting dataset will have max_length_datasets*nb_dataset samples.
+ - with given probabilities, the resulting dataset will have more samples if some datasets have really low probability of visiting.
+
+ Output:
+ `datasets.IterableDataset`
+ """
+ datasets = [d._resolve_features() for d in datasets]
+
+ # Perform checks
+ _check_if_features_can_be_aligned([dset.features for dset in datasets])
+
+ # TODO: improve this to account for a mix of ClassLabel and Value for example
+ # right now it would keep the type of the first dataset in the list
+ features = Features(
+ {k: v for features in _align_features([dset.features for dset in datasets]) for k, v in features.items()}
+ )
+
+ ex_iterables = [copy.deepcopy(d._ex_iterable) for d in datasets]
+ if all(ex_iterable.iter_arrow for ex_iterable in ex_iterables):
+ ex_iterables = [RebatchedArrowExamplesIterable(ex_iterable, batch_size=1) for ex_iterable in ex_iterables]
+ # Use cycling or random cycling of sources
+ if probabilities is None:
+ ex_iterable = CyclingMultiSourcesExamplesIterable(ex_iterables, stopping_strategy=stopping_strategy)
+ else:
+ generator = np.random.default_rng(seed)
+ ex_iterable = RandomlyCyclingMultiSourcesExamplesIterable(
+ ex_iterables,
+ generator=generator,
+ probabilities=probabilities,
+ stopping_strategy=stopping_strategy,
+ )
+ # Set new info - we update the features
+ # setting the features also ensures to fill missing columns with None
+ if info is None:
+ info = DatasetInfo.from_merge([d.info for d in datasets])
+ else:
+ info = info.copy()
+ info.features = features
+ # Get all the auth tokens per repository - in case the datasets come from different private repositories
+ token_per_repo_id = {
+ repo_id: token for dataset in datasets for repo_id, token in dataset._token_per_repo_id.items()
+ }
+ # Return new daset
+ return IterableDataset(ex_iterable=ex_iterable, info=info, split=split, token_per_repo_id=token_per_repo_id)
+
+
+def _split_by_node_iterable_dataset(dataset: IterableDataset, rank: int, world_size: int) -> IterableDataset:
+ """
+ Split an iterable dataset for the node at rank `rank` in a pool of nodes of size `world_size`.
+
+ If the dataset has a number of shards that is a factor of `world_size` (i.e. if `dataset.num_shards % world_size == 0`),
+ then the shards are evenly assigned across the nodes, which is the most optimized.
+ Otherwise, each node keeps 1 example out of `world_size`, skipping the other examples.
+
+ Args:
+ dataset ([`IterableDataset`]):
+ The iterable dataset to split by node.
+ rank (`int`):
+ Rank of the current node.
+ world_size (`int`):
+ Total number of nodes.
+
+ Returns:
+ [`IterableDataset`]: The iterable dataset to be used on the node at rank `rank`.
+ """
+ if dataset._distributed:
+ rank = world_size * dataset._distributed.rank + rank
+ world_size = world_size * dataset._distributed.world_size
+ distributed = DistributedConfig(rank=rank, world_size=world_size)
+ return IterableDataset(
+ ex_iterable=dataset._ex_iterable,
+ info=dataset._info.copy(),
+ split=dataset._split,
+ formatting=dataset._formatting,
+ shuffling=copy.deepcopy(dataset._shuffling),
+ distributed=distributed,
+ token_per_repo_id=dataset._token_per_repo_id,
+ )
+
+
+async def _apply_async(pool, func, x):
+ future = pool.apply_async(func, (x,))
+ while True:
+ if future.ready():
+ return future.get()
+ else:
+ await asyncio.sleep(0)
+
+
+def _batch_fn(unbatched):
+ return {k: [v] for k, v in unbatched.items()}
diff --git a/venv/lib/python3.10/site-packages/datasets/keyhash.py b/venv/lib/python3.10/site-packages/datasets/keyhash.py
new file mode 100644
index 0000000000000000000000000000000000000000..5ba2686e259868e19ee7439d21b995dfd4c8de51
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/keyhash.py
@@ -0,0 +1,104 @@
+# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+
+"""
+Hashing function for dataset keys using `hashlib.md5`
+
+Requirements for the hash function:
+
+- Provides a uniformly distributed hash from random space
+- Adequately fast speed
+- Working with multiple input types (in this case, `str`, `int` or `bytes`)
+- Should be platform independent (generates same hash on different OS and systems)
+
+The hashing function provides a unique 128-bit integer hash of the key provided.
+
+The split name is being used here as the hash salt to avoid having same hashes
+in different splits due to same keys
+"""
+
+from typing import Union
+
+from huggingface_hub.utils import insecure_hashlib
+
+
+def _as_bytes(hash_data: Union[str, int, bytes, bytearray]) -> bytes:
+ """
+ Returns the input hash_data in its bytes form
+
+ Args:
+ hash_data: the hash salt/key to be converted to bytes
+ """
+ if isinstance(hash_data, (bytes, bytearray)):
+ # Data already in bytes, returns as it as
+ return hash_data
+ elif isinstance(hash_data, str):
+ # We keep the data as it as for it ot be later encoded to UTF-8
+ # However replace `\\` with `/` for Windows compatibility
+ hash_data = hash_data.replace("\\", "/")
+ elif isinstance(hash_data, int):
+ hash_data = str(hash_data)
+ else:
+ # If data is not of the required type, raise error
+ raise InvalidKeyError(hash_data)
+
+ return hash_data.encode("utf-8")
+
+
+class InvalidKeyError(Exception):
+ """Raises an error when given key is of invalid datatype."""
+
+ def __init__(self, hash_data):
+ self.prefix = "\nFAILURE TO GENERATE DATASET: Invalid key type detected"
+ self.err_msg = f"\nFound Key {hash_data} of type {type(hash_data)}"
+ self.suffix = "\nKeys should be either str, int or bytes type"
+ super().__init__(f"{self.prefix}{self.err_msg}{self.suffix}")
+
+
+class DuplicatedKeysError(Exception):
+ """Raise an error when duplicate key found."""
+
+ def __init__(self, key, duplicate_key_indices, fix_msg=""):
+ self.key = key
+ self.duplicate_key_indices = duplicate_key_indices
+ self.fix_msg = fix_msg
+ self.prefix = "Found multiple examples generated with the same key"
+ if len(duplicate_key_indices) <= 20:
+ self.err_msg = f"\nThe examples at index {', '.join(duplicate_key_indices)} have the key {key}"
+ else:
+ self.err_msg = f"\nThe examples at index {', '.join(duplicate_key_indices[:20])}... ({len(duplicate_key_indices) - 20} more) have the key {key}"
+ self.suffix = "\n" + fix_msg if fix_msg else ""
+ super().__init__(f"{self.prefix}{self.err_msg}{self.suffix}")
+
+
+class KeyHasher:
+ """KeyHasher class for providing hash using md5"""
+
+ def __init__(self, hash_salt: str):
+ self._split_md5 = insecure_hashlib.md5(_as_bytes(hash_salt))
+
+ def hash(self, key: Union[str, int, bytes]) -> int:
+ """Returns 128-bits unique hash of input key
+
+ Args:
+ key: the input key to be hashed (should be str, int or bytes)
+
+ Returns: 128-bit int hash key"""
+ md5 = self._split_md5.copy()
+ byte_key = _as_bytes(key)
+ md5.update(byte_key)
+ # Convert to integer with hexadecimal conversion
+ return int(md5.hexdigest(), 16)
diff --git a/venv/lib/python3.10/site-packages/datasets/load.py b/venv/lib/python3.10/site-packages/datasets/load.py
new file mode 100644
index 0000000000000000000000000000000000000000..ae3b9825970c1109dad2493e69a6141cd95858b0
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/load.py
@@ -0,0 +1,1481 @@
+# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""Access datasets."""
+
+import glob
+import importlib
+import inspect
+import json
+import os
+import posixpath
+from collections import Counter
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Optional, Union
+
+import fsspec
+import httpx
+import requests
+import yaml
+from fsspec.core import url_to_fs
+from huggingface_hub import DatasetCard, DatasetCardData, HfApi
+from huggingface_hub.utils import (
+ EntryNotFoundError,
+ GatedRepoError,
+ LocalEntryNotFoundError,
+ OfflineModeIsEnabled,
+ RepositoryNotFoundError,
+ RevisionNotFoundError,
+ get_session,
+)
+
+from . import __version__, config
+from .arrow_dataset import Dataset
+from .builder import BuilderConfig, DatasetBuilder
+from .data_files import (
+ DataFilesDict,
+ DataFilesList,
+ DataFilesPatternsDict,
+ EmptyDatasetError,
+ get_data_patterns,
+ sanitize_patterns,
+)
+from .dataset_dict import DatasetDict, IterableDatasetDict
+from .download.download_config import DownloadConfig
+from .download.download_manager import DownloadMode
+from .download.streaming_download_manager import StreamingDownloadManager, xbasename, xglob, xjoin
+from .exceptions import DataFilesNotFoundError, DatasetNotFoundError
+from .features import Features
+from .features.features import _fix_for_backward_compatible_features
+from .fingerprint import Hasher
+from .info import DatasetInfo, DatasetInfosDict
+from .iterable_dataset import IterableDataset
+from .naming import camelcase_to_snakecase, snakecase_to_camelcase
+from .packaged_modules import (
+ _EXTENSION_TO_MODULE,
+ _MODULE_TO_EXTENSIONS,
+ _MODULE_TO_METADATA_FILE_NAMES,
+ _PACKAGED_DATASETS_MODULES,
+)
+from .packaged_modules.folder_based_builder.folder_based_builder import FolderBasedBuilder
+from .splits import Split
+from .utils import _dataset_viewer
+from .utils.file_utils import (
+ _raise_if_offline_mode_is_enabled,
+ cached_path,
+ get_datasets_user_agent,
+ is_relative_path,
+ relative_to_absolute_path,
+)
+from .utils.hub import hf_dataset_url
+from .utils.info_utils import VerificationMode, is_small_dataset
+from .utils.logging import get_logger
+from .utils.metadata import MetadataConfigs
+from .utils.typing import PathLike
+from .utils.version import Version
+
+
+logger = get_logger(__name__)
+
+ALL_ALLOWED_EXTENSIONS = list(_EXTENSION_TO_MODULE.keys()) + [".zip"]
+
+
+class _InitializeConfiguredDatasetBuilder:
+ """
+ From https://stackoverflow.com/questions/4647566/pickle-a-dynamically-parameterized-sub-class
+ See also ConfiguredDatasetBuilder.__reduce__
+ When called with the param value as the only argument, returns an
+ un-initialized instance of the parameterized class. Subsequent __setstate__
+ will be called by pickle.
+ """
+
+ def __call__(self, builder_cls, metadata_configs, default_config_name, name):
+ # make a simple object which has no complex __init__ (this one will do)
+ obj = _InitializeConfiguredDatasetBuilder()
+ obj.__class__ = configure_builder_class(
+ builder_cls, metadata_configs, default_config_name=default_config_name, dataset_name=name
+ )
+ return obj
+
+
+def configure_builder_class(
+ builder_cls: type[DatasetBuilder],
+ builder_configs: list[BuilderConfig],
+ default_config_name: Optional[str],
+ dataset_name: str,
+) -> type[DatasetBuilder]:
+ """
+ Dynamically create a builder class with custom builder configs parsed from README.md file,
+ i.e. set BUILDER_CONFIGS class variable of a builder class to custom configs list.
+ """
+
+ class ConfiguredDatasetBuilder(builder_cls):
+ BUILDER_CONFIGS = builder_configs
+ DEFAULT_CONFIG_NAME = default_config_name
+
+ __module__ = builder_cls.__module__ # so that the actual packaged builder can be imported
+
+ def __reduce__(self): # to make dynamically created class pickable, see _InitializeParameterizedDatasetBuilder
+ parent_builder_cls = self.__class__.__mro__[1]
+ return (
+ _InitializeConfiguredDatasetBuilder(),
+ (
+ parent_builder_cls,
+ self.BUILDER_CONFIGS,
+ self.DEFAULT_CONFIG_NAME,
+ self.dataset_name,
+ ),
+ self.__dict__.copy(),
+ )
+
+ ConfiguredDatasetBuilder.__name__ = (
+ f"{builder_cls.__name__.lower().capitalize()}{snakecase_to_camelcase(dataset_name)}"
+ )
+ ConfiguredDatasetBuilder.__qualname__ = (
+ f"{builder_cls.__name__.lower().capitalize()}{snakecase_to_camelcase(dataset_name)}"
+ )
+
+ return ConfiguredDatasetBuilder
+
+
+def import_main_class(module_path) -> Optional[type[DatasetBuilder]]:
+ """Import a module at module_path and return its main class: a DatasetBuilder"""
+ module = importlib.import_module(module_path)
+ # Find the main class in our imported module
+ module_main_cls = None
+ for name, obj in module.__dict__.items():
+ if inspect.isclass(obj) and issubclass(obj, DatasetBuilder):
+ if inspect.isabstract(obj):
+ continue
+ module_main_cls = obj
+ obj_module = inspect.getmodule(obj)
+ if obj_module is not None and module == obj_module:
+ break
+
+ return module_main_cls
+
+
+def get_dataset_builder_class(
+ dataset_module: "DatasetModule", dataset_name: Optional[str] = None
+) -> type[DatasetBuilder]:
+ builder_cls = import_main_class(dataset_module.module_path)
+ if dataset_module.builder_configs_parameters.builder_configs:
+ dataset_name = dataset_name or dataset_module.builder_kwargs.get("dataset_name")
+ if dataset_name is None:
+ raise ValueError("dataset_name should be specified but got None")
+ builder_cls = configure_builder_class(
+ builder_cls,
+ builder_configs=dataset_module.builder_configs_parameters.builder_configs,
+ default_config_name=dataset_module.builder_configs_parameters.default_config_name,
+ dataset_name=dataset_name,
+ )
+ return builder_cls
+
+
+def increase_load_count(name: str):
+ """Update the download count of a dataset."""
+ if not config.HF_HUB_OFFLINE and config.HF_UPDATE_DOWNLOAD_COUNTS:
+ try:
+ get_session().head(
+ "/".join((config.S3_DATASETS_BUCKET_PREFIX, name, name + ".py")),
+ headers={"User-Agent": get_datasets_user_agent()},
+ timeout=3,
+ )
+ except Exception:
+ pass
+
+
+def infer_module_for_data_files_list(
+ data_files_list: DataFilesList, download_config: Optional[DownloadConfig] = None
+) -> tuple[Optional[str], dict]:
+ """Infer module (and builder kwargs) from list of data files.
+
+ It picks the module based on the most common file extension.
+ In case of a draw ".parquet" is the favorite, and then alphabetical order.
+
+ Args:
+ data_files_list (DataFilesList): List of data files.
+ download_config (bool or str, optional): Mainly use `token` or `storage_options` to support different platforms and auth types.
+
+ Returns:
+ tuple[str, dict[str, Any]]: Tuple with
+ - inferred module name
+ - dict of builder kwargs
+ """
+ extensions_counter = Counter(
+ ("." + suffix.lower(), xbasename(filepath) in FolderBasedBuilder.METADATA_FILENAMES)
+ for filepath in data_files_list[: config.DATA_FILES_MAX_NUMBER_FOR_MODULE_INFERENCE]
+ for suffix in xbasename(filepath).split(".")[1:]
+ )
+ if extensions_counter:
+
+ def sort_key(ext_count: tuple[tuple[str, bool], int]) -> tuple[int, bool]:
+ """Sort by count and set ".parquet" as the favorite in case of a draw, and ignore metadata files"""
+ (ext, is_metadata), count = ext_count
+ return (not is_metadata, count, ext == ".parquet", ext == ".jsonl", ext == ".json", ext == ".csv", ext)
+
+ for (ext, _), _ in sorted(extensions_counter.items(), key=sort_key, reverse=True):
+ if ext in _EXTENSION_TO_MODULE:
+ return _EXTENSION_TO_MODULE[ext]
+ elif ext == ".zip":
+ return infer_module_for_data_files_list_in_archives(data_files_list, download_config=download_config)
+ return None, {}
+
+
+def infer_module_for_data_files_list_in_archives(
+ data_files_list: DataFilesList, download_config: Optional[DownloadConfig] = None
+) -> tuple[Optional[str], dict]:
+ """Infer module (and builder kwargs) from list of archive data files.
+
+ Args:
+ data_files_list (DataFilesList): List of data files.
+ download_config (bool or str, optional): Mainly use `token` or `storage_options` to support different platforms and auth types.
+
+ Returns:
+ tuple[str, dict[str, Any]]: Tuple with
+ - inferred module name
+ - dict of builder kwargs
+ """
+ archived_files = []
+ archive_files_counter = 0
+ for filepath in data_files_list:
+ if str(filepath).endswith(".zip"):
+ archive_files_counter += 1
+ if archive_files_counter > config.GLOBBED_DATA_FILES_MAX_NUMBER_FOR_MODULE_INFERENCE:
+ break
+ extracted = xjoin(StreamingDownloadManager().extract(filepath), "**")
+ archived_files += [
+ f.split("::")[0]
+ for f in xglob(extracted, recursive=True, download_config=download_config)[
+ : config.ARCHIVED_DATA_FILES_MAX_NUMBER_FOR_MODULE_INFERENCE
+ ]
+ ]
+ extensions_counter = Counter(
+ "." + suffix.lower() for filepath in archived_files for suffix in xbasename(filepath).split(".")[1:]
+ )
+ if extensions_counter:
+ most_common = extensions_counter.most_common(1)[0][0]
+ if most_common in _EXTENSION_TO_MODULE:
+ return _EXTENSION_TO_MODULE[most_common]
+ return None, {}
+
+
+def infer_module_for_data_files(
+ data_files: DataFilesDict, path: Optional[str] = None, download_config: Optional[DownloadConfig] = None
+) -> tuple[Optional[str], dict[str, Any]]:
+ """Infer module (and builder kwargs) from data files. Raise if module names for different splits don't match.
+
+ Args:
+ data_files ([`DataFilesDict`]): Dict of list of data files.
+ path (str, *optional*): Dataset name or path.
+ download_config ([`DownloadConfig`], *optional*):
+ Specific download configuration parameters to authenticate on the Hugging Face Hub for private remote files.
+
+ Returns:
+ tuple[str, dict[str, Any]]: Tuple with
+ - inferred module name
+ - builder kwargs
+ """
+ split_modules = {
+ split: infer_module_for_data_files_list(data_files_list, download_config=download_config)
+ for split, data_files_list in data_files.items()
+ }
+ module_name, default_builder_kwargs = next(iter(split_modules.values()))
+ if any((module_name, default_builder_kwargs) != split_module for split_module in split_modules.values()):
+ raise ValueError(f"Couldn't infer the same data file format for all splits. Got {split_modules}")
+ if not module_name:
+ raise DataFilesNotFoundError("No (supported) data files found" + (f" in {path}" if path else ""))
+ return module_name, default_builder_kwargs
+
+
+def create_builder_configs_from_metadata_configs(
+ module_path: str,
+ metadata_configs: MetadataConfigs,
+ base_path: Optional[str] = None,
+ default_builder_kwargs: dict[str, Any] = None,
+ download_config: Optional[DownloadConfig] = None,
+) -> tuple[list[BuilderConfig], str]:
+ builder_cls = import_main_class(module_path)
+ builder_config_cls = builder_cls.BUILDER_CONFIG_CLASS
+ default_config_name = metadata_configs.get_default_config_name()
+ builder_configs = []
+ default_builder_kwargs = {} if default_builder_kwargs is None else default_builder_kwargs
+
+ base_path = base_path if base_path is not None else ""
+ for config_name, config_params in metadata_configs.items():
+ config_data_files = config_params.get("data_files")
+ config_data_dir = config_params.get("data_dir")
+ config_base_path = xjoin(base_path, config_data_dir) if config_data_dir else base_path
+ try:
+ config_patterns = (
+ sanitize_patterns(config_data_files)
+ if config_data_files is not None
+ else get_data_patterns(config_base_path, download_config=download_config)
+ )
+ config_data_files_dict = DataFilesPatternsDict.from_patterns(
+ config_patterns,
+ allowed_extensions=ALL_ALLOWED_EXTENSIONS,
+ )
+ except EmptyDatasetError as e:
+ raise EmptyDatasetError(
+ f"Dataset at '{base_path}' doesn't contain data files matching the patterns for config '{config_name}',"
+ f" check `data_files` and `data_fir` parameters in the `configs` YAML field in README.md. "
+ ) from e
+ ignored_params = [
+ param for param in config_params if not hasattr(builder_config_cls, param) and param != "default"
+ ]
+ if ignored_params:
+ logger.warning(
+ f"Some datasets params were ignored: {ignored_params}. "
+ "Make sure to use only valid params for the dataset builder and to have "
+ "a up-to-date version of the `datasets` library."
+ )
+ builder_configs.append(
+ builder_config_cls(
+ name=config_name,
+ data_files=config_data_files_dict,
+ data_dir=config_data_dir,
+ **{
+ param: value
+ for param, value in {**default_builder_kwargs, **config_params}.items()
+ if hasattr(builder_config_cls, param) and param not in ("default", "data_files", "data_dir")
+ },
+ )
+ )
+ return builder_configs, default_config_name
+
+
+@dataclass
+class BuilderConfigsParameters:
+ """Dataclass containing objects related to creation of builder configurations from yaml's metadata content.
+
+ Attributes:
+ metadata_configs (`MetadataConfigs`, *optional*):
+ Configs parsed from yaml's metadata.
+ builder_configs (`list[BuilderConfig]`, *optional*):
+ List of BuilderConfig objects created from metadata_configs above.
+ default_config_name (`str`):
+ Name of default config taken from yaml's metadata.
+ """
+
+ metadata_configs: Optional[MetadataConfigs] = None
+ builder_configs: Optional[list[BuilderConfig]] = None
+ default_config_name: Optional[str] = None
+
+
+@dataclass
+class DatasetModule:
+ module_path: str
+ hash: str
+ builder_kwargs: dict
+ builder_configs_parameters: BuilderConfigsParameters = field(default_factory=BuilderConfigsParameters)
+ dataset_infos: Optional[DatasetInfosDict] = None
+
+
+class _DatasetModuleFactory:
+ def get_module(self) -> DatasetModule:
+ raise NotImplementedError
+
+
+class LocalDatasetModuleFactory(_DatasetModuleFactory):
+ """Get the module of a dataset loaded from the user's data files. The dataset builder module to use is inferred
+ from the data files extensions."""
+
+ def __init__(
+ self,
+ path: str,
+ data_dir: Optional[str] = None,
+ data_files: Optional[Union[str, list, dict]] = None,
+ download_mode: Optional[Union[DownloadMode, str]] = None,
+ ):
+ if data_dir and os.path.isabs(data_dir):
+ raise ValueError(f"`data_dir` must be relative to a dataset directory's root: {path}")
+
+ self.path = Path(path).as_posix()
+ self.name = Path(path).stem
+ self.data_files = data_files
+ self.data_dir = data_dir
+ self.download_mode = download_mode
+
+ def get_module(self) -> DatasetModule:
+ readme_path = os.path.join(self.path, config.REPOCARD_FILENAME)
+ standalone_yaml_path = os.path.join(self.path, config.REPOYAML_FILENAME)
+ dataset_card_data = DatasetCard.load(readme_path).data if os.path.isfile(readme_path) else DatasetCardData()
+ if os.path.exists(standalone_yaml_path):
+ with open(standalone_yaml_path, encoding="utf-8") as f:
+ standalone_yaml_data = yaml.safe_load(f.read())
+ if standalone_yaml_data:
+ _dataset_card_data_dict = dataset_card_data.to_dict()
+ _dataset_card_data_dict.update(standalone_yaml_data)
+ dataset_card_data = DatasetCardData(**_dataset_card_data_dict)
+ metadata_configs = MetadataConfigs.from_dataset_card_data(dataset_card_data)
+ dataset_infos = DatasetInfosDict.from_dataset_card_data(dataset_card_data)
+ # we need a set of data files to find which dataset builder to use
+ # because we need to infer module name by files extensions
+ base_path = Path(self.path, self.data_dir or "").expanduser().resolve().as_posix()
+ if self.data_files is not None:
+ patterns = sanitize_patterns(self.data_files)
+ elif metadata_configs and not self.data_dir and "data_files" in next(iter(metadata_configs.values())):
+ patterns = sanitize_patterns(next(iter(metadata_configs.values()))["data_files"])
+ else:
+ patterns = get_data_patterns(base_path)
+ data_files = DataFilesDict.from_patterns(
+ patterns,
+ base_path=base_path,
+ allowed_extensions=ALL_ALLOWED_EXTENSIONS,
+ )
+ module_name, default_builder_kwargs = infer_module_for_data_files(
+ data_files=data_files,
+ path=self.path,
+ )
+ data_files = data_files.filter(
+ extensions=_MODULE_TO_EXTENSIONS[module_name], file_names=_MODULE_TO_METADATA_FILE_NAMES[module_name]
+ )
+ module_path, _ = _PACKAGED_DATASETS_MODULES[module_name]
+ if metadata_configs:
+ builder_configs, default_config_name = create_builder_configs_from_metadata_configs(
+ module_path,
+ metadata_configs,
+ base_path=base_path,
+ default_builder_kwargs=default_builder_kwargs,
+ )
+ else:
+ builder_configs: list[BuilderConfig] = [
+ import_main_class(module_path).BUILDER_CONFIG_CLASS(
+ data_files=data_files,
+ **default_builder_kwargs,
+ )
+ ]
+ default_config_name = None
+ builder_kwargs = {
+ "base_path": self.path,
+ "dataset_name": camelcase_to_snakecase(Path(self.path).name),
+ }
+ if self.data_dir:
+ builder_kwargs["data_files"] = data_files
+ # this file is deprecated and was created automatically in old versions of push_to_hub
+ if os.path.isfile(os.path.join(self.path, config.DATASETDICT_INFOS_FILENAME)):
+ with open(os.path.join(self.path, config.DATASETDICT_INFOS_FILENAME), encoding="utf-8") as f:
+ legacy_dataset_infos = DatasetInfosDict(
+ {
+ config_name: DatasetInfo.from_dict(dataset_info_dict)
+ for config_name, dataset_info_dict in json.load(f).items()
+ }
+ )
+ if len(legacy_dataset_infos) == 1:
+ # old config e.g. named "username--dataset_name"
+ legacy_config_name = next(iter(legacy_dataset_infos))
+ legacy_dataset_infos["default"] = legacy_dataset_infos.pop(legacy_config_name)
+ legacy_dataset_infos.update(dataset_infos)
+ dataset_infos = legacy_dataset_infos
+ if default_config_name is None and len(dataset_infos) == 1:
+ default_config_name = next(iter(dataset_infos))
+
+ hash = Hasher.hash({"dataset_infos": dataset_infos, "builder_configs": builder_configs})
+ return DatasetModule(
+ module_path,
+ hash,
+ builder_kwargs,
+ dataset_infos=dataset_infos,
+ builder_configs_parameters=BuilderConfigsParameters(
+ metadata_configs=metadata_configs,
+ builder_configs=builder_configs,
+ default_config_name=default_config_name,
+ ),
+ )
+
+
+class PackagedDatasetModuleFactory(_DatasetModuleFactory):
+ """Get the dataset builder module from the ones that are packaged with the library: csv, json, etc."""
+
+ def __init__(
+ self,
+ name: str,
+ data_dir: Optional[str] = None,
+ data_files: Optional[Union[str, list, dict]] = None,
+ download_config: Optional[DownloadConfig] = None,
+ download_mode: Optional[Union[DownloadMode, str]] = None,
+ ):
+ self.name = name
+ self.data_files = data_files
+ self.data_dir = data_dir
+ self.download_config = download_config
+ self.download_mode = download_mode
+ increase_load_count(name)
+
+ def get_module(self) -> DatasetModule:
+ base_path = Path(self.data_dir or "").expanduser().resolve().as_posix()
+ patterns = (
+ sanitize_patterns(self.data_files)
+ if self.data_files is not None
+ else get_data_patterns(base_path, download_config=self.download_config)
+ )
+ data_files = DataFilesDict.from_patterns(
+ patterns,
+ download_config=self.download_config,
+ base_path=base_path,
+ )
+
+ module_path, hash = _PACKAGED_DATASETS_MODULES[self.name]
+
+ builder_kwargs = {
+ "data_files": data_files,
+ "dataset_name": self.name,
+ }
+
+ return DatasetModule(module_path, hash, builder_kwargs)
+
+
+class HubDatasetModuleFactory(_DatasetModuleFactory):
+ """
+ Get the module of a dataset loaded from data files of a dataset repository.
+ The dataset builder module to use is inferred from the data files extensions.
+ """
+
+ def __init__(
+ self,
+ name: str,
+ commit_hash: str,
+ data_dir: Optional[str] = None,
+ data_files: Optional[Union[str, list, dict]] = None,
+ download_config: Optional[DownloadConfig] = None,
+ download_mode: Optional[Union[DownloadMode, str]] = None,
+ use_exported_dataset_infos: bool = False,
+ ):
+ self.name = name
+ self.commit_hash = commit_hash
+ self.data_files = data_files
+ self.data_dir = data_dir
+ self.download_config = download_config or DownloadConfig()
+ self.download_mode = download_mode
+ self.use_exported_dataset_infos = use_exported_dataset_infos
+ increase_load_count(name)
+
+ def get_module(self) -> DatasetModule:
+ # Get the Dataset Card and fix the revision in case there are new commits in the meantime
+ api = HfApi(
+ endpoint=config.HF_ENDPOINT,
+ token=self.download_config.token,
+ library_name="datasets",
+ library_version=__version__,
+ user_agent=get_datasets_user_agent(self.download_config.user_agent),
+ )
+ try:
+ dataset_readme_path = api.hf_hub_download(
+ repo_id=self.name,
+ filename=config.REPOCARD_FILENAME,
+ repo_type="dataset",
+ revision=self.commit_hash,
+ proxies=self.download_config.proxies,
+ )
+ dataset_card_data = DatasetCard.load(dataset_readme_path).data
+ except EntryNotFoundError:
+ dataset_card_data = DatasetCardData()
+ download_config = self.download_config.copy()
+ if download_config.download_desc is None:
+ download_config.download_desc = "Downloading standalone yaml"
+ try:
+ standalone_yaml_path = cached_path(
+ hf_dataset_url(self.name, config.REPOYAML_FILENAME, revision=self.commit_hash),
+ download_config=download_config,
+ )
+ with open(standalone_yaml_path, encoding="utf-8") as f:
+ standalone_yaml_data = yaml.safe_load(f.read())
+ if standalone_yaml_data:
+ _dataset_card_data_dict = dataset_card_data.to_dict()
+ _dataset_card_data_dict.update(standalone_yaml_data)
+ dataset_card_data = DatasetCardData(**_dataset_card_data_dict)
+ except FileNotFoundError:
+ pass
+ base_path = f"hf://datasets/{self.name}@{self.commit_hash}/{self.data_dir or ''}".rstrip("/")
+ metadata_configs = MetadataConfigs.from_dataset_card_data(dataset_card_data)
+ dataset_infos = DatasetInfosDict.from_dataset_card_data(dataset_card_data)
+ if config.USE_PARQUET_EXPORT and self.use_exported_dataset_infos:
+ try:
+ exported_dataset_infos = _dataset_viewer.get_exported_dataset_infos(
+ dataset=self.name, commit_hash=self.commit_hash, token=self.download_config.token
+ )
+ exported_dataset_infos = DatasetInfosDict(
+ {
+ config_name: DatasetInfo.from_dict(exported_dataset_infos[config_name])
+ for config_name in exported_dataset_infos
+ }
+ )
+ except _dataset_viewer.DatasetViewerError:
+ exported_dataset_infos = None
+ else:
+ exported_dataset_infos = None
+ if exported_dataset_infos:
+ exported_dataset_infos.update(dataset_infos)
+ dataset_infos = exported_dataset_infos
+ # we need a set of data files to find which dataset builder to use
+ # because we need to infer module name by files extensions
+ if self.data_files is not None:
+ patterns = sanitize_patterns(self.data_files)
+ elif metadata_configs and not self.data_dir and "data_files" in next(iter(metadata_configs.values())):
+ patterns = sanitize_patterns(next(iter(metadata_configs.values()))["data_files"])
+ else:
+ patterns = get_data_patterns(base_path, download_config=self.download_config)
+ data_files = DataFilesDict.from_patterns(
+ patterns,
+ base_path=base_path,
+ allowed_extensions=ALL_ALLOWED_EXTENSIONS,
+ download_config=self.download_config,
+ )
+ module_name, default_builder_kwargs = infer_module_for_data_files(
+ data_files=data_files,
+ path=self.name,
+ download_config=self.download_config,
+ )
+ data_files = data_files.filter(
+ extensions=_MODULE_TO_EXTENSIONS[module_name], file_names=_MODULE_TO_METADATA_FILE_NAMES[module_name]
+ )
+ module_path, _ = _PACKAGED_DATASETS_MODULES[module_name]
+ if metadata_configs:
+ builder_configs, default_config_name = create_builder_configs_from_metadata_configs(
+ module_path,
+ metadata_configs,
+ base_path=base_path,
+ default_builder_kwargs=default_builder_kwargs,
+ download_config=self.download_config,
+ )
+ else:
+ builder_configs: list[BuilderConfig] = [
+ import_main_class(module_path).BUILDER_CONFIG_CLASS(
+ data_files=data_files,
+ **default_builder_kwargs,
+ )
+ ]
+ default_config_name = None
+ builder_kwargs = {
+ "base_path": hf_dataset_url(self.name, "", revision=self.commit_hash).rstrip("/"),
+ "repo_id": self.name,
+ "dataset_name": camelcase_to_snakecase(Path(self.name).name),
+ }
+ if self.data_dir:
+ builder_kwargs["data_files"] = data_files
+ download_config = self.download_config.copy()
+ if download_config.download_desc is None:
+ download_config.download_desc = "Downloading metadata"
+ try:
+ # this file is deprecated and was created automatically in old versions of push_to_hub
+ dataset_infos_path = cached_path(
+ hf_dataset_url(self.name, config.DATASETDICT_INFOS_FILENAME, revision=self.commit_hash),
+ download_config=download_config,
+ )
+ with open(dataset_infos_path, encoding="utf-8") as f:
+ legacy_dataset_infos = DatasetInfosDict(
+ {
+ config_name: DatasetInfo.from_dict(dataset_info_dict)
+ for config_name, dataset_info_dict in json.load(f).items()
+ }
+ )
+ if len(legacy_dataset_infos) == 1:
+ # old config e.g. named "username--dataset_name"
+ legacy_config_name = next(iter(legacy_dataset_infos))
+ legacy_dataset_infos["default"] = legacy_dataset_infos.pop(legacy_config_name)
+ legacy_dataset_infos.update(dataset_infos)
+ dataset_infos = legacy_dataset_infos
+ except FileNotFoundError:
+ pass
+ if default_config_name is None and len(dataset_infos) == 1:
+ default_config_name = next(iter(dataset_infos))
+
+ return DatasetModule(
+ module_path,
+ self.commit_hash,
+ builder_kwargs,
+ dataset_infos=dataset_infos,
+ builder_configs_parameters=BuilderConfigsParameters(
+ metadata_configs=metadata_configs,
+ builder_configs=builder_configs,
+ default_config_name=default_config_name,
+ ),
+ )
+
+
+class HubDatasetModuleFactoryWithParquetExport(_DatasetModuleFactory):
+ """
+ Get the module of a dataset loaded from parquet files of a dataset repository parquet export.
+ """
+
+ def __init__(
+ self,
+ name: str,
+ commit_hash: str,
+ download_config: Optional[DownloadConfig] = None,
+ ):
+ self.name = name
+ self.commit_hash = commit_hash
+ self.download_config = download_config or DownloadConfig()
+ increase_load_count(name)
+
+ def get_module(self) -> DatasetModule:
+ exported_parquet_files = _dataset_viewer.get_exported_parquet_files(
+ dataset=self.name, commit_hash=self.commit_hash, token=self.download_config.token
+ )
+ exported_dataset_infos = _dataset_viewer.get_exported_dataset_infos(
+ dataset=self.name, commit_hash=self.commit_hash, token=self.download_config.token
+ )
+ dataset_infos = DatasetInfosDict(
+ {
+ config_name: DatasetInfo.from_dict(exported_dataset_infos[config_name])
+ for config_name in exported_dataset_infos
+ }
+ )
+ parquet_commit_hash = (
+ HfApi(
+ endpoint=config.HF_ENDPOINT,
+ token=self.download_config.token,
+ library_name="datasets",
+ library_version=__version__,
+ user_agent=get_datasets_user_agent(self.download_config.user_agent),
+ )
+ .dataset_info(
+ self.name,
+ revision="refs/convert/parquet",
+ token=self.download_config.token,
+ timeout=100.0,
+ )
+ .sha
+ ) # fix the revision in case there are new commits in the meantime
+ metadata_configs = MetadataConfigs._from_exported_parquet_files_and_dataset_infos(
+ parquet_commit_hash=parquet_commit_hash,
+ exported_parquet_files=exported_parquet_files,
+ dataset_infos=dataset_infos,
+ )
+ module_path, _ = _PACKAGED_DATASETS_MODULES["parquet"]
+ builder_configs, default_config_name = create_builder_configs_from_metadata_configs(
+ module_path,
+ metadata_configs,
+ download_config=self.download_config,
+ )
+ builder_kwargs = {
+ "repo_id": self.name,
+ "dataset_name": camelcase_to_snakecase(Path(self.name).name),
+ }
+
+ return DatasetModule(
+ module_path,
+ self.commit_hash,
+ builder_kwargs,
+ dataset_infos=dataset_infos,
+ builder_configs_parameters=BuilderConfigsParameters(
+ metadata_configs=metadata_configs,
+ builder_configs=builder_configs,
+ default_config_name=default_config_name,
+ ),
+ )
+
+
+class CachedDatasetModuleFactory(_DatasetModuleFactory):
+ """
+ Get the module of a dataset that has been loaded once already and cached.
+ """
+
+ def __init__(
+ self,
+ name: str,
+ cache_dir: Optional[str] = None,
+ ):
+ self.name = name
+ self.cache_dir = cache_dir
+ assert self.name.count("/") <= 1
+
+ def get_module(self) -> DatasetModule:
+ cache_dir = os.path.expanduser(str(self.cache_dir or config.HF_DATASETS_CACHE))
+ namespace_and_dataset_name = self.name.split("/")
+ namespace_and_dataset_name[-1] = camelcase_to_snakecase(namespace_and_dataset_name[-1])
+ cached_relative_path = "___".join(namespace_and_dataset_name)
+ cached_datasets_directory_path_root = os.path.join(cache_dir, cached_relative_path)
+ cached_directory_paths = [
+ cached_directory_path
+ for cached_directory_path in glob.glob(os.path.join(cached_datasets_directory_path_root, "*", "*", "*"))
+ if os.path.isdir(cached_directory_path)
+ ]
+ if cached_directory_paths:
+ builder_kwargs = {
+ "repo_id": self.name,
+ "dataset_name": self.name.split("/")[-1],
+ }
+ warning_msg = f"Using the latest cached version of the dataset since {self.name} couldn't be found on the Hugging Face Hub"
+ if config.HF_HUB_OFFLINE:
+ warning_msg += " (offline mode is enabled)."
+ logger.warning(warning_msg)
+ return DatasetModule(
+ "datasets.packaged_modules.cache.cache",
+ "auto",
+ {**builder_kwargs, "version": "auto"},
+ )
+ raise FileNotFoundError(f"Dataset {self.name} is not cached in {self.cache_dir}")
+
+
+def dataset_module_factory(
+ path: str,
+ revision: Optional[Union[str, Version]] = None,
+ download_config: Optional[DownloadConfig] = None,
+ download_mode: Optional[Union[DownloadMode, str]] = None,
+ data_dir: Optional[str] = None,
+ data_files: Optional[Union[dict, list, str, DataFilesDict]] = None,
+ cache_dir: Optional[str] = None,
+ **download_kwargs,
+) -> DatasetModule:
+ """
+ Download/extract/cache a dataset module.
+
+ Dataset codes are cached inside the dynamic modules cache to allow easy import (avoid ugly sys.path tweaks).
+
+ Args:
+
+ path (str): Path or name of the dataset.
+ Depending on ``path``, the dataset builder that is used comes from one of the generic dataset builders (JSON, CSV, Parquet, text etc.).
+
+ For local datasets:
+
+ - if ``path`` is a local directory (containing data files only)
+ -> load a generic dataset builder (csv, json, text etc.) based on the content of the directory
+ e.g. ``'./path/to/directory/with/my/csv/data'``.
+
+ For datasets on the Hugging Face Hub (list all available datasets with ``huggingface_hub.list_datasets()``)
+
+ - if ``path`` is a dataset repository on the HF hub (containing data files only)
+ -> load a generic dataset builder (csv, text etc.) based on the content of the repository
+ e.g. ``'username/dataset_name'``, a dataset repository on the HF hub containing your data files.
+
+ revision (:class:`~utils.Version` or :obj:`str`, optional): Version of the dataset to load.
+ As datasets have their own git repository on the Datasets Hub, the default version "main" corresponds to their "main" branch.
+ You can specify a different version than the default "main" by using a commit SHA or a git tag of the dataset repository.
+ download_config (:class:`DownloadConfig`, optional): Specific download configuration parameters.
+ download_mode (:class:`DownloadMode` or :obj:`str`, default ``REUSE_DATASET_IF_EXISTS``): Download/generate mode.
+ data_dir (:obj:`str`, optional): Directory with the data files. Used only if `data_files` is not specified,
+ in which case it's equal to pass `os.path.join(data_dir, "**")` as `data_files`.
+ data_files (:obj:`Union[Dict, List, str]`, optional): Defining the data_files of the dataset configuration.
+ cache_dir (`str`, *optional*):
+ Directory to read/write data. Defaults to `"~/.cache/huggingface/datasets"`.
+
+
+
+ **download_kwargs (additional keyword arguments): optional attributes for DownloadConfig() which will override
+ the attributes in download_config if supplied.
+
+ Returns:
+ DatasetModule
+ """
+ if download_config is None:
+ download_config = DownloadConfig(**download_kwargs)
+ download_mode = DownloadMode(download_mode or DownloadMode.REUSE_DATASET_IF_EXISTS)
+ download_config.extract_compressed_file = True
+ download_config.force_extract = True
+ download_config.force_download = download_mode == DownloadMode.FORCE_REDOWNLOAD
+
+ filename = list(filter(lambda x: x, path.replace(os.sep, "/").split("/")))[-1]
+ if not filename.endswith(".py"):
+ filename = filename + ".py"
+ combined_path = os.path.join(path, filename)
+
+ # We have several ways to get a dataset builder:
+ #
+ # - if path is the name of a packaged dataset module
+ # -> use the packaged module (json, csv, etc.)
+ #
+ # - if os.path.join(path, name) is a local python file
+ # -> use the module from the python file
+ # - if path is a local directory (but no python file)
+ # -> use a packaged module (csv, text etc.) based on content of the directory
+ #
+ # - if path has one "/" and is dataset repository on the HF hub with a python file
+ # -> the module from the python file in the dataset repository
+ # - if path has one "/" and is dataset repository on the HF hub without a python file
+ # -> use a packaged module (csv, text etc.) based on content of the repository
+
+ # Try packaged
+ if path in _PACKAGED_DATASETS_MODULES:
+ return PackagedDatasetModuleFactory(
+ path,
+ data_dir=data_dir,
+ data_files=data_files,
+ download_config=download_config,
+ download_mode=download_mode,
+ ).get_module()
+ # Try locally
+ elif path.endswith(filename):
+ raise RuntimeError(f"Dataset scripts are no longer supported, but found {filename}")
+ elif os.path.isfile(combined_path):
+ raise RuntimeError(f"Dataset scripts are no longer supported, but found {filename}")
+ elif os.path.isdir(path):
+ return LocalDatasetModuleFactory(
+ path, data_dir=data_dir, data_files=data_files, download_mode=download_mode
+ ).get_module()
+ # Try remotely
+ elif is_relative_path(path) and path.count("/") <= 1:
+ try:
+ # Get the Dataset Card + get the revision + check authentication all at in one call
+ # We fix the commit_hash in case there are new commits in the meantime
+ api = HfApi(
+ endpoint=config.HF_ENDPOINT,
+ token=download_config.token,
+ library_name="datasets",
+ library_version=__version__,
+ user_agent=get_datasets_user_agent(download_config.user_agent),
+ )
+ try:
+ _raise_if_offline_mode_is_enabled()
+ dataset_readme_path = api.hf_hub_download(
+ repo_id=path,
+ filename=config.REPOCARD_FILENAME,
+ repo_type="dataset",
+ revision=revision,
+ proxies=download_config.proxies,
+ )
+ commit_hash = os.path.basename(os.path.dirname(dataset_readme_path))
+ except LocalEntryNotFoundError as e:
+ if isinstance(
+ e.__cause__,
+ (
+ OfflineModeIsEnabled,
+ requests.exceptions.Timeout,
+ requests.exceptions.ConnectionError,
+ httpx.ConnectError,
+ httpx.TimeoutException,
+ ),
+ ):
+ raise ConnectionError(f"Couldn't reach '{path}' on the Hub ({e.__class__.__name__})") from e
+ else:
+ raise
+ except EntryNotFoundError:
+ commit_hash = api.dataset_info(
+ path,
+ revision=revision,
+ timeout=100.0,
+ ).sha
+ except (
+ OfflineModeIsEnabled,
+ requests.exceptions.Timeout,
+ requests.exceptions.ConnectionError,
+ httpx.ConnectError,
+ httpx.TimeoutException,
+ ) as e:
+ raise ConnectionError(f"Couldn't reach '{path}' on the Hub ({e.__class__.__name__})") from e
+ except GatedRepoError as e:
+ message = f"Dataset '{path}' is a gated dataset on the Hub."
+ if e.response.status_code == 401:
+ message += " You must be authenticated to access it."
+ elif e.response.status_code == 403:
+ message += f" Visit the dataset page at https://huggingface.co/datasets/{path} to ask for access."
+ raise DatasetNotFoundError(message) from e
+ except RevisionNotFoundError as e:
+ raise DatasetNotFoundError(
+ f"Revision '{revision}' doesn't exist for dataset '{path}' on the Hub."
+ ) from e
+ except RepositoryNotFoundError as e:
+ raise DatasetNotFoundError(f"Dataset '{path}' doesn't exist on the Hub or cannot be accessed.") from e
+ try:
+ api.hf_hub_download(
+ repo_id=path,
+ filename=filename,
+ repo_type="dataset",
+ revision=commit_hash,
+ proxies=download_config.proxies,
+ )
+ raise RuntimeError(f"Dataset scripts are no longer supported, but found {filename}")
+ except EntryNotFoundError:
+ # Use the infos from the parquet export except in some cases:
+ if data_dir or data_files or (revision and revision != "main"):
+ use_exported_dataset_infos = False
+ else:
+ use_exported_dataset_infos = True
+ return HubDatasetModuleFactory(
+ path,
+ commit_hash=commit_hash,
+ data_dir=data_dir,
+ data_files=data_files,
+ download_config=download_config,
+ download_mode=download_mode,
+ use_exported_dataset_infos=use_exported_dataset_infos,
+ ).get_module()
+ except GatedRepoError as e:
+ message = f"Dataset '{path}' is a gated dataset on the Hub."
+ if e.response.status_code == 401:
+ message += " You must be authenticated to access it."
+ elif e.response.status_code == 403:
+ message += f" Visit the dataset page at https://huggingface.co/datasets/{path} to ask for access."
+ raise DatasetNotFoundError(message) from e
+ except RevisionNotFoundError as e:
+ raise DatasetNotFoundError(
+ f"Revision '{revision}' doesn't exist for dataset '{path}' on the Hub."
+ ) from e
+ except Exception as e1:
+ # All the attempts failed, before raising the error we should check if the module is already cached
+ try:
+ return CachedDatasetModuleFactory(path, cache_dir=cache_dir).get_module()
+ except Exception:
+ # If it's not in the cache, then it doesn't exist.
+ if isinstance(e1, OfflineModeIsEnabled):
+ raise ConnectionError(f"Couldn't reach the Hugging Face Hub for dataset '{path}': {e1}") from None
+ if isinstance(e1, (DataFilesNotFoundError, DatasetNotFoundError, EmptyDatasetError)):
+ raise e1 from None
+ if isinstance(e1, FileNotFoundError):
+ raise FileNotFoundError(
+ f"Couldn't find any data file at {relative_to_absolute_path(path)}. "
+ f"Couldn't find '{path}' on the Hugging Face Hub either: {type(e1).__name__}: {e1}"
+ ) from None
+ raise e1 from None
+ else:
+ raise FileNotFoundError(f"Couldn't find any data file at {relative_to_absolute_path(path)}.")
+
+
+def load_dataset_builder(
+ path: str,
+ name: Optional[str] = None,
+ data_dir: Optional[str] = None,
+ data_files: Optional[Union[str, Sequence[str], Mapping[str, Union[str, Sequence[str]]]]] = None,
+ cache_dir: Optional[str] = None,
+ features: Optional[Features] = None,
+ download_config: Optional[DownloadConfig] = None,
+ download_mode: Optional[Union[DownloadMode, str]] = None,
+ revision: Optional[Union[str, Version]] = None,
+ token: Optional[Union[bool, str]] = None,
+ storage_options: Optional[dict] = None,
+ **config_kwargs,
+) -> DatasetBuilder:
+ """Load a dataset builder which can be used to:
+
+ - Inspect general information that is required to build a dataset (cache directory, config, dataset info, features, data files, etc.)
+ - Download and prepare the dataset as Arrow files in the cache
+ - Get a streaming dataset without downloading or caching anything
+
+ You can find the list of datasets on the [Hub](https://huggingface.co/datasets) or with [`huggingface_hub.list_datasets`].
+
+ A dataset is a directory that contains some data files in generic formats (JSON, CSV, Parquet, etc.) and possibly
+ in a generic structure (Webdataset, ImageFolder, AudioFolder, VideoFolder, etc.)
+
+ Args:
+
+ path (`str`):
+ Path or name of the dataset.
+
+ - if `path` is a dataset repository on the HF hub (list all available datasets with [`huggingface_hub.list_datasets`])
+ -> load the dataset builder from supported files in the repository (csv, json, parquet, etc.)
+ e.g. `'username/dataset_name'`, a dataset repository on the HF hub containing the data files.
+
+ - if `path` is a local directory
+ -> load the dataset builder from supported files in the directory (csv, json, parquet, etc.)
+ e.g. `'./path/to/directory/with/my/csv/data'`.
+
+ - if `path` is the name of a dataset builder and `data_files` or `data_dir` is specified
+ (available builders are "json", "csv", "parquet", "arrow", "text", "xml", "webdataset", "imagefolder", "audiofolder", "videofolder")
+ -> load the dataset builder from the files in `data_files` or `data_dir`
+ e.g. `'parquet'`.
+
+ name (`str`, *optional*):
+ Defining the name of the dataset configuration.
+ data_dir (`str`, *optional*):
+ Defining the `data_dir` of the dataset configuration. If specified for the generic builders (csv, text etc.) or the Hub datasets and `data_files` is `None`,
+ the behavior is equal to passing `os.path.join(data_dir, **)` as `data_files` to reference all the files in a directory.
+ data_files (`str` or `Sequence` or `Mapping`, *optional*):
+ Path(s) to source data file(s).
+ cache_dir (`str`, *optional*):
+ Directory to read/write data. Defaults to `"~/.cache/huggingface/datasets"`.
+ features ([`Features`], *optional*):
+ Set the features type to use for this dataset.
+ download_config ([`DownloadConfig`], *optional*):
+ Specific download configuration parameters.
+ download_mode ([`DownloadMode`] or `str`, defaults to `REUSE_DATASET_IF_EXISTS`):
+ Download/generate mode.
+ revision ([`Version`] or `str`, *optional*):
+ Version of the dataset to load.
+ As datasets have their own git repository on the Datasets Hub, the default version "main" corresponds to their "main" branch.
+ You can specify a different version than the default "main" by using a commit SHA or a git tag of the dataset repository.
+ token (`str` or `bool`, *optional*):
+ Optional string or boolean to use as Bearer token for remote files on the Datasets Hub.
+ If `True`, or not specified, will get token from `"~/.huggingface"`.
+ storage_options (`dict`, *optional*, defaults to `None`):
+ **Experimental**. Key/value pairs to be passed on to the dataset file-system backend, if any.
+
+
+
+ **config_kwargs (additional keyword arguments):
+ Keyword arguments to be passed to the [`BuilderConfig`]
+ and used in the [`DatasetBuilder`].
+
+ Returns:
+ [`DatasetBuilder`]
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset_builder
+ >>> ds_builder = load_dataset_builder('cornell-movie-review-data/rotten_tomatoes')
+ >>> ds_builder.info.features
+ {'label': ClassLabel(names=['neg', 'pos']),
+ 'text': Value('string')}
+ ```
+ """
+ download_mode = DownloadMode(download_mode or DownloadMode.REUSE_DATASET_IF_EXISTS)
+ if token is not None:
+ download_config = download_config.copy() if download_config else DownloadConfig()
+ download_config.token = token
+ if storage_options is not None:
+ download_config = download_config.copy() if download_config else DownloadConfig()
+ download_config.storage_options.update(storage_options)
+ if features is not None:
+ features = _fix_for_backward_compatible_features(features)
+ dataset_module = dataset_module_factory(
+ path,
+ revision=revision,
+ download_config=download_config,
+ download_mode=download_mode,
+ data_dir=data_dir,
+ data_files=data_files,
+ cache_dir=cache_dir,
+ )
+ # Get dataset builder class
+ builder_kwargs = dataset_module.builder_kwargs
+ data_dir = builder_kwargs.pop("data_dir", data_dir)
+ data_files = builder_kwargs.pop("data_files", data_files)
+ config_name = builder_kwargs.pop(
+ "config_name", name or dataset_module.builder_configs_parameters.default_config_name
+ )
+ dataset_name = builder_kwargs.pop("dataset_name", None)
+ info = dataset_module.dataset_infos.get(config_name) if dataset_module.dataset_infos else None
+
+ if (
+ path in _PACKAGED_DATASETS_MODULES
+ and data_files is None
+ and dataset_module.builder_configs_parameters.builder_configs[0].data_files is None
+ ):
+ error_msg = f"Please specify the data files or data directory to load for the {path} dataset builder."
+ example_extensions = [
+ extension for extension in _EXTENSION_TO_MODULE if _EXTENSION_TO_MODULE[extension] == path
+ ]
+ if example_extensions:
+ error_msg += f'\nFor example `data_files={{"train": "path/to/data/train/*.{example_extensions[0]}"}}`'
+ raise ValueError(error_msg)
+
+ builder_cls = get_dataset_builder_class(dataset_module, dataset_name=dataset_name)
+ # Instantiate the dataset builder
+ builder_instance: DatasetBuilder = builder_cls(
+ cache_dir=cache_dir,
+ dataset_name=dataset_name,
+ config_name=config_name,
+ data_dir=data_dir,
+ data_files=data_files,
+ hash=dataset_module.hash,
+ info=info,
+ features=features,
+ token=token,
+ storage_options=storage_options,
+ **builder_kwargs,
+ **config_kwargs,
+ )
+ builder_instance._use_legacy_cache_dir_if_possible(dataset_module)
+
+ return builder_instance
+
+
+def load_dataset(
+ path: str,
+ name: Optional[str] = None,
+ data_dir: Optional[str] = None,
+ data_files: Optional[Union[str, Sequence[str], Mapping[str, Union[str, Sequence[str]]]]] = None,
+ split: Optional[Union[str, Split, list[str], list[Split]]] = None,
+ cache_dir: Optional[str] = None,
+ features: Optional[Features] = None,
+ download_config: Optional[DownloadConfig] = None,
+ download_mode: Optional[Union[DownloadMode, str]] = None,
+ verification_mode: Optional[Union[VerificationMode, str]] = None,
+ keep_in_memory: Optional[bool] = None,
+ save_infos: bool = False,
+ revision: Optional[Union[str, Version]] = None,
+ token: Optional[Union[bool, str]] = None,
+ streaming: bool = False,
+ num_proc: Optional[int] = None,
+ storage_options: Optional[dict] = None,
+ **config_kwargs,
+) -> Union[DatasetDict, Dataset, IterableDatasetDict, IterableDataset]:
+ """Load a dataset from the Hugging Face Hub, or a local dataset.
+
+ You can find the list of datasets on the [Hub](https://huggingface.co/datasets) or with [`huggingface_hub.list_datasets`].
+
+ A dataset is a directory that contains some data files in generic formats (JSON, CSV, Parquet, etc.) and possibly
+ in a generic structure (Webdataset, ImageFolder, AudioFolder, VideoFolder, etc.)
+
+ This function does the following under the hood:
+
+ 1. Load a dataset builder:
+
+ * Find the most common data format in the dataset and pick its associated builder (JSON, CSV, Parquet, Webdataset, ImageFolder, AudioFolder, etc.)
+ * Find which file goes into which split (e.g. train/test) based on file and directory names or on the YAML configuration
+ * It is also possible to specify `data_files` manually, and which dataset builder to use (e.g. "parquet").
+
+ 2. Run the dataset builder:
+
+ In the general case:
+
+ * Download the data files from the dataset if they are not already available locally or cached.
+ * Process and cache the dataset in typed Arrow tables for caching.
+
+ Arrow table are arbitrarily long, typed tables which can store nested objects and be mapped to numpy/pandas/python generic types.
+ They can be directly accessed from disk, loaded in RAM or even streamed over the web.
+
+ In the streaming case:
+
+ * Don't download or cache anything. Instead, the dataset is lazily loaded and will be streamed on-the-fly when iterating on it.
+
+ 3. Return a dataset built from the requested splits in `split` (default: all).
+
+ Args:
+
+ path (`str`):
+ Path or name of the dataset.
+
+ - if `path` is a dataset repository on the HF hub (list all available datasets with [`huggingface_hub.list_datasets`])
+ -> load the dataset from supported files in the repository (csv, json, parquet, etc.)
+ e.g. `'username/dataset_name'`, a dataset repository on the HF hub containing the data files.
+
+ - if `path` is a local directory
+ -> load the dataset from supported files in the directory (csv, json, parquet, etc.)
+ e.g. `'./path/to/directory/with/my/csv/data'`.
+
+ - if `path` is the name of a dataset builder and `data_files` or `data_dir` is specified
+ (available builders are "json", "csv", "parquet", "arrow", "text", "xml", "webdataset", "imagefolder", "audiofolder", "videofolder")
+ -> load the dataset from the files in `data_files` or `data_dir`
+ e.g. `'parquet'`.
+
+ name (`str`, *optional*):
+ Defining the name of the dataset configuration.
+ data_dir (`str`, *optional*):
+ Defining the `data_dir` of the dataset configuration. If specified for the generic builders (csv, text etc.) or the Hub datasets and `data_files` is `None`,
+ the behavior is equal to passing `os.path.join(data_dir, **)` as `data_files` to reference all the files in a directory.
+ data_files (`str` or `Sequence` or `Mapping`, *optional*):
+ Path(s) to source data file(s).
+ split (`Split` or `str`):
+ Which split of the data to load.
+ If `None`, will return a `dict` with all splits (typically `datasets.Split.TRAIN` and `datasets.Split.TEST`).
+ If given, will return a single Dataset.
+ Splits can be combined and specified like in tensorflow-datasets.
+ cache_dir (`str`, *optional*):
+ Directory to read/write data. Defaults to `"~/.cache/huggingface/datasets"`.
+ features (`Features`, *optional*):
+ Set the features type to use for this dataset.
+ download_config ([`DownloadConfig`], *optional*):
+ Specific download configuration parameters.
+ download_mode ([`DownloadMode`] or `str`, defaults to `REUSE_DATASET_IF_EXISTS`):
+ Download/generate mode.
+ verification_mode ([`VerificationMode`] or `str`, defaults to `BASIC_CHECKS`):
+ Verification mode determining the checks to run on the downloaded/processed dataset information (checksums/size/splits/...).
+
+
+ keep_in_memory (`bool`, defaults to `None`):
+ Whether to copy the dataset in-memory. If `None`, the dataset
+ will not be copied in-memory unless explicitly enabled by setting `datasets.config.IN_MEMORY_MAX_SIZE` to
+ nonzero. See more details in the [improve performance](../cache#improve-performance) section.
+ revision ([`Version`] or `str`, *optional*):
+ Version of the dataset to load.
+ As datasets have their own git repository on the Datasets Hub, the default version "main" corresponds to their "main" branch.
+ You can specify a different version than the default "main" by using a commit SHA or a git tag of the dataset repository.
+ token (`str` or `bool`, *optional*):
+ Optional string or boolean to use as Bearer token for remote files on the Datasets Hub.
+ If `True`, or not specified, will get token from `"~/.huggingface"`.
+ streaming (`bool`, defaults to `False`):
+ If set to `True`, don't download the data files. Instead, it streams the data progressively while
+ iterating on the dataset. An [`IterableDataset`] or [`IterableDatasetDict`] is returned instead in this case.
+
+ Note that streaming works for datasets that use data formats that support being iterated over like txt, csv, jsonl for example.
+ Json files may be downloaded completely. Also streaming from remote zip or gzip files is supported but other compressed formats
+ like rar and xz are not yet supported. The tgz format doesn't allow streaming.
+ num_proc (`int`, *optional*, defaults to `None`):
+ Number of processes when downloading and generating the dataset locally.
+ Multiprocessing is disabled by default.
+
+
+ storage_options (`dict`, *optional*, defaults to `None`):
+ **Experimental**. Key/value pairs to be passed on to the dataset file-system backend, if any.
+
+
+ **config_kwargs (additional keyword arguments):
+ Keyword arguments to be passed to the `BuilderConfig`
+ and used in the [`DatasetBuilder`].
+
+ Returns:
+ [`Dataset`] or [`DatasetDict`]:
+ - if `split` is not `None`: the dataset requested,
+ - if `split` is `None`, a [`~datasets.DatasetDict`] with each split.
+
+ or [`IterableDataset`] or [`IterableDatasetDict`]: if `streaming=True`
+
+ - if `split` is not `None`, the dataset is requested
+ - if `split` is `None`, a [`~datasets.streaming.IterableDatasetDict`] with each split.
+
+ Example:
+
+ Load a dataset from the Hugging Face Hub:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset('cornell-movie-review-data/rotten_tomatoes', split='train')
+
+ # Load a subset or dataset configuration (here 'sst2')
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset('nyu-mll/glue', 'sst2', split='train')
+
+ # Manual mapping of data files to splits
+ >>> data_files = {'train': 'train.csv', 'test': 'test.csv'}
+ >>> ds = load_dataset('namespace/your_dataset_name', data_files=data_files)
+
+ # Manual selection of a directory to load
+ >>> ds = load_dataset('namespace/your_dataset_name', data_dir='folder_name')
+ ```
+
+ Load a local dataset:
+
+ ```py
+ # Load a CSV file
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset('csv', data_files='path/to/local/my_dataset.csv')
+
+ # Load a JSON file
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset('json', data_files='path/to/local/my_dataset.json')
+ ```
+
+ Load an [`~datasets.IterableDataset`]:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset('cornell-movie-review-data/rotten_tomatoes', split='train', streaming=True)
+ ```
+
+ Load an image dataset with the `ImageFolder` dataset builder:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset('imagefolder', data_dir='/path/to/images', split='train')
+ ```
+ """
+ if "trust_remote_code" in config_kwargs:
+ if config_kwargs.pop("trust_remote_code"):
+ logger.error(
+ "`trust_remote_code` is not supported anymore.\n"
+ f"Please check that the Hugging Face dataset '{path}' isn't based on a loading script and remove `trust_remote_code`.\n"
+ "If the dataset is based on a loading script, please ask the dataset author to remove it and convert it to a standard format like Parquet."
+ )
+ if data_files is not None and not data_files:
+ raise ValueError(f"Empty 'data_files': '{data_files}'. It should be either non-empty or None (default).")
+ if Path(path, config.DATASET_STATE_JSON_FILENAME).exists():
+ raise ValueError(
+ "You are trying to load a dataset that was saved using `save_to_disk`. "
+ "Please use `load_from_disk` instead."
+ )
+
+ if streaming and num_proc is not None:
+ raise NotImplementedError(
+ "Loading a streaming dataset in parallel with `num_proc` is not implemented. "
+ "To parallelize streaming, you can wrap the dataset with a PyTorch DataLoader using `num_workers` > 1 instead."
+ )
+
+ download_mode = DownloadMode(download_mode or DownloadMode.REUSE_DATASET_IF_EXISTS)
+ verification_mode = VerificationMode(
+ (verification_mode or VerificationMode.BASIC_CHECKS) if not save_infos else VerificationMode.ALL_CHECKS
+ )
+
+ # Create a dataset builder
+ builder_instance = load_dataset_builder(
+ path=path,
+ name=name,
+ data_dir=data_dir,
+ data_files=data_files,
+ cache_dir=cache_dir,
+ features=features,
+ download_config=download_config,
+ download_mode=download_mode,
+ revision=revision,
+ token=token,
+ storage_options=storage_options,
+ **config_kwargs,
+ )
+
+ # Return iterable dataset in case of streaming
+ if streaming:
+ return builder_instance.as_streaming_dataset(split=split)
+
+ # Download and prepare data
+ builder_instance.download_and_prepare(
+ download_config=download_config,
+ download_mode=download_mode,
+ verification_mode=verification_mode,
+ num_proc=num_proc,
+ storage_options=storage_options,
+ )
+
+ # Build dataset for splits
+ keep_in_memory = (
+ keep_in_memory if keep_in_memory is not None else is_small_dataset(builder_instance.info.dataset_size)
+ )
+ ds = builder_instance.as_dataset(split=split, verification_mode=verification_mode, in_memory=keep_in_memory)
+
+ return ds
+
+
+def load_from_disk(
+ dataset_path: PathLike, keep_in_memory: Optional[bool] = None, storage_options: Optional[dict] = None
+) -> Union[Dataset, DatasetDict]:
+ """
+ Loads a dataset that was previously saved using [`~Dataset.save_to_disk`] from a dataset directory, or
+ from a filesystem using any implementation of `fsspec.spec.AbstractFileSystem`.
+
+ Args:
+ dataset_path (`path-like`):
+ Path (e.g. `"dataset/train"`) or remote URI (e.g. `"s3://my-bucket/dataset/train"`)
+ of the [`Dataset`] or [`DatasetDict`] directory where the dataset/dataset-dict will be
+ loaded from.
+ keep_in_memory (`bool`, defaults to `None`):
+ Whether to copy the dataset in-memory. If `None`, the dataset
+ will not be copied in-memory unless explicitly enabled by setting `datasets.config.IN_MEMORY_MAX_SIZE` to
+ nonzero. See more details in the [improve performance](../cache#improve-performance) section.
+
+ storage_options (`dict`, *optional*):
+ Key/value pairs to be passed on to the file-system backend, if any.
+
+
+
+ Returns:
+ [`Dataset`] or [`DatasetDict`]:
+ - If `dataset_path` is a path of a dataset directory: the dataset requested.
+ - If `dataset_path` is a path of a dataset dict directory, a [`DatasetDict`] with each split.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_from_disk
+ >>> ds = load_from_disk('path/to/dataset/directory')
+ ```
+ """
+ fs: fsspec.AbstractFileSystem
+ fs, *_ = url_to_fs(dataset_path, **(storage_options or {}))
+ if not fs.exists(dataset_path):
+ raise FileNotFoundError(f"Directory {dataset_path} not found")
+ if fs.isfile(posixpath.join(dataset_path, config.DATASET_INFO_FILENAME)) and fs.isfile(
+ posixpath.join(dataset_path, config.DATASET_STATE_JSON_FILENAME)
+ ):
+ return Dataset.load_from_disk(dataset_path, keep_in_memory=keep_in_memory, storage_options=storage_options)
+ elif fs.isfile(posixpath.join(dataset_path, config.DATASETDICT_JSON_FILENAME)):
+ return DatasetDict.load_from_disk(dataset_path, keep_in_memory=keep_in_memory, storage_options=storage_options)
+ else:
+ raise FileNotFoundError(
+ f"Directory {dataset_path} is neither a `Dataset` directory nor a `DatasetDict` directory."
+ )
diff --git a/venv/lib/python3.10/site-packages/datasets/naming.py b/venv/lib/python3.10/site-packages/datasets/naming.py
new file mode 100644
index 0000000000000000000000000000000000000000..65e7ede10dcde8701823223ae98e7971f705f945
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/naming.py
@@ -0,0 +1,84 @@
+# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""Utilities for file names."""
+
+import itertools
+import os
+import re
+
+
+_uppercase_uppercase_re = re.compile(r"([A-Z]+)([A-Z][a-z])")
+_lowercase_uppercase_re = re.compile(r"([a-z\d])([A-Z])")
+
+_single_underscore_re = re.compile(r"(?:/\|?*"
+
+
+def camelcase_to_snakecase(name):
+ """Convert camel-case string to snake-case."""
+ name = _uppercase_uppercase_re.sub(r"\1_\2", name)
+ name = _lowercase_uppercase_re.sub(r"\1_\2", name)
+ return name.lower()
+
+
+def snakecase_to_camelcase(name):
+ """Convert snake-case string to camel-case string."""
+ name = _single_underscore_re.split(name)
+ name = [_multiple_underscores_re.split(n) for n in name]
+ return "".join(n.capitalize() for n in itertools.chain.from_iterable(name) if n != "")
+
+
+def filename_prefix_for_name(name):
+ if os.path.basename(name) != name:
+ raise ValueError(f"Should be a dataset name, not a path: {name}")
+ return camelcase_to_snakecase(name)
+
+
+def filename_prefix_for_split(name, split):
+ if os.path.basename(name) != name:
+ raise ValueError(f"Should be a dataset name, not a path: {name}")
+ if not re.match(_split_re, split):
+ raise ValueError(f"Split name should match '{_split_re}'' but got '{split}'.")
+ return f"{filename_prefix_for_name(name)}-{split}"
+
+
+def filepattern_for_dataset_split(dataset_name, split, data_dir, filetype_suffix=None):
+ prefix = filename_prefix_for_split(dataset_name, split)
+ if filetype_suffix:
+ prefix += f".{filetype_suffix}"
+ filepath = os.path.join(data_dir, prefix)
+ return f"{filepath}*"
+
+
+def filenames_for_dataset_split(path, dataset_name, split, filetype_suffix=None, shard_lengths=None):
+ prefix = filename_prefix_for_split(dataset_name, split)
+ prefix = os.path.join(path, prefix)
+
+ if shard_lengths:
+ num_shards = len(shard_lengths)
+ filenames = [f"{prefix}-{shard_id:05d}-of-{num_shards:05d}" for shard_id in range(num_shards)]
+ if filetype_suffix:
+ filenames = [filename + f".{filetype_suffix}" for filename in filenames]
+ return filenames
+ else:
+ filename = prefix
+ if filetype_suffix:
+ filename += f".{filetype_suffix}"
+ return [filename]
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/__init__.py b/venv/lib/python3.10/site-packages/datasets/packaged_modules/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..9d076df44b77fab27566e1884c7ac89b6382057c
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/packaged_modules/__init__.py
@@ -0,0 +1,113 @@
+import inspect
+import re
+from typing import Dict, List, Tuple
+
+from huggingface_hub.utils import insecure_hashlib
+
+from .arrow import arrow
+from .audiofolder import audiofolder
+from .cache import cache
+from .csv import csv
+from .hdf5 import hdf5
+from .imagefolder import imagefolder
+from .json import json
+from .niftifolder import niftifolder
+from .pandas import pandas
+from .parquet import parquet
+from .pdffolder import pdffolder
+from .sql import sql
+from .text import text
+from .videofolder import videofolder
+from .webdataset import webdataset
+from .xml import xml
+
+
+def _hash_python_lines(lines: list[str]) -> str:
+ filtered_lines = []
+ for line in lines:
+ line = re.sub(r"#.*", "", line) # remove comments
+ if line:
+ filtered_lines.append(line)
+ full_str = "\n".join(filtered_lines)
+
+ # Make a hash from all this code
+ full_bytes = full_str.encode("utf-8")
+ return insecure_hashlib.sha256(full_bytes).hexdigest()
+
+
+# get importable module names and hash for caching
+_PACKAGED_DATASETS_MODULES = {
+ "csv": (csv.__name__, _hash_python_lines(inspect.getsource(csv).splitlines())),
+ "json": (json.__name__, _hash_python_lines(inspect.getsource(json).splitlines())),
+ "pandas": (pandas.__name__, _hash_python_lines(inspect.getsource(pandas).splitlines())),
+ "parquet": (parquet.__name__, _hash_python_lines(inspect.getsource(parquet).splitlines())),
+ "arrow": (arrow.__name__, _hash_python_lines(inspect.getsource(arrow).splitlines())),
+ "text": (text.__name__, _hash_python_lines(inspect.getsource(text).splitlines())),
+ "imagefolder": (imagefolder.__name__, _hash_python_lines(inspect.getsource(imagefolder).splitlines())),
+ "audiofolder": (audiofolder.__name__, _hash_python_lines(inspect.getsource(audiofolder).splitlines())),
+ "videofolder": (videofolder.__name__, _hash_python_lines(inspect.getsource(videofolder).splitlines())),
+ "pdffolder": (pdffolder.__name__, _hash_python_lines(inspect.getsource(pdffolder).splitlines())),
+ "niftifolder": (niftifolder.__name__, _hash_python_lines(inspect.getsource(niftifolder).splitlines())),
+ "webdataset": (webdataset.__name__, _hash_python_lines(inspect.getsource(webdataset).splitlines())),
+ "xml": (xml.__name__, _hash_python_lines(inspect.getsource(xml).splitlines())),
+ "hdf5": (hdf5.__name__, _hash_python_lines(inspect.getsource(hdf5).splitlines())),
+}
+
+# get importable module names and hash for caching
+_PACKAGED_DATASETS_MODULES_2_15_HASHES = {
+ "csv": "eea64c71ca8b46dd3f537ed218fc9bf495d5707789152eb2764f5c78fa66d59d",
+ "json": "8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96",
+ "pandas": "3ac4ffc4563c796122ef66899b9485a3f1a977553e2d2a8a318c72b8cc6f2202",
+ "parquet": "ca31c69184d9832faed373922c2acccec0b13a0bb5bbbe19371385c3ff26f1d1",
+ "arrow": "74f69db2c14c2860059d39860b1f400a03d11bf7fb5a8258ca38c501c878c137",
+ "text": "c4a140d10f020282918b5dd1b8a49f0104729c6177f60a6b49ec2a365ec69f34",
+ "imagefolder": "7b7ce5247a942be131d49ad4f3de5866083399a0f250901bd8dc202f8c5f7ce5",
+ "audiofolder": "d3c1655c66c8f72e4efb5c79e952975fa6e2ce538473a6890241ddbddee9071c",
+}
+
+# Used to infer the module to use based on the data files extensions
+_EXTENSION_TO_MODULE: dict[str, tuple[str, dict]] = {
+ ".csv": ("csv", {}),
+ ".tsv": ("csv", {"sep": "\t"}),
+ ".json": ("json", {}),
+ ".jsonl": ("json", {}),
+ # ndjson is no longer maintained (see: https://github.com/ndjson/ndjson-spec/issues/35#issuecomment-1285673417)
+ ".ndjson": ("json", {}),
+ ".parquet": ("parquet", {}),
+ ".geoparquet": ("parquet", {}),
+ ".gpq": ("parquet", {}),
+ ".arrow": ("arrow", {}),
+ ".txt": ("text", {}),
+ ".tar": ("webdataset", {}),
+ ".xml": ("xml", {}),
+ ".hdf5": ("hdf5", {}),
+ ".h5": ("hdf5", {}),
+}
+_EXTENSION_TO_MODULE.update({ext: ("imagefolder", {}) for ext in imagefolder.ImageFolder.EXTENSIONS})
+_EXTENSION_TO_MODULE.update({ext.upper(): ("imagefolder", {}) for ext in imagefolder.ImageFolder.EXTENSIONS})
+_EXTENSION_TO_MODULE.update({ext: ("audiofolder", {}) for ext in audiofolder.AudioFolder.EXTENSIONS})
+_EXTENSION_TO_MODULE.update({ext.upper(): ("audiofolder", {}) for ext in audiofolder.AudioFolder.EXTENSIONS})
+_EXTENSION_TO_MODULE.update({ext: ("videofolder", {}) for ext in videofolder.VideoFolder.EXTENSIONS})
+_EXTENSION_TO_MODULE.update({ext.upper(): ("videofolder", {}) for ext in videofolder.VideoFolder.EXTENSIONS})
+_EXTENSION_TO_MODULE.update({ext: ("pdffolder", {}) for ext in pdffolder.PdfFolder.EXTENSIONS})
+_EXTENSION_TO_MODULE.update({ext.upper(): ("pdffolder", {}) for ext in pdffolder.PdfFolder.EXTENSIONS})
+_EXTENSION_TO_MODULE.update({ext: ("niftifolder", {}) for ext in niftifolder.NiftiFolder.EXTENSIONS})
+_EXTENSION_TO_MODULE.update({ext.upper(): ("niftifolder", {}) for ext in niftifolder.NiftiFolder.EXTENSIONS})
+
+# Used to filter data files based on extensions given a module name
+_MODULE_TO_EXTENSIONS: dict[str, list[str]] = {}
+for _ext, (_module, _) in _EXTENSION_TO_MODULE.items():
+ _MODULE_TO_EXTENSIONS.setdefault(_module, []).append(_ext)
+
+for _module in _MODULE_TO_EXTENSIONS:
+ _MODULE_TO_EXTENSIONS[_module].append(".zip")
+
+# Used to filter data files based on file names
+_MODULE_TO_METADATA_FILE_NAMES: Dict[str, List[str]] = {}
+for _module in _MODULE_TO_EXTENSIONS:
+ _MODULE_TO_METADATA_FILE_NAMES[_module] = []
+_MODULE_TO_METADATA_FILE_NAMES["imagefolder"] = imagefolder.ImageFolder.METADATA_FILENAMES
+_MODULE_TO_METADATA_FILE_NAMES["audiofolder"] = imagefolder.ImageFolder.METADATA_FILENAMES
+_MODULE_TO_METADATA_FILE_NAMES["videofolder"] = imagefolder.ImageFolder.METADATA_FILENAMES
+_MODULE_TO_METADATA_FILE_NAMES["pdffolder"] = imagefolder.ImageFolder.METADATA_FILENAMES
+_MODULE_TO_METADATA_FILE_NAMES["niftifolder"] = imagefolder.ImageFolder.METADATA_FILENAMES
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/__pycache__/__init__.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/packaged_modules/__pycache__/__init__.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..ea6637bfe2b57b8503b959db1d5b8e609f05eabe
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/packaged_modules/__pycache__/__init__.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/arrow/__init__.py b/venv/lib/python3.10/site-packages/datasets/packaged_modules/arrow/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/arrow/__pycache__/__init__.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/packaged_modules/arrow/__pycache__/__init__.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..e3db1b03e1aa7896f7dee01dbc0b94201e68584a
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/packaged_modules/arrow/__pycache__/__init__.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/arrow/__pycache__/arrow.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/packaged_modules/arrow/__pycache__/arrow.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..98a43a0e1f8405b0db67ad802ee1fd02ef415239
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/packaged_modules/arrow/__pycache__/arrow.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/arrow/arrow.py b/venv/lib/python3.10/site-packages/datasets/packaged_modules/arrow/arrow.py
new file mode 100644
index 0000000000000000000000000000000000000000..bcf31c473d2ae532da908600c440a440f8a2d3d6
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/packaged_modules/arrow/arrow.py
@@ -0,0 +1,79 @@
+import itertools
+from dataclasses import dataclass
+from typing import Optional
+
+import pyarrow as pa
+
+import datasets
+from datasets.table import table_cast
+
+
+logger = datasets.utils.logging.get_logger(__name__)
+
+
+@dataclass
+class ArrowConfig(datasets.BuilderConfig):
+ """BuilderConfig for Arrow."""
+
+ features: Optional[datasets.Features] = None
+
+ def __post_init__(self):
+ super().__post_init__()
+
+
+class Arrow(datasets.ArrowBasedBuilder):
+ BUILDER_CONFIG_CLASS = ArrowConfig
+
+ def _info(self):
+ return datasets.DatasetInfo(features=self.config.features)
+
+ def _split_generators(self, dl_manager):
+ """We handle string, list and dicts in datafiles"""
+ if not self.config.data_files:
+ raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}")
+ dl_manager.download_config.extract_on_the_fly = True
+ data_files = dl_manager.download_and_extract(self.config.data_files)
+ splits = []
+ for split_name, files in data_files.items():
+ if isinstance(files, str):
+ files = [files]
+ # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
+ files = [dl_manager.iter_files(file) for file in files]
+ # Infer features if they are stored in the arrow schema
+ if self.info.features is None:
+ for file in itertools.chain.from_iterable(files):
+ with open(file, "rb") as f:
+ try:
+ reader = pa.ipc.open_stream(f)
+ except (OSError, pa.lib.ArrowInvalid):
+ reader = pa.ipc.open_file(f)
+ self.info.features = datasets.Features.from_arrow_schema(reader.schema)
+ break
+ splits.append(datasets.SplitGenerator(name=split_name, gen_kwargs={"files": files}))
+ return splits
+
+ def _cast_table(self, pa_table: pa.Table) -> pa.Table:
+ if self.info.features is not None:
+ # more expensive cast to support nested features with keys in a different order
+ # allows str <-> int/float or str to Audio for example
+ pa_table = table_cast(pa_table, self.info.features.arrow_schema)
+ return pa_table
+
+ def _generate_tables(self, files):
+ for file_idx, file in enumerate(itertools.chain.from_iterable(files)):
+ with open(file, "rb") as f:
+ try:
+ try:
+ batches = pa.ipc.open_stream(f)
+ except (OSError, pa.lib.ArrowInvalid):
+ reader = pa.ipc.open_file(f)
+ batches = (reader.get_batch(i) for i in range(reader.num_record_batches))
+ for batch_idx, record_batch in enumerate(batches):
+ pa_table = pa.Table.from_batches([record_batch])
+ # Uncomment for debugging (will print the Arrow table size and elements)
+ # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
+ # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
+ yield f"{file_idx}_{batch_idx}", self._cast_table(pa_table)
+ except ValueError as e:
+ logger.error(f"Failed to read file '{file}' with error {type(e)}: {e}")
+ raise
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/audiofolder/__init__.py b/venv/lib/python3.10/site-packages/datasets/packaged_modules/audiofolder/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/audiofolder/__pycache__/__init__.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/packaged_modules/audiofolder/__pycache__/__init__.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..4e8d639e0eb7e082e9910c20d497a3224e662e79
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/packaged_modules/audiofolder/__pycache__/__init__.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/audiofolder/__pycache__/audiofolder.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/packaged_modules/audiofolder/__pycache__/audiofolder.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..a7dbde9cfce52aa2bbff23689cabb10aa5624556
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/packaged_modules/audiofolder/__pycache__/audiofolder.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/audiofolder/audiofolder.py b/venv/lib/python3.10/site-packages/datasets/packaged_modules/audiofolder/audiofolder.py
new file mode 100644
index 0000000000000000000000000000000000000000..9e71a6bbbae8d138c8bf637846cdbebc1a5dc8f8
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/packaged_modules/audiofolder/audiofolder.py
@@ -0,0 +1,86 @@
+import datasets
+
+from ..folder_based_builder import folder_based_builder
+
+
+logger = datasets.utils.logging.get_logger(__name__)
+
+
+class AudioFolderConfig(folder_based_builder.FolderBasedBuilderConfig):
+ """Builder Config for AudioFolder."""
+
+ drop_labels: bool = None
+ drop_metadata: bool = None
+
+ def __post_init__(self):
+ super().__post_init__()
+
+
+class AudioFolder(folder_based_builder.FolderBasedBuilder):
+ BASE_FEATURE = datasets.Audio
+ BASE_COLUMN_NAME = "audio"
+ BUILDER_CONFIG_CLASS = AudioFolderConfig
+ EXTENSIONS: list[str] # definition at the bottom of the script
+
+
+# Obtained with:
+# ```
+# import soundfile as sf
+#
+# AUDIO_EXTENSIONS = [f".{format.lower()}" for format in sf.available_formats().keys()]
+#
+# # .opus decoding is supported if libsndfile >= 1.0.31:
+# AUDIO_EXTENSIONS.extend([".opus"])
+# ```
+# We intentionally did not run this code on launch because:
+# (1) Soundfile was an optional dependency, so importing it in global namespace is not allowed
+# (2) To ensure the list of supported extensions is deterministic
+# (3) We use TorchCodec now anyways instead of Soundfile
+AUDIO_EXTENSIONS = [
+ ".aiff",
+ ".au",
+ ".avr",
+ ".caf",
+ ".flac",
+ ".htk",
+ ".svx",
+ ".mat4",
+ ".mat5",
+ ".mpc2k",
+ ".ogg",
+ ".paf",
+ ".pvf",
+ ".raw",
+ ".rf64",
+ ".sd2",
+ ".sds",
+ ".ircam",
+ ".voc",
+ ".w64",
+ ".wav",
+ ".nist",
+ ".wavex",
+ ".wve",
+ ".xi",
+ ".mp3",
+ ".opus",
+ ".3gp",
+ ".3g2",
+ ".avi",
+ ".asf",
+ ".flv",
+ ".mp4",
+ ".mov",
+ ".m4v",
+ ".mkv",
+ ".mpg",
+ ".webm",
+ ".f4v",
+ ".wmv",
+ ".wma",
+ ".ogg",
+ ".ogm",
+ ".mxf",
+ ".nut",
+]
+AudioFolder.EXTENSIONS = AUDIO_EXTENSIONS
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/cache/__init__.py b/venv/lib/python3.10/site-packages/datasets/packaged_modules/cache/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/cache/__pycache__/__init__.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/packaged_modules/cache/__pycache__/__init__.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..bed4f93fea475f9051032ee99f920f5449241bc0
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/packaged_modules/cache/__pycache__/__init__.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/cache/__pycache__/cache.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/packaged_modules/cache/__pycache__/cache.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..fb78e94e64ca9096e7f4a05fe725be1e45b1102a
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/packaged_modules/cache/__pycache__/cache.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/cache/cache.py b/venv/lib/python3.10/site-packages/datasets/packaged_modules/cache/cache.py
new file mode 100644
index 0000000000000000000000000000000000000000..cdcfb4c20b6b4912e3b2e98b57cc140fe36c357a
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/packaged_modules/cache/cache.py
@@ -0,0 +1,191 @@
+import glob
+import json
+import os
+import shutil
+import time
+from pathlib import Path
+from typing import Optional, Union
+
+import pyarrow as pa
+
+import datasets
+import datasets.config
+import datasets.data_files
+from datasets.naming import camelcase_to_snakecase, filenames_for_dataset_split
+
+
+logger = datasets.utils.logging.get_logger(__name__)
+
+
+def _get_modification_time(cached_directory_path):
+ return (Path(cached_directory_path)).stat().st_mtime
+
+
+def _find_hash_in_cache(
+ dataset_name: str,
+ config_name: Optional[str],
+ cache_dir: Optional[str],
+ config_kwargs: dict,
+ custom_features: Optional[datasets.Features],
+) -> tuple[str, str, str]:
+ if config_name or config_kwargs or custom_features:
+ config_id = datasets.BuilderConfig(config_name or "default").create_config_id(
+ config_kwargs=config_kwargs, custom_features=custom_features
+ )
+ else:
+ config_id = None
+ cache_dir = os.path.expanduser(str(cache_dir or datasets.config.HF_DATASETS_CACHE))
+ namespace_and_dataset_name = dataset_name.split("/")
+ namespace_and_dataset_name[-1] = camelcase_to_snakecase(namespace_and_dataset_name[-1])
+ cached_relative_path = "___".join(namespace_and_dataset_name)
+ cached_datasets_directory_path_root = os.path.join(cache_dir, cached_relative_path)
+ cached_directory_paths = [
+ cached_directory_path
+ for cached_directory_path in glob.glob(
+ os.path.join(cached_datasets_directory_path_root, config_id or "*", "*", "*")
+ )
+ if os.path.isdir(cached_directory_path)
+ and (
+ config_kwargs
+ or custom_features
+ or json.loads(Path(cached_directory_path, "dataset_info.json").read_text(encoding="utf-8"))["config_name"]
+ == Path(cached_directory_path).parts[-3] # no extra params => config_id == config_name
+ )
+ ]
+ if not cached_directory_paths:
+ cached_directory_paths = [
+ cached_directory_path
+ for cached_directory_path in glob.glob(os.path.join(cached_datasets_directory_path_root, "*", "*", "*"))
+ if os.path.isdir(cached_directory_path)
+ ]
+ available_configs = sorted(
+ {Path(cached_directory_path).parts[-3] for cached_directory_path in cached_directory_paths}
+ )
+ raise ValueError(
+ f"Couldn't find cache for {dataset_name}"
+ + (f" for config '{config_id}'" if config_id else "")
+ + (f"\nAvailable configs in the cache: {available_configs}" if available_configs else "")
+ )
+ # get most recent
+ cached_directory_path = Path(sorted(cached_directory_paths, key=_get_modification_time)[-1])
+ version, hash = cached_directory_path.parts[-2:]
+ other_configs = [
+ Path(_cached_directory_path).parts[-3]
+ for _cached_directory_path in glob.glob(os.path.join(cached_datasets_directory_path_root, "*", version, hash))
+ if os.path.isdir(_cached_directory_path)
+ and (
+ config_kwargs
+ or custom_features
+ or json.loads(Path(_cached_directory_path, "dataset_info.json").read_text(encoding="utf-8"))["config_name"]
+ == Path(_cached_directory_path).parts[-3] # no extra params => config_id == config_name
+ )
+ ]
+ if not config_id and len(other_configs) > 1:
+ raise ValueError(
+ f"There are multiple '{dataset_name}' configurations in the cache: {', '.join(other_configs)}"
+ f"\nPlease specify which configuration to reload from the cache, e.g."
+ f"\n\tload_dataset('{dataset_name}', '{other_configs[0]}')"
+ )
+ config_name = cached_directory_path.parts[-3]
+ warning_msg = (
+ f"Found the latest cached dataset configuration '{config_name}' at {cached_directory_path} "
+ f"(last modified on {time.ctime(_get_modification_time(cached_directory_path))})."
+ )
+ logger.warning(warning_msg)
+ return config_name, version, hash
+
+
+class Cache(datasets.ArrowBasedBuilder):
+ def __init__(
+ self,
+ cache_dir: Optional[str] = None,
+ dataset_name: Optional[str] = None,
+ config_name: Optional[str] = None,
+ version: Optional[str] = "0.0.0",
+ hash: Optional[str] = None,
+ base_path: Optional[str] = None,
+ info: Optional[datasets.DatasetInfo] = None,
+ features: Optional[datasets.Features] = None,
+ token: Optional[Union[bool, str]] = None,
+ repo_id: Optional[str] = None,
+ data_files: Optional[Union[str, list, dict, datasets.data_files.DataFilesDict]] = None,
+ data_dir: Optional[str] = None,
+ storage_options: Optional[dict] = None,
+ writer_batch_size: Optional[int] = None,
+ **config_kwargs,
+ ):
+ if repo_id is None and dataset_name is None:
+ raise ValueError("repo_id or dataset_name is required for the Cache dataset builder")
+ if data_files is not None:
+ config_kwargs["data_files"] = data_files
+ if data_dir is not None:
+ config_kwargs["data_dir"] = data_dir
+ if hash == "auto" and version == "auto":
+ config_name, version, hash = _find_hash_in_cache(
+ dataset_name=repo_id or dataset_name,
+ config_name=config_name,
+ cache_dir=cache_dir,
+ config_kwargs=config_kwargs,
+ custom_features=features,
+ )
+ elif hash == "auto" or version == "auto":
+ raise NotImplementedError("Pass both hash='auto' and version='auto' instead")
+ super().__init__(
+ cache_dir=cache_dir,
+ dataset_name=dataset_name,
+ config_name=config_name,
+ version=version,
+ hash=hash,
+ base_path=base_path,
+ info=info,
+ token=token,
+ repo_id=repo_id,
+ storage_options=storage_options,
+ writer_batch_size=writer_batch_size,
+ )
+
+ def _info(self) -> datasets.DatasetInfo:
+ return datasets.DatasetInfo()
+
+ def download_and_prepare(self, output_dir: Optional[str] = None, *args, **kwargs):
+ if not os.path.exists(self.cache_dir):
+ raise ValueError(f"Cache directory for {self.dataset_name} doesn't exist at {self.cache_dir}")
+ if output_dir is not None and output_dir != self.cache_dir:
+ shutil.copytree(self.cache_dir, output_dir)
+
+ def _split_generators(self, dl_manager):
+ # used to stream from cache
+ if isinstance(self.info.splits, datasets.SplitDict):
+ split_infos: list[datasets.SplitInfo] = list(self.info.splits.values())
+ else:
+ raise ValueError(f"Missing splits info for {self.dataset_name} in cache directory {self.cache_dir}")
+ return [
+ datasets.SplitGenerator(
+ name=split_info.name,
+ gen_kwargs={
+ "files": filenames_for_dataset_split(
+ self.cache_dir,
+ dataset_name=self.dataset_name,
+ split=split_info.name,
+ filetype_suffix="arrow",
+ shard_lengths=split_info.shard_lengths,
+ )
+ },
+ )
+ for split_info in split_infos
+ ]
+
+ def _generate_tables(self, files):
+ # used to stream from cache
+ for file_idx, file in enumerate(files):
+ with open(file, "rb") as f:
+ try:
+ for batch_idx, record_batch in enumerate(pa.ipc.open_stream(f)):
+ pa_table = pa.Table.from_batches([record_batch])
+ # Uncomment for debugging (will print the Arrow table size and elements)
+ # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
+ # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
+ yield f"{file_idx}_{batch_idx}", pa_table
+ except ValueError as e:
+ logger.error(f"Failed to read file '{file}' with error {type(e)}: {e}")
+ raise
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/csv/__init__.py b/venv/lib/python3.10/site-packages/datasets/packaged_modules/csv/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/csv/__pycache__/__init__.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/packaged_modules/csv/__pycache__/__init__.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..b39f85c34b7b353b2685290ca3beb2e07b04d394
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/packaged_modules/csv/__pycache__/__init__.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/csv/__pycache__/csv.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/packaged_modules/csv/__pycache__/csv.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..fe0bd55c0e0599eb54c5d8afc6d6c3f61e435afb
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/packaged_modules/csv/__pycache__/csv.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/csv/csv.py b/venv/lib/python3.10/site-packages/datasets/packaged_modules/csv/csv.py
new file mode 100644
index 0000000000000000000000000000000000000000..2ae95ff51428b826b84b32a7eaf2825cd7593960
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/packaged_modules/csv/csv.py
@@ -0,0 +1,198 @@
+import itertools
+from dataclasses import dataclass
+from typing import Any, Callable, Optional, Union
+
+import pandas as pd
+import pyarrow as pa
+
+import datasets
+import datasets.config
+from datasets.features.features import require_storage_cast
+from datasets.table import table_cast
+from datasets.utils.py_utils import Literal
+
+
+logger = datasets.utils.logging.get_logger(__name__)
+
+_PANDAS_READ_CSV_NO_DEFAULT_PARAMETERS = ["names", "prefix"]
+_PANDAS_READ_CSV_DEPRECATED_PARAMETERS = ["warn_bad_lines", "error_bad_lines", "mangle_dupe_cols"]
+_PANDAS_READ_CSV_NEW_1_3_0_PARAMETERS = ["encoding_errors", "on_bad_lines"]
+_PANDAS_READ_CSV_NEW_2_0_0_PARAMETERS = ["date_format"]
+_PANDAS_READ_CSV_DEPRECATED_2_2_0_PARAMETERS = ["verbose"]
+
+
+@dataclass
+class CsvConfig(datasets.BuilderConfig):
+ """BuilderConfig for CSV."""
+
+ sep: str = ","
+ delimiter: Optional[str] = None
+ header: Optional[Union[int, list[int], str]] = "infer"
+ names: Optional[list[str]] = None
+ column_names: Optional[list[str]] = None
+ index_col: Optional[Union[int, str, list[int], list[str]]] = None
+ usecols: Optional[Union[list[int], list[str]]] = None
+ prefix: Optional[str] = None
+ mangle_dupe_cols: bool = True
+ engine: Optional[Literal["c", "python", "pyarrow"]] = None
+ converters: dict[Union[int, str], Callable[[Any], Any]] = None
+ true_values: Optional[list] = None
+ false_values: Optional[list] = None
+ skipinitialspace: bool = False
+ skiprows: Optional[Union[int, list[int]]] = None
+ nrows: Optional[int] = None
+ na_values: Optional[Union[str, list[str]]] = None
+ keep_default_na: bool = True
+ na_filter: bool = True
+ verbose: bool = False
+ skip_blank_lines: bool = True
+ thousands: Optional[str] = None
+ decimal: str = "."
+ lineterminator: Optional[str] = None
+ quotechar: str = '"'
+ quoting: int = 0
+ escapechar: Optional[str] = None
+ comment: Optional[str] = None
+ encoding: Optional[str] = None
+ dialect: Optional[str] = None
+ error_bad_lines: bool = True
+ warn_bad_lines: bool = True
+ skipfooter: int = 0
+ doublequote: bool = True
+ memory_map: bool = False
+ float_precision: Optional[str] = None
+ chunksize: int = 10_000
+ features: Optional[datasets.Features] = None
+ encoding_errors: Optional[str] = "strict"
+ on_bad_lines: Literal["error", "warn", "skip"] = "error"
+ date_format: Optional[str] = None
+
+ def __post_init__(self):
+ super().__post_init__()
+ if self.delimiter is not None:
+ self.sep = self.delimiter
+ if self.column_names is not None:
+ self.names = self.column_names
+
+ @property
+ def pd_read_csv_kwargs(self):
+ pd_read_csv_kwargs = {
+ "sep": self.sep,
+ "header": self.header,
+ "names": self.names,
+ "index_col": self.index_col,
+ "usecols": self.usecols,
+ "prefix": self.prefix,
+ "mangle_dupe_cols": self.mangle_dupe_cols,
+ "engine": self.engine,
+ "converters": self.converters,
+ "true_values": self.true_values,
+ "false_values": self.false_values,
+ "skipinitialspace": self.skipinitialspace,
+ "skiprows": self.skiprows,
+ "nrows": self.nrows,
+ "na_values": self.na_values,
+ "keep_default_na": self.keep_default_na,
+ "na_filter": self.na_filter,
+ "verbose": self.verbose,
+ "skip_blank_lines": self.skip_blank_lines,
+ "thousands": self.thousands,
+ "decimal": self.decimal,
+ "lineterminator": self.lineterminator,
+ "quotechar": self.quotechar,
+ "quoting": self.quoting,
+ "escapechar": self.escapechar,
+ "comment": self.comment,
+ "encoding": self.encoding,
+ "dialect": self.dialect,
+ "error_bad_lines": self.error_bad_lines,
+ "warn_bad_lines": self.warn_bad_lines,
+ "skipfooter": self.skipfooter,
+ "doublequote": self.doublequote,
+ "memory_map": self.memory_map,
+ "float_precision": self.float_precision,
+ "chunksize": self.chunksize,
+ "encoding_errors": self.encoding_errors,
+ "on_bad_lines": self.on_bad_lines,
+ "date_format": self.date_format,
+ }
+
+ # some kwargs must not be passed if they don't have a default value
+ # some others are deprecated and we can also not pass them if they are the default value
+ for pd_read_csv_parameter in _PANDAS_READ_CSV_NO_DEFAULT_PARAMETERS + _PANDAS_READ_CSV_DEPRECATED_PARAMETERS:
+ if pd_read_csv_kwargs[pd_read_csv_parameter] == getattr(CsvConfig(), pd_read_csv_parameter):
+ del pd_read_csv_kwargs[pd_read_csv_parameter]
+
+ # Remove 1.3 new arguments
+ if not (datasets.config.PANDAS_VERSION.major >= 1 and datasets.config.PANDAS_VERSION.minor >= 3):
+ for pd_read_csv_parameter in _PANDAS_READ_CSV_NEW_1_3_0_PARAMETERS:
+ del pd_read_csv_kwargs[pd_read_csv_parameter]
+
+ # Remove 2.0 new arguments
+ if not (datasets.config.PANDAS_VERSION.major >= 2):
+ for pd_read_csv_parameter in _PANDAS_READ_CSV_NEW_2_0_0_PARAMETERS:
+ del pd_read_csv_kwargs[pd_read_csv_parameter]
+
+ # Remove 2.2 deprecated arguments
+ if datasets.config.PANDAS_VERSION.release >= (2, 2):
+ for pd_read_csv_parameter in _PANDAS_READ_CSV_DEPRECATED_2_2_0_PARAMETERS:
+ if pd_read_csv_kwargs[pd_read_csv_parameter] == getattr(CsvConfig(), pd_read_csv_parameter):
+ del pd_read_csv_kwargs[pd_read_csv_parameter]
+
+ return pd_read_csv_kwargs
+
+
+class Csv(datasets.ArrowBasedBuilder):
+ BUILDER_CONFIG_CLASS = CsvConfig
+
+ def _info(self):
+ return datasets.DatasetInfo(features=self.config.features)
+
+ def _split_generators(self, dl_manager):
+ """We handle string, list and dicts in datafiles"""
+ if not self.config.data_files:
+ raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}")
+ dl_manager.download_config.extract_on_the_fly = True
+ data_files = dl_manager.download_and_extract(self.config.data_files)
+ splits = []
+ for split_name, files in data_files.items():
+ if isinstance(files, str):
+ files = [files]
+ files = [dl_manager.iter_files(file) for file in files]
+ splits.append(datasets.SplitGenerator(name=split_name, gen_kwargs={"files": files}))
+ return splits
+
+ def _cast_table(self, pa_table: pa.Table) -> pa.Table:
+ if self.config.features is not None:
+ schema = self.config.features.arrow_schema
+ if all(not require_storage_cast(feature) for feature in self.config.features.values()):
+ # cheaper cast
+ pa_table = pa.Table.from_arrays([pa_table[field.name] for field in schema], schema=schema)
+ else:
+ # more expensive cast; allows str <-> int/float or str to Audio for example
+ pa_table = table_cast(pa_table, schema)
+ return pa_table
+
+ def _generate_tables(self, files):
+ schema = self.config.features.arrow_schema if self.config.features else None
+ # dtype allows reading an int column as str
+ dtype = (
+ {
+ name: dtype.to_pandas_dtype() if not require_storage_cast(feature) else object
+ for name, dtype, feature in zip(schema.names, schema.types, self.config.features.values())
+ }
+ if schema is not None
+ else None
+ )
+ for file_idx, file in enumerate(itertools.chain.from_iterable(files)):
+ csv_file_reader = pd.read_csv(file, iterator=True, dtype=dtype, **self.config.pd_read_csv_kwargs)
+ try:
+ for batch_idx, df in enumerate(csv_file_reader):
+ pa_table = pa.Table.from_pandas(df)
+ # Uncomment for debugging (will print the Arrow table size and elements)
+ # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
+ # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
+ yield (file_idx, batch_idx), self._cast_table(pa_table)
+ except ValueError as e:
+ logger.error(f"Failed to read file '{file}' with error {type(e)}: {e}")
+ raise
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/folder_based_builder/__init__.py b/venv/lib/python3.10/site-packages/datasets/packaged_modules/folder_based_builder/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/folder_based_builder/__pycache__/__init__.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/packaged_modules/folder_based_builder/__pycache__/__init__.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..3bee3d1c5dff48193225d9dc54fc4763379c2b42
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/packaged_modules/folder_based_builder/__pycache__/__init__.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/folder_based_builder/__pycache__/folder_based_builder.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/packaged_modules/folder_based_builder/__pycache__/folder_based_builder.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..3425f0f86b8cb4d3fa4a7a5747a90893f0015403
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/packaged_modules/folder_based_builder/__pycache__/folder_based_builder.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/folder_based_builder/folder_based_builder.py b/venv/lib/python3.10/site-packages/datasets/packaged_modules/folder_based_builder/folder_based_builder.py
new file mode 100644
index 0000000000000000000000000000000000000000..182de467b14d38403a782829b4e36f75958f4182
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/packaged_modules/folder_based_builder/folder_based_builder.py
@@ -0,0 +1,426 @@
+import collections
+import io
+import itertools
+import os
+from dataclasses import dataclass
+from typing import Any, Callable, Iterator, Optional, Union
+
+import pandas as pd
+import pyarrow as pa
+import pyarrow.dataset as ds
+import pyarrow.json as paj
+import pyarrow.parquet as pq
+
+import datasets
+from datasets import config
+from datasets.features.features import FeatureType, _visit, _visit_with_path, _VisitPath, require_storage_cast
+from datasets.utils.file_utils import readline
+
+
+logger = datasets.utils.logging.get_logger(__name__)
+
+
+def count_path_segments(path):
+ return path.replace("\\", "/").count("/")
+
+
+@dataclass
+class FolderBasedBuilderConfig(datasets.BuilderConfig):
+ """BuilderConfig for AutoFolder."""
+
+ features: Optional[datasets.Features] = None
+ drop_labels: bool = None
+ drop_metadata: bool = None
+ metadata_filenames: list[str] = None
+ filters: Optional[Union[ds.Expression, list[tuple], list[list[tuple]]]] = None
+
+ def __post_init__(self):
+ super().__post_init__()
+
+
+class FolderBasedBuilder(datasets.GeneratorBasedBuilder):
+ """
+ Base class for generic data loaders for vision and image data.
+
+
+ Abstract class attributes to be overridden by a child class:
+ BASE_FEATURE: feature object to decode data (i.e. datasets.Image, datasets.Audio, ...)
+ BASE_COLUMN_NAME: string key name of a base feature (i.e. "image", "audio", ...)
+ BUILDER_CONFIG_CLASS: builder config inherited from `folder_based_builder.FolderBasedBuilderConfig`
+ EXTENSIONS: list of allowed extensions (only files with these extensions and METADATA_FILENAME files
+ will be included in a dataset)
+ """
+
+ BASE_FEATURE: type[FeatureType]
+ BASE_COLUMN_NAME: str
+ BUILDER_CONFIG_CLASS: FolderBasedBuilderConfig
+ EXTENSIONS: list[str]
+
+ METADATA_FILENAMES: list[str] = ["metadata.csv", "metadata.jsonl", "metadata.parquet"]
+
+ def _info(self):
+ if not self.config.data_dir and not self.config.data_files:
+ raise ValueError(
+ "Folder-based datasets require either `data_dir` or `data_files` to be specified. "
+ "Neither was provided."
+ )
+
+ return datasets.DatasetInfo(features=self.config.features)
+
+ def _split_generators(self, dl_manager):
+ if not self.config.data_files:
+ raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}")
+ dl_manager.download_config.extract_on_the_fly = True
+ # Do an early pass if:
+ # * `drop_labels` is None (default) or False, to infer the class labels
+ # * `drop_metadata` is None (default) or False, to find the metadata files
+ do_analyze = not self.config.drop_labels or not self.config.drop_metadata
+ labels, path_depths = set(), set()
+ metadata_files = collections.defaultdict(set)
+ metadata_filenames = self.config.metadata_filenames or self.METADATA_FILENAMES
+
+ def analyze(files_or_archives, downloaded_files_or_dirs, split):
+ if len(downloaded_files_or_dirs) == 0:
+ return
+ # The files are separated from the archives at this point, so check the first sample
+ # to see if it's a file or a directory and iterate accordingly
+ if os.path.isfile(downloaded_files_or_dirs[0]):
+ original_files, downloaded_files = files_or_archives, downloaded_files_or_dirs
+ for original_file, downloaded_file in zip(original_files, downloaded_files):
+ original_file, downloaded_file = str(original_file), str(downloaded_file)
+ _, original_file_ext = os.path.splitext(original_file)
+ if original_file_ext.lower() in self.EXTENSIONS:
+ if not self.config.drop_labels:
+ labels.add(os.path.basename(os.path.dirname(original_file)))
+ path_depths.add(count_path_segments(original_file))
+ elif os.path.basename(original_file) in metadata_filenames:
+ metadata_files[split].add((original_file, downloaded_file))
+ else:
+ original_file_name = os.path.basename(original_file)
+ logger.debug(
+ f"The file '{original_file_name}' was ignored: it is not a {self.BASE_COLUMN_NAME}, and is not {metadata_filenames} either."
+ )
+ else:
+ archives, downloaded_dirs = files_or_archives, downloaded_files_or_dirs
+ for archive, downloaded_dir in zip(archives, downloaded_dirs):
+ archive, downloaded_dir = str(archive), str(downloaded_dir)
+ for downloaded_dir_file in dl_manager.iter_files(downloaded_dir):
+ _, downloaded_dir_file_ext = os.path.splitext(downloaded_dir_file)
+ if downloaded_dir_file_ext in self.EXTENSIONS:
+ if not self.config.drop_labels:
+ labels.add(os.path.basename(os.path.dirname(downloaded_dir_file)))
+ path_depths.add(count_path_segments(downloaded_dir_file))
+ elif os.path.basename(downloaded_dir_file) in metadata_filenames:
+ metadata_files[split].add((None, downloaded_dir_file))
+ else:
+ archive_file_name = os.path.basename(archive)
+ original_file_name = os.path.basename(downloaded_dir_file)
+ logger.debug(
+ f"The file '{original_file_name}' from the archive '{archive_file_name}' was ignored: it is not a {self.BASE_COLUMN_NAME}, and is not {metadata_filenames} either."
+ )
+
+ data_files = self.config.data_files
+ splits = []
+ for split_name, files in data_files.items():
+ if isinstance(files, str):
+ files = [files]
+ files, archives = self._split_files_and_archives(files)
+ downloaded_files = dl_manager.download(files)
+ downloaded_dirs = dl_manager.download_and_extract(archives)
+ if do_analyze: # drop_metadata is None or False, drop_labels is None or False
+ logger.info(f"Searching for labels and/or metadata files in {split_name} data files...")
+ analyze(files, downloaded_files, split_name)
+ analyze(archives, downloaded_dirs, split_name)
+
+ if metadata_files:
+ # add metadata if `metadata_files` are found and `drop_metadata` is None (default) or False
+ add_metadata = not self.config.drop_metadata
+ # if `metadata_files` are found, don't add labels
+ add_labels = False
+ else:
+ # if `metadata_files` are not found, don't add metadata
+ add_metadata = False
+ # if `metadata_files` are not found and `drop_labels` is None (default) -
+ # add labels if files are on the same level in directory hierarchy and there is more than one label
+ add_labels = (
+ (len(labels) > 1 and len(path_depths) == 1)
+ if self.config.drop_labels is None
+ else not self.config.drop_labels
+ )
+
+ if add_labels:
+ logger.info("Adding the labels inferred from data directories to the dataset's features...")
+ if add_metadata:
+ logger.info("Adding metadata to the dataset...")
+ else:
+ add_labels, add_metadata, metadata_files = False, False, {}
+
+ splits.append(
+ datasets.SplitGenerator(
+ name=split_name,
+ gen_kwargs={
+ "files": tuple(zip(files, downloaded_files))
+ + tuple((None, dl_manager.iter_files(downloaded_dir)) for downloaded_dir in downloaded_dirs),
+ "metadata_files": metadata_files.get(split_name, []),
+ "add_labels": add_labels,
+ "add_metadata": add_metadata,
+ },
+ )
+ )
+
+ if add_metadata:
+ # Verify that:
+ # * all metadata files have the same set of features in each split
+ # * the `file_name` key is one of the metadata keys and is of type string
+ features_per_metadata_file: list[tuple[str, datasets.Features]] = []
+
+ # Check that all metadata files share the same format
+ metadata_ext = {
+ os.path.splitext(original_metadata_file or downloaded_metadata_file)[-1]
+ for original_metadata_file, downloaded_metadata_file in itertools.chain.from_iterable(
+ metadata_files.values()
+ )
+ }
+ if len(metadata_ext) > 1:
+ raise ValueError(f"Found metadata files with different extensions: {list(metadata_ext)}")
+ metadata_ext = metadata_ext.pop()
+
+ for split_metadata_files in metadata_files.values():
+ pa_metadata_table = None
+ for _, downloaded_metadata_file in split_metadata_files:
+ for pa_metadata_table in self._read_metadata(downloaded_metadata_file, metadata_ext=metadata_ext):
+ break # just fetch the first rows
+ if pa_metadata_table is not None:
+ features_per_metadata_file.append(
+ (downloaded_metadata_file, datasets.Features.from_arrow_schema(pa_metadata_table.schema))
+ )
+ break # no need to fetch all the files
+ for downloaded_metadata_file, metadata_features in features_per_metadata_file:
+ if metadata_features != features_per_metadata_file[0][1]:
+ raise ValueError(
+ f"Metadata files {downloaded_metadata_file} and {features_per_metadata_file[0][0]} have different features: {features_per_metadata_file[0]} != {metadata_features}"
+ )
+ metadata_features = features_per_metadata_file[0][1]
+ feature_not_found = True
+
+ def _set_feature(feature):
+ nonlocal feature_not_found
+ if isinstance(feature, dict):
+ out = type(feature)()
+ for key in feature:
+ if (key == "file_name" or key.endswith("_file_name")) and feature[key] == datasets.Value(
+ "string"
+ ):
+ key = key[: -len("_file_name")] or self.BASE_COLUMN_NAME
+ out[key] = self.BASE_FEATURE()
+ feature_not_found = False
+ elif (key == "file_names" or key.endswith("_file_names")) and feature[key] == datasets.List(
+ datasets.Value("string")
+ ):
+ key = key[: -len("_file_names")] or (self.BASE_COLUMN_NAME + "s")
+ out[key] = datasets.List(self.BASE_FEATURE())
+ feature_not_found = False
+ elif (key == "file_names" or key.endswith("_file_names")) and feature[key] == [
+ datasets.Value("string")
+ ]:
+ key = key[: -len("_file_names")] or (self.BASE_COLUMN_NAME + "s")
+ out[key] = [self.BASE_FEATURE()]
+ feature_not_found = False
+ else:
+ out[key] = feature[key]
+ return out
+ return feature
+
+ metadata_features = _visit(metadata_features, _set_feature)
+
+ if feature_not_found:
+ raise ValueError(
+ "`file_name` or `*_file_name` must be present as dictionary key (with type string) in metadata files"
+ )
+ else:
+ metadata_features = None
+
+ # Normally, we would do this in _info, but we need to know the labels and/or metadata
+ # before building the features
+ if self.config.features is None:
+ if add_metadata:
+ self.info.features = metadata_features
+ elif add_labels:
+ self.info.features = datasets.Features(
+ {
+ self.BASE_COLUMN_NAME: self.BASE_FEATURE(),
+ "label": datasets.ClassLabel(names=sorted(labels)),
+ }
+ )
+ else:
+ self.info.features = datasets.Features({self.BASE_COLUMN_NAME: self.BASE_FEATURE()})
+
+ return splits
+
+ def _split_files_and_archives(self, data_files):
+ files, archives = [], []
+ metadata_filenames = self.config.metadata_filenames or self.METADATA_FILENAMES
+ for data_file in data_files:
+ _, data_file_ext = os.path.splitext(data_file)
+ if data_file_ext.lower() in self.EXTENSIONS:
+ files.append(data_file)
+ elif os.path.basename(data_file) in metadata_filenames:
+ files.append(data_file)
+ else:
+ archives.append(data_file)
+ return files, archives
+
+ def _read_metadata(self, metadata_file: str, metadata_ext: str = "") -> Iterator[pa.Table]:
+ """using the same logic as the Csv, Json and Parquet dataset builders to stream the data"""
+ if self.config.filters is not None:
+ filter_expr = (
+ pq.filters_to_expression(self.config.filters)
+ if isinstance(self.config.filters, list)
+ else self.config.filters
+ )
+ else:
+ filter_expr = None
+ if metadata_ext == ".csv":
+ chunksize = 10_000 # 10k lines
+ schema = self.config.features.arrow_schema if self.config.features else None
+ # dtype allows reading an int column as str
+ dtype = (
+ {
+ name: dtype.to_pandas_dtype() if not require_storage_cast(feature) else object
+ for name, dtype, feature in zip(schema.names, schema.types, self.config.features.values())
+ }
+ if schema is not None
+ else None
+ )
+ csv_file_reader = pd.read_csv(metadata_file, iterator=True, dtype=dtype, chunksize=chunksize)
+ for df in csv_file_reader:
+ pa_table = pa.Table.from_pandas(df)
+ if self.config.filters is not None:
+ pa_table = pa_table.filter(filter_expr)
+ if len(pa_table) > 0:
+ yield pa_table
+ elif metadata_ext == ".jsonl":
+ with open(metadata_file, "rb") as f:
+ chunksize: int = 10 << 20 # 10MB
+ # Use block_size equal to the chunk size divided by 32 to leverage multithreading
+ # Set a default minimum value of 16kB if the chunk size is really small
+ block_size = max(chunksize // 32, 16 << 10)
+ while True:
+ batch = f.read(chunksize)
+ if not batch:
+ break
+ # Finish current line
+ try:
+ batch += f.readline()
+ except (AttributeError, io.UnsupportedOperation):
+ batch += readline(f)
+ while True:
+ try:
+ pa_table = paj.read_json(
+ io.BytesIO(batch), read_options=paj.ReadOptions(block_size=block_size)
+ )
+ break
+ except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
+ if (
+ isinstance(e, pa.ArrowInvalid)
+ and "straddling" not in str(e)
+ or block_size > len(batch)
+ ):
+ raise
+ else:
+ # Increase the block size in case it was too small.
+ # The block size will be reset for the next file.
+ logger.debug(
+ f"Batch of {len(batch)} bytes couldn't be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}."
+ )
+ block_size *= 2
+ if self.config.filters is not None:
+ pa_table = pa_table.filter(filter_expr)
+ if len(pa_table) > 0:
+ yield pa_table
+ else:
+ with open(metadata_file, "rb") as f:
+ parquet_fragment = ds.ParquetFileFormat().make_fragment(f)
+ if parquet_fragment.row_groups:
+ batch_size = parquet_fragment.row_groups[0].num_rows
+ else:
+ batch_size = config.DEFAULT_MAX_BATCH_SIZE
+ for record_batch in parquet_fragment.to_batches(
+ batch_size=batch_size,
+ filter=filter_expr,
+ batch_readahead=0,
+ fragment_readahead=0,
+ ):
+ yield pa.Table.from_batches([record_batch])
+
+ def _generate_examples(self, files, metadata_files, add_metadata, add_labels):
+ sample_idx = 0
+ if add_metadata:
+ feature_paths = []
+
+ def find_feature_path(feature, feature_path):
+ nonlocal feature_paths
+ if feature_path and isinstance(feature, self.BASE_FEATURE):
+ feature_paths.append(feature_path)
+
+ _visit_with_path(self.info.features, find_feature_path)
+
+ for original_metadata_file, downloaded_metadata_file in metadata_files:
+ metadata_ext = os.path.splitext(original_metadata_file or downloaded_metadata_file)[-1]
+ downloaded_metadata_dir = os.path.dirname(downloaded_metadata_file)
+
+ def set_feature(item, feature_path: _VisitPath):
+ if len(feature_path) == 2 and isinstance(feature_path[0], str) and feature_path[1] == 0:
+ item[feature_path[0]] = item.pop("file_names", None) or item.pop(
+ feature_path[0] + "_file_names", None
+ )
+ elif len(feature_path) == 1 and isinstance(feature_path[0], str):
+ item[feature_path[0]] = item.pop("file_name", None) or item.pop(
+ feature_path[0] + "_file_name", None
+ )
+ elif len(feature_path) == 0:
+ file_relpath = os.path.normpath(item).replace("\\", "/")
+ item = os.path.join(downloaded_metadata_dir, file_relpath)
+ return item
+
+ for pa_metadata_table in self._read_metadata(downloaded_metadata_file, metadata_ext=metadata_ext):
+ for sample in pa_metadata_table.to_pylist():
+ for feature_path in feature_paths:
+ _nested_apply(sample, feature_path, set_feature)
+ yield sample_idx, sample
+ sample_idx += 1
+ else:
+ if self.config.filters is not None:
+ filter_expr = (
+ pq.filters_to_expression(self.config.filters)
+ if isinstance(self.config.filters, list)
+ else self.config.filters
+ )
+ for original_file, downloaded_file_or_dir in files:
+ downloaded_files = [downloaded_file_or_dir] if original_file else downloaded_file_or_dir
+ for downloaded_file in downloaded_files:
+ original_file_ext = os.path.splitext(original_file or downloaded_file)[-1]
+ if original_file_ext.lower() not in self.EXTENSIONS:
+ continue
+ sample = {self.BASE_COLUMN_NAME: downloaded_file}
+ if add_labels:
+ sample["label"] = os.path.basename(os.path.dirname(original_file or downloaded_file))
+ if self.config.filters is not None:
+ pa_table = pa.Table.from_pylist([sample]).filter(filter_expr)
+ if len(pa_table) == 0:
+ continue
+ yield sample_idx, sample
+ sample_idx += 1
+
+
+def _nested_apply(item: Any, feature_path: _VisitPath, func: Callable[[Any, _VisitPath], Any]):
+ # see _visit_with_path() to see how feature paths are constructed
+ item = func(item, feature_path)
+ if feature_path:
+ key = feature_path[0]
+ if key == 0:
+ for i in range(len(item)):
+ item[i] = _nested_apply(item[i], feature_path[1:], func)
+ else:
+ item[key] = _nested_apply(item[key], feature_path[1:], func)
+ return item
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/generator/__init__.py b/venv/lib/python3.10/site-packages/datasets/packaged_modules/generator/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/generator/__pycache__/__init__.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/packaged_modules/generator/__pycache__/__init__.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..39ff6a359e83afadaa0334191cd0de1fe66e7da8
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/packaged_modules/generator/__pycache__/__init__.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/generator/__pycache__/generator.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/packaged_modules/generator/__pycache__/generator.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..9d3ad5a92d88eeedfc23246562da0b99a9e17c66
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/packaged_modules/generator/__pycache__/generator.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/generator/generator.py b/venv/lib/python3.10/site-packages/datasets/packaged_modules/generator/generator.py
new file mode 100644
index 0000000000000000000000000000000000000000..dac3f0d92dfea56636e24c9c8e0cba9e46e96a90
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/packaged_modules/generator/generator.py
@@ -0,0 +1,33 @@
+from dataclasses import dataclass
+from typing import Callable, Optional
+
+import datasets
+
+
+@dataclass
+class GeneratorConfig(datasets.BuilderConfig):
+ generator: Optional[Callable] = None
+ gen_kwargs: Optional[dict] = None
+ features: Optional[datasets.Features] = None
+ split: datasets.NamedSplit = datasets.Split.TRAIN
+
+ def __post_init__(self):
+ super().__post_init__()
+ if self.generator is None:
+ raise ValueError("generator must be specified")
+
+ if self.gen_kwargs is None:
+ self.gen_kwargs = {}
+
+
+class Generator(datasets.GeneratorBasedBuilder):
+ BUILDER_CONFIG_CLASS = GeneratorConfig
+
+ def _info(self):
+ return datasets.DatasetInfo(features=self.config.features)
+
+ def _split_generators(self, dl_manager):
+ return [datasets.SplitGenerator(name=self.config.split, gen_kwargs=self.config.gen_kwargs)]
+
+ def _generate_examples(self, **gen_kwargs):
+ yield from enumerate(self.config.generator(**gen_kwargs))
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/hdf5/__init__.py b/venv/lib/python3.10/site-packages/datasets/packaged_modules/hdf5/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/hdf5/__pycache__/__init__.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/packaged_modules/hdf5/__pycache__/__init__.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..095946550bd9d5e22968d02d8491cc7c8426ab79
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/packaged_modules/hdf5/__pycache__/__init__.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/hdf5/__pycache__/hdf5.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/packaged_modules/hdf5/__pycache__/hdf5.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..0c4f429fee1bbbc519d64e934cbe8a8b02fd917d
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/packaged_modules/hdf5/__pycache__/hdf5.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/hdf5/hdf5.py b/venv/lib/python3.10/site-packages/datasets/packaged_modules/hdf5/hdf5.py
new file mode 100644
index 0000000000000000000000000000000000000000..1b0e80aa6a8c78c3695b5ba0ff07ca7a74606cf4
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/packaged_modules/hdf5/hdf5.py
@@ -0,0 +1,391 @@
+import itertools
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Optional
+
+import numpy as np
+import pyarrow as pa
+
+import datasets
+from datasets.features.features import (
+ Array2D,
+ Array3D,
+ Array4D,
+ Array5D,
+ Features,
+ LargeList,
+ List,
+ Value,
+ _ArrayXD,
+ _arrow_to_datasets_dtype,
+)
+from datasets.table import cast_table_to_features
+
+
+if TYPE_CHECKING:
+ import h5py
+
+logger = datasets.utils.logging.get_logger(__name__)
+
+EXTENSIONS = [".h5", ".hdf5"]
+
+
+@dataclass
+class HDF5Config(datasets.BuilderConfig):
+ """BuilderConfig for HDF5."""
+
+ batch_size: Optional[int] = None
+ features: Optional[datasets.Features] = None
+
+
+class HDF5(datasets.ArrowBasedBuilder):
+ """ArrowBasedBuilder that converts HDF5 files to Arrow tables using the HF extension types."""
+
+ BUILDER_CONFIG_CLASS = HDF5Config
+
+ def _info(self):
+ return datasets.DatasetInfo(features=self.config.features)
+
+ def _split_generators(self, dl_manager):
+ import h5py
+
+ if not self.config.data_files:
+ raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}")
+ dl_manager.download_config.extract_on_the_fly = True
+ data_files = dl_manager.download_and_extract(self.config.data_files)
+ splits = []
+ for split_name, files in data_files.items():
+ if isinstance(files, str):
+ files = [files]
+
+ files = [dl_manager.iter_files(file) for file in files]
+ # Infer features from first file
+ if self.info.features is None:
+ for first_file in itertools.chain.from_iterable(files):
+ with open(first_file, "rb") as f:
+ with h5py.File(f, "r") as h5:
+ self.info.features = _recursive_infer_features(h5)
+ break
+ splits.append(datasets.SplitGenerator(name=split_name, gen_kwargs={"files": files}))
+ return splits
+
+ def _generate_tables(self, files):
+ import h5py
+
+ batch_size_cfg = self.config.batch_size
+ for file_idx, file in enumerate(itertools.chain.from_iterable(files)):
+ try:
+ with open(file, "rb") as f:
+ with h5py.File(f, "r") as h5:
+ # Infer features and lengths from first file
+ if self.info.features is None:
+ self.info.features = _recursive_infer_features(h5)
+ num_rows = _check_dataset_lengths(h5, self.info.features)
+ if num_rows is None:
+ logger.warning(f"File {file} contains no data, skipping...")
+ continue
+ effective_batch = batch_size_cfg or self._writer_batch_size or num_rows
+ for start in range(0, num_rows, effective_batch):
+ end = min(start + effective_batch, num_rows)
+ pa_table = _recursive_load_arrays(h5, self.info.features, start, end)
+ if pa_table is None:
+ logger.warning(f"File {file} contains no data, skipping...")
+ continue
+ yield f"{file_idx}_{start}", cast_table_to_features(pa_table, self.info.features)
+ except ValueError as e:
+ logger.error(f"Failed to read file '{file}' with error {type(e)}: {e}")
+ raise
+
+
+# ┌───────────┐
+# │ Complex │
+# └───────────┘
+
+
+def _is_complex_dtype(dtype: np.dtype) -> bool:
+ if dtype.kind == "c":
+ return True
+ if dtype.subdtype is not None:
+ return _is_complex_dtype(dtype.subdtype[0])
+ return False
+
+
+def _create_complex_features(dset) -> Features:
+ if dset.dtype.subdtype is not None:
+ dtype, data_shape = dset.dtype.subdtype
+ else:
+ data_shape = dset.shape[1:]
+ dtype = dset.dtype
+
+ if dtype == np.complex64:
+ # two float32s
+ value_type = Value("float32")
+ elif dtype == np.complex128:
+ # two float64s
+ value_type = Value("float64")
+ else:
+ logger.warning(f"Found complex dtype {dtype} that is not supported. Converting to float64...")
+ value_type = Value("float64")
+
+ return Features(
+ {
+ "real": _create_sized_feature_impl(data_shape, value_type),
+ "imag": _create_sized_feature_impl(data_shape, value_type),
+ }
+ )
+
+
+def _convert_complex_to_nested(arr: np.ndarray) -> pa.StructArray:
+ data = {
+ "real": datasets.features.features.numpy_to_pyarrow_listarray(arr.real),
+ "imag": datasets.features.features.numpy_to_pyarrow_listarray(arr.imag),
+ }
+ return pa.StructArray.from_arrays([data["real"], data["imag"]], names=["real", "imag"])
+
+
+# ┌────────────┐
+# │ Compound │
+# └────────────┘
+
+
+def _is_compound_dtype(dtype: np.dtype) -> bool:
+ return dtype.kind == "V"
+
+
+@dataclass
+class _CompoundGroup:
+ dset: "h5py.Dataset"
+ data: np.ndarray = None
+
+ def items(self):
+ for field_name in self.dset.dtype.names:
+ field_dtype = self.dset.dtype[field_name]
+ yield field_name, _CompoundField(self.data, field_name, field_dtype)
+
+
+@dataclass
+class _CompoundField:
+ data: Optional[np.ndarray]
+ name: str
+ dtype: np.dtype
+ shape: tuple[int, ...] = field(init=False)
+
+ def __post_init__(self):
+ self.shape = (len(self.data) if self.data is not None else 0,) + self.dtype.shape
+
+ def __getitem__(self, key):
+ return self.data[key][self.name]
+
+
+def _create_compound_features(dset) -> Features:
+ mock_group = _CompoundGroup(dset)
+ return _recursive_infer_features(mock_group)
+
+
+def _convert_compound_to_nested(arr, dset) -> pa.StructArray:
+ mock_group = _CompoundGroup(dset, data=arr)
+ features = _create_compound_features(dset)
+ return _recursive_load_arrays(mock_group, features, 0, len(arr))
+
+
+# ┌───────────────────┐
+# │ Variable-Length │
+# └───────────────────┘
+
+
+def _is_vlen_dtype(dtype: np.dtype) -> bool:
+ if dtype.metadata and "vlen" in dtype.metadata:
+ return True
+ return False
+
+
+def _create_vlen_features(dset) -> Features:
+ vlen_dtype = dset.dtype.metadata["vlen"]
+ if vlen_dtype in (str, bytes):
+ return Value("string")
+ inner_feature = _np_to_pa_to_hf_value(vlen_dtype)
+ return List(inner_feature)
+
+
+def _convert_vlen_to_array(arr: np.ndarray) -> pa.Array:
+ return datasets.features.features.numpy_to_pyarrow_listarray(arr)
+
+
+# ┌───────────┐
+# │ Generic │
+# └───────────┘
+
+
+def _recursive_infer_features(h5_obj) -> Features:
+ features_dict = {}
+ for path, dset in h5_obj.items():
+ if _is_group(dset):
+ features = _recursive_infer_features(dset)
+ if features:
+ features_dict[path] = features
+ elif _is_dataset(dset):
+ features = _infer_feature(dset)
+ if features:
+ features_dict[path] = features
+
+ return Features(features_dict)
+
+
+def _infer_feature(dset):
+ if _is_complex_dtype(dset.dtype):
+ return _create_complex_features(dset)
+ elif _is_compound_dtype(dset.dtype) or dset.dtype.kind == "V":
+ return _create_compound_features(dset)
+ elif _is_vlen_dtype(dset.dtype):
+ return _create_vlen_features(dset)
+ return _create_sized_feature(dset)
+
+
+def _load_array(dset, path: str, start: int, end: int) -> pa.Array:
+ arr = dset[start:end]
+
+ if _is_vlen_dtype(dset.dtype):
+ return _convert_vlen_to_array(arr)
+ elif _is_complex_dtype(dset.dtype):
+ return _convert_complex_to_nested(arr)
+ elif _is_compound_dtype(dset.dtype):
+ return _convert_compound_to_nested(arr, dset)
+ elif dset.dtype.kind == "O":
+ raise ValueError(
+ f"Object dtype dataset '{path}' is not supported. "
+ f"For variable-length data, please use h5py.vlen_dtype() "
+ f"when creating the HDF5 file. "
+ f"See: https://docs.h5py.org/en/stable/special.html#variable-length-strings"
+ )
+ else:
+ # If any non-batch dimension is zero, emit an unsized pa.list_
+ # to avoid creating FixedSizeListArray with list_size=0.
+ if any(dim == 0 for dim in dset.shape[1:]):
+ inner_type = pa.from_numpy_dtype(dset.dtype)
+ return pa.array([[] for _ in arr], type=pa.list_(inner_type))
+ else:
+ return datasets.features.features.numpy_to_pyarrow_listarray(arr)
+
+
+def _recursive_load_arrays(h5_obj, features: Features, start: int, end: int):
+ batch_dict = {}
+ for path, dset in h5_obj.items():
+ if path not in features:
+ continue
+ if _is_group(dset):
+ arr = _recursive_load_arrays(dset, features[path], start, end)
+ elif _is_dataset(dset):
+ arr = _load_array(dset, path, start, end)
+ else:
+ raise ValueError(f"Unexpected type {type(dset)}")
+
+ if arr is not None:
+ batch_dict[path] = arr
+
+ if _is_file(h5_obj):
+ return pa.Table.from_pydict(batch_dict)
+
+ if batch_dict:
+ should_chunk, keys, values = False, [], []
+ for k, v in batch_dict.items():
+ if isinstance(v, pa.ChunkedArray):
+ should_chunk = True
+ v = v.combine_chunks()
+ keys.append(k)
+ values.append(v)
+
+ sarr = pa.StructArray.from_arrays(values, names=keys)
+ return pa.chunked_array(sarr) if should_chunk else sarr
+
+
+# ┌─────────────┐
+# │ Utilities │
+# └─────────────┘
+
+
+def _create_sized_feature(dset):
+ dset_shape = dset.shape[1:]
+ value_feature = _np_to_pa_to_hf_value(dset.dtype)
+ return _create_sized_feature_impl(dset_shape, value_feature)
+
+
+def _create_sized_feature_impl(dset_shape, value_feature):
+ dtype_str = value_feature.dtype
+ if any(dim == 0 for dim in dset_shape):
+ logger.warning(
+ f"HDF5 to Arrow: Found a dataset with shape {dset_shape} and dtype {dtype_str} that has a dimension with size 0. Shape information will be lost in the conversion to List({value_feature})."
+ )
+ return List(value_feature)
+
+ rank = len(dset_shape)
+ if rank == 0:
+ return value_feature
+ elif rank == 1:
+ return List(value_feature, length=dset_shape[0])
+ elif rank <= 5:
+ return _sized_arrayxd(rank)(shape=dset_shape, dtype=dtype_str)
+ else:
+ raise TypeError(f"Array{rank}D not supported. Maximum 5 dimensions allowed.")
+
+
+def _sized_arrayxd(rank: int):
+ return {2: Array2D, 3: Array3D, 4: Array4D, 5: Array5D}[rank]
+
+
+def _np_to_pa_to_hf_value(numpy_dtype: np.dtype) -> Value:
+ return Value(dtype=_arrow_to_datasets_dtype(pa.from_numpy_dtype(numpy_dtype)))
+
+
+def _first_dataset(h5_obj, features: Features, prefix=""):
+ for path, dset in h5_obj.items():
+ if path not in features:
+ continue
+ if _is_group(dset):
+ found = _first_dataset(dset, features[path], prefix=f"{prefix}{path}/")
+ if found is not None:
+ return found
+ elif _is_dataset(dset):
+ return f"{prefix}{path}"
+
+
+def _check_dataset_lengths(h5_obj, features: Features) -> int:
+ first_path = _first_dataset(h5_obj, features)
+ if first_path is None:
+ return None
+
+ num_rows = h5_obj[first_path].shape[0]
+ for path, dset in h5_obj.items():
+ if path not in features:
+ continue
+ if _is_dataset(dset):
+ if dset.shape[0] != num_rows:
+ raise ValueError(f"Dataset '{path}' has length {dset.shape[0]} but expected {num_rows}")
+ return num_rows
+
+
+def _is_group(h5_obj) -> bool:
+ import h5py
+
+ return isinstance(h5_obj, h5py.Group) or isinstance(h5_obj, _CompoundGroup)
+
+
+def _is_dataset(h5_obj) -> bool:
+ import h5py
+
+ return isinstance(h5_obj, h5py.Dataset) or isinstance(h5_obj, _CompoundField)
+
+
+def _is_file(h5_obj) -> bool:
+ import h5py
+
+ return isinstance(h5_obj, h5py.File)
+
+
+def _has_zero_dimensions(feature):
+ if isinstance(feature, _ArrayXD):
+ return any(dim == 0 for dim in feature.shape)
+ elif isinstance(feature, List):
+ return feature.length == 0 or _has_zero_dimensions(feature.feature)
+ elif isinstance(feature, LargeList):
+ return _has_zero_dimensions(feature.feature)
+ else:
+ return False
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/imagefolder/__init__.py b/venv/lib/python3.10/site-packages/datasets/packaged_modules/imagefolder/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/imagefolder/__pycache__/__init__.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/packaged_modules/imagefolder/__pycache__/__init__.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..6ca62c9784e6ad8899284b254dbbe682c37a190d
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/packaged_modules/imagefolder/__pycache__/__init__.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/imagefolder/__pycache__/imagefolder.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/packaged_modules/imagefolder/__pycache__/imagefolder.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..a148bead6eb95b2c5646c46af55af95977861bb2
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/packaged_modules/imagefolder/__pycache__/imagefolder.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/imagefolder/imagefolder.py b/venv/lib/python3.10/site-packages/datasets/packaged_modules/imagefolder/imagefolder.py
new file mode 100644
index 0000000000000000000000000000000000000000..aba9452ad4e12833c60ef78df8b3a0cae519f4a4
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/packaged_modules/imagefolder/imagefolder.py
@@ -0,0 +1,103 @@
+import datasets
+
+from ..folder_based_builder import folder_based_builder
+
+
+logger = datasets.utils.logging.get_logger(__name__)
+
+
+class ImageFolderConfig(folder_based_builder.FolderBasedBuilderConfig):
+ """BuilderConfig for ImageFolder."""
+
+ drop_labels: bool = None
+ drop_metadata: bool = None
+
+ def __post_init__(self):
+ super().__post_init__()
+
+
+class ImageFolder(folder_based_builder.FolderBasedBuilder):
+ BASE_FEATURE = datasets.Image
+ BASE_COLUMN_NAME = "image"
+ BUILDER_CONFIG_CLASS = ImageFolderConfig
+ EXTENSIONS: list[str] # definition at the bottom of the script
+
+
+# Obtained with:
+# ```
+# import PIL.Image
+# IMAGE_EXTENSIONS = []
+# PIL.Image.init()
+# for ext, format in PIL.Image.EXTENSION.items():
+# if format in PIL.Image.OPEN:
+# IMAGE_EXTENSIONS.append(ext[1:])
+# ```
+# We intentionally do not run this code on launch because:
+# (1) Pillow is an optional dependency, so importing Pillow in global namespace is not allowed
+# (2) To ensure the list of supported extensions is deterministic
+IMAGE_EXTENSIONS = [
+ ".blp",
+ ".bmp",
+ ".dib",
+ ".bufr",
+ ".cur",
+ ".pcx",
+ ".dcx",
+ ".dds",
+ ".ps",
+ ".eps",
+ ".fit",
+ ".fits",
+ ".fli",
+ ".flc",
+ ".ftc",
+ ".ftu",
+ ".gbr",
+ ".gif",
+ ".grib",
+ # ".h5", # may contain zero or several images
+ # ".hdf", # may contain zero or several images
+ ".png",
+ ".apng",
+ ".jp2",
+ ".j2k",
+ ".jpc",
+ ".jpf",
+ ".jpx",
+ ".j2c",
+ ".icns",
+ ".ico",
+ ".im",
+ ".iim",
+ ".tif",
+ ".tiff",
+ ".jfif",
+ ".jpe",
+ ".jpg",
+ ".jpeg",
+ ".mpg",
+ ".mpeg",
+ ".msp",
+ ".pcd",
+ ".pxr",
+ ".pbm",
+ ".pgm",
+ ".ppm",
+ ".pnm",
+ ".psd",
+ ".bw",
+ ".rgb",
+ ".rgba",
+ ".sgi",
+ ".ras",
+ ".tga",
+ ".icb",
+ ".vda",
+ ".vst",
+ ".webp",
+ ".wmf",
+ ".emf",
+ ".xbm",
+ ".xpm",
+]
+ImageFolder.EXTENSIONS = IMAGE_EXTENSIONS
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/json/__init__.py b/venv/lib/python3.10/site-packages/datasets/packaged_modules/json/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/json/__pycache__/__init__.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/packaged_modules/json/__pycache__/__init__.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..e586097aef70355de7ba63864c5cc68b4a2cc2e0
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/packaged_modules/json/__pycache__/__init__.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/json/__pycache__/json.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/packaged_modules/json/__pycache__/json.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..0ae9f82f37ed5dff12823a5c3b57355cd3db51e4
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/packaged_modules/json/__pycache__/json.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/json/json.py b/venv/lib/python3.10/site-packages/datasets/packaged_modules/json/json.py
new file mode 100644
index 0000000000000000000000000000000000000000..c5d8bcd03fc8ea48298e7515b188761f8a5d9442
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/packaged_modules/json/json.py
@@ -0,0 +1,192 @@
+import io
+import itertools
+from dataclasses import dataclass
+from typing import Optional
+
+import pandas as pd
+import pyarrow as pa
+import pyarrow.json as paj
+
+import datasets
+import datasets.config
+from datasets.table import table_cast
+from datasets.utils.file_utils import readline
+
+
+logger = datasets.utils.logging.get_logger(__name__)
+
+
+def ujson_dumps(*args, **kwargs):
+ try:
+ return pd.io.json.ujson_dumps(*args, **kwargs)
+ except AttributeError:
+ # Before pandas-2.2.0, ujson_dumps was renamed to dumps: import ujson_dumps as dumps
+ return pd.io.json.dumps(*args, **kwargs)
+
+
+def ujson_loads(*args, **kwargs):
+ try:
+ return pd.io.json.ujson_loads(*args, **kwargs)
+ except AttributeError:
+ # Before pandas-2.2.0, ujson_loads was renamed to loads: import ujson_loads as loads
+ return pd.io.json.loads(*args, **kwargs)
+
+
+def pandas_read_json(path_or_buf, **kwargs):
+ if datasets.config.PANDAS_VERSION.major >= 2:
+ kwargs["dtype_backend"] = "pyarrow"
+ return pd.read_json(path_or_buf, **kwargs)
+
+
+@dataclass
+class JsonConfig(datasets.BuilderConfig):
+ """BuilderConfig for JSON."""
+
+ features: Optional[datasets.Features] = None
+ encoding: str = "utf-8"
+ encoding_errors: Optional[str] = None
+ field: Optional[str] = None
+ use_threads: bool = True # deprecated
+ block_size: Optional[int] = None # deprecated
+ chunksize: int = 10 << 20 # 10MB
+ newlines_in_values: Optional[bool] = None
+
+ def __post_init__(self):
+ super().__post_init__()
+
+
+class Json(datasets.ArrowBasedBuilder):
+ BUILDER_CONFIG_CLASS = JsonConfig
+
+ def _info(self):
+ if self.config.block_size is not None:
+ logger.warning("The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead")
+ self.config.chunksize = self.config.block_size
+ if self.config.use_threads is not True:
+ logger.warning(
+ "The JSON loader parameter `use_threads` is deprecated and doesn't have any effect anymore."
+ )
+ if self.config.newlines_in_values is not None:
+ raise ValueError("The JSON loader parameter `newlines_in_values` is no longer supported")
+ return datasets.DatasetInfo(features=self.config.features)
+
+ def _split_generators(self, dl_manager):
+ """We handle string, list and dicts in datafiles"""
+ if not self.config.data_files:
+ raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}")
+ dl_manager.download_config.extract_on_the_fly = True
+ data_files = dl_manager.download_and_extract(self.config.data_files)
+ splits = []
+ for split_name, files in data_files.items():
+ if isinstance(files, str):
+ files = [files]
+ files = [dl_manager.iter_files(file) for file in files]
+ splits.append(datasets.SplitGenerator(name=split_name, gen_kwargs={"files": files}))
+ return splits
+
+ def _cast_table(self, pa_table: pa.Table) -> pa.Table:
+ if self.config.features is not None:
+ # adding missing columns
+ for column_name in set(self.config.features) - set(pa_table.column_names):
+ type = self.config.features.arrow_schema.field(column_name).type
+ pa_table = pa_table.append_column(column_name, pa.array([None] * len(pa_table), type=type))
+ # convert to string when needed
+ for i, column_name in enumerate(pa_table.column_names):
+ if pa.types.is_struct(pa_table[column_name].type) and self.config.features.get(
+ column_name, None
+ ) == datasets.Value("string"):
+ jsonl = (
+ pa_table[column_name]
+ .to_pandas(types_mapper=pd.ArrowDtype)
+ .to_json(orient="records", lines=True)
+ )
+ string_array = pa.array(
+ ("{" + x.rstrip() for x in ("\n" + jsonl).split("\n{") if x), type=pa.string()
+ )
+ pa_table = pa_table.set_column(i, column_name, string_array)
+ # more expensive cast to support nested structures with keys in a different order
+ # allows str <-> int/float or str to Audio for example
+ pa_table = table_cast(pa_table, self.config.features.arrow_schema)
+ return pa_table
+
+ def _generate_tables(self, files):
+ for file_idx, file in enumerate(itertools.chain.from_iterable(files)):
+ # If the file is one json object and if we need to look at the items in one specific field
+ if self.config.field is not None:
+ with open(file, encoding=self.config.encoding, errors=self.config.encoding_errors) as f:
+ dataset = ujson_loads(f.read())
+ # We keep only the field we are interested in
+ dataset = dataset[self.config.field]
+ df = pandas_read_json(io.StringIO(ujson_dumps(dataset)))
+ if df.columns.tolist() == [0]:
+ df.columns = list(self.config.features) if self.config.features else ["text"]
+ pa_table = pa.Table.from_pandas(df, preserve_index=False)
+ yield file_idx, self._cast_table(pa_table)
+
+ # If the file has one json object per line
+ else:
+ with open(file, "rb") as f:
+ batch_idx = 0
+ # Use block_size equal to the chunk size divided by 32 to leverage multithreading
+ # Set a default minimum value of 16kB if the chunk size is really small
+ block_size = max(self.config.chunksize // 32, 16 << 10)
+ encoding_errors = (
+ self.config.encoding_errors if self.config.encoding_errors is not None else "strict"
+ )
+ while True:
+ batch = f.read(self.config.chunksize)
+ if not batch:
+ break
+ # Finish current line
+ try:
+ batch += f.readline()
+ except (AttributeError, io.UnsupportedOperation):
+ batch += readline(f)
+ # PyArrow only accepts utf-8 encoded bytes
+ if self.config.encoding != "utf-8":
+ batch = batch.decode(self.config.encoding, errors=encoding_errors).encode("utf-8")
+ try:
+ while True:
+ try:
+ pa_table = paj.read_json(
+ io.BytesIO(batch), read_options=paj.ReadOptions(block_size=block_size)
+ )
+ break
+ except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
+ if (
+ isinstance(e, pa.ArrowInvalid)
+ and "straddling" not in str(e)
+ or block_size > len(batch)
+ ):
+ raise
+ else:
+ # Increase the block size in case it was too small.
+ # The block size will be reset for the next file.
+ logger.debug(
+ f"Batch of {len(batch)} bytes couldn't be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}."
+ )
+ block_size *= 2
+ except pa.ArrowInvalid as e:
+ try:
+ with open(
+ file, encoding=self.config.encoding, errors=self.config.encoding_errors
+ ) as f:
+ df = pandas_read_json(f)
+ except ValueError:
+ logger.error(f"Failed to load JSON from file '{file}' with error {type(e)}: {e}")
+ raise e
+ if df.columns.tolist() == [0]:
+ df.columns = list(self.config.features) if self.config.features else ["text"]
+ try:
+ pa_table = pa.Table.from_pandas(df, preserve_index=False)
+ except pa.ArrowInvalid as e:
+ logger.error(
+ f"Failed to convert pandas DataFrame to Arrow Table from file '{file}' with error {type(e)}: {e}"
+ )
+ raise ValueError(
+ f"Failed to convert pandas DataFrame to Arrow Table from file {file}."
+ ) from None
+ yield file_idx, self._cast_table(pa_table)
+ break
+ yield (file_idx, batch_idx), self._cast_table(pa_table)
+ batch_idx += 1
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/niftifolder/__init__.py b/venv/lib/python3.10/site-packages/datasets/packaged_modules/niftifolder/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/niftifolder/__pycache__/__init__.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/packaged_modules/niftifolder/__pycache__/__init__.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..83700705994862b7445e3c600d6abdc4166bfb30
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/packaged_modules/niftifolder/__pycache__/__init__.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/niftifolder/__pycache__/niftifolder.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/packaged_modules/niftifolder/__pycache__/niftifolder.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..bff48b82b18137f64772a29ea8924ef3fc03f212
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/packaged_modules/niftifolder/__pycache__/niftifolder.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/niftifolder/niftifolder.py b/venv/lib/python3.10/site-packages/datasets/packaged_modules/niftifolder/niftifolder.py
new file mode 100644
index 0000000000000000000000000000000000000000..c6d039419f192cfbe43d9c5cad3f975e6caa5b76
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/packaged_modules/niftifolder/niftifolder.py
@@ -0,0 +1,23 @@
+import datasets
+
+from ..folder_based_builder import folder_based_builder
+
+
+logger = datasets.utils.logging.get_logger(__name__)
+
+
+class NiftiFolderConfig(folder_based_builder.FolderBasedBuilderConfig):
+ """BuilderConfig for NiftiFolder."""
+
+ drop_labels: bool = None
+ drop_metadata: bool = None
+
+ def __post_init__(self):
+ super().__post_init__()
+
+
+class NiftiFolder(folder_based_builder.FolderBasedBuilder):
+ BASE_FEATURE = datasets.Nifti
+ BASE_COLUMN_NAME = "nifti"
+ BUILDER_CONFIG_CLASS = NiftiFolderConfig
+ EXTENSIONS: list[str] = [".nii", ".nii.gz"]
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/pandas/__init__.py b/venv/lib/python3.10/site-packages/datasets/packaged_modules/pandas/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/pandas/__pycache__/__init__.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/packaged_modules/pandas/__pycache__/__init__.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..45fcbb81709f63581dce4e8e1d5a7856a8bfa18e
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/packaged_modules/pandas/__pycache__/__init__.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/pandas/__pycache__/pandas.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/packaged_modules/pandas/__pycache__/pandas.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..4968269552e53cfc5236f55b43d6731c7c6cc190
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/packaged_modules/pandas/__pycache__/pandas.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/pandas/pandas.py b/venv/lib/python3.10/site-packages/datasets/packaged_modules/pandas/pandas.py
new file mode 100644
index 0000000000000000000000000000000000000000..d1eb50d33c899d83421167bf6dc70680e437e1e4
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/packaged_modules/pandas/pandas.py
@@ -0,0 +1,65 @@
+import itertools
+import warnings
+from dataclasses import dataclass
+from typing import Optional
+
+import pandas as pd
+import pyarrow as pa
+
+import datasets
+from datasets.table import table_cast
+
+
+@dataclass
+class PandasConfig(datasets.BuilderConfig):
+ """BuilderConfig for Pandas."""
+
+ features: Optional[datasets.Features] = None
+
+ def __post_init__(self):
+ super().__post_init__()
+
+
+class Pandas(datasets.ArrowBasedBuilder):
+ BUILDER_CONFIG_CLASS = PandasConfig
+
+ def _info(self):
+ warnings.warn(
+ "The Pandas builder is deprecated and will be removed in the next major version of datasets.",
+ FutureWarning,
+ )
+ return datasets.DatasetInfo(features=self.config.features)
+
+ def _split_generators(self, dl_manager):
+ """We handle string, list and dicts in datafiles"""
+ if not self.config.data_files:
+ raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}")
+ data_files = dl_manager.download_and_extract(self.config.data_files)
+ if isinstance(data_files, (str, list, tuple)):
+ files = data_files
+ if isinstance(files, str):
+ files = [files]
+ # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
+ files = [dl_manager.iter_files(file) for file in files]
+ return [datasets.SplitGenerator(name=datasets.Split.TRAIN, gen_kwargs={"files": files})]
+ splits = []
+ for split_name, files in data_files.items():
+ if isinstance(files, str):
+ files = [files]
+ # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
+ files = [dl_manager.iter_files(file) for file in files]
+ splits.append(datasets.SplitGenerator(name=split_name, gen_kwargs={"files": files}))
+ return splits
+
+ def _cast_table(self, pa_table: pa.Table) -> pa.Table:
+ if self.config.features is not None:
+ # more expensive cast to support nested features with keys in a different order
+ # allows str <-> int/float or str to Audio for example
+ pa_table = table_cast(pa_table, self.config.features.arrow_schema)
+ return pa_table
+
+ def _generate_tables(self, files):
+ for i, file in enumerate(itertools.chain.from_iterable(files)):
+ with open(file, "rb") as f:
+ pa_table = pa.Table.from_pandas(pd.read_pickle(f))
+ yield i, self._cast_table(pa_table)
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/parquet/__init__.py b/venv/lib/python3.10/site-packages/datasets/packaged_modules/parquet/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/parquet/__pycache__/__init__.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/packaged_modules/parquet/__pycache__/__init__.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..f7f437c82fcf58f80c40d35bd1c209749e2dd75e
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/packaged_modules/parquet/__pycache__/__init__.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/parquet/__pycache__/parquet.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/packaged_modules/parquet/__pycache__/parquet.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..7f1084b3c97e4cb6f36cca47e0a64945ddfbb393
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/packaged_modules/parquet/__pycache__/parquet.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/parquet/parquet.py b/venv/lib/python3.10/site-packages/datasets/packaged_modules/parquet/parquet.py
new file mode 100644
index 0000000000000000000000000000000000000000..52a675d41c7bd61e78b2372af610280ed3deb2ec
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/packaged_modules/parquet/parquet.py
@@ -0,0 +1,189 @@
+import itertools
+from dataclasses import dataclass
+from typing import Literal, Optional, Union
+
+import pyarrow as pa
+import pyarrow.dataset as ds
+import pyarrow.parquet as pq
+
+import datasets
+from datasets.table import table_cast
+
+
+logger = datasets.utils.logging.get_logger(__name__)
+
+
+@dataclass
+class ParquetConfig(datasets.BuilderConfig):
+ """
+ BuilderConfig for Parquet.
+
+ Args:
+ batch_size (`int`, *optional*):
+ Size of the RecordBatches to iterate on.
+ The default is the row group size (defined by the first row group).
+ columns (`list[str]`, *optional*)
+ List of columns to load, the other ones are ignored.
+ All columns are loaded by default.
+ features: (`Features`, *optional*):
+ Cast the data to `features`.
+ filters (`Union[pyarrow.dataset.Expression, list[tuple], list[list[tuple]]]`, *optional*):
+ Return only the rows matching the filter.
+ If possible the predicate will be pushed down to exploit the partition information
+ or internal metadata found in the data source, e.g. Parquet statistics.
+ Otherwise filters the loaded RecordBatches before yielding them.
+ fragment_scan_options (`pyarrow.dataset.ParquetFragmentScanOptions`, *optional*)
+ Scan-specific options for Parquet fragments.
+ This is especially useful to configure buffering and caching.
+
+
+ on_bad_files (`Literal["error", "warn", "skip"]`, *optional*, defaults to "error")
+ Specify what to do upon encountering a bad file (a file that can't be read). Allowed values are :
+ * 'error', raise an Exception when a bad file is encountered.
+ * 'warn', raise a warning when a bad file is encountered and skip that file.
+ * 'skip', skip bad files without raising or warning when they are encountered.
+
+
+
+ Example:
+
+ Load a subset of columns:
+
+ ```python
+ >>> ds = load_dataset(parquet_dataset_id, columns=["col_0", "col_1"])
+ ```
+
+ Stream data and efficiently filter data, possibly skipping entire files or row groups:
+
+ ```python
+ >>> filters = [("col_0", "==", 0)]
+ >>> ds = load_dataset(parquet_dataset_id, streaming=True, filters=filters)
+ ```
+
+ Increase the minimum request size when streaming from 32MiB (default) to 128MiB and enable prefetching:
+
+ ```python
+ >>> import pyarrow
+ >>> import pyarrow.dataset
+ >>> fragment_scan_options = pyarrow.dataset.ParquetFragmentScanOptions(
+ ... cache_options=pyarrow.CacheOptions(
+ ... prefetch_limit=1,
+ ... range_size_limit=128 << 20
+ ... ),
+ ... )
+ >>> ds = load_dataset(parquet_dataset_id, streaming=True, fragment_scan_options=fragment_scan_options)
+ ```
+
+ """
+
+ batch_size: Optional[int] = None
+ columns: Optional[list[str]] = None
+ features: Optional[datasets.Features] = None
+ filters: Optional[Union[ds.Expression, list[tuple], list[list[tuple]]]] = None
+ fragment_scan_options: Optional[ds.ParquetFragmentScanOptions] = None
+ on_bad_files: Literal["error", "warn", "skip"] = "error"
+
+ def __post_init__(self):
+ super().__post_init__()
+
+
+class Parquet(datasets.ArrowBasedBuilder):
+ BUILDER_CONFIG_CLASS = ParquetConfig
+
+ def _info(self):
+ if (
+ self.config.columns is not None
+ and self.config.features is not None
+ and set(self.config.columns) != set(self.config.features)
+ ):
+ raise ValueError(
+ "The columns and features argument must contain the same columns, but got ",
+ f"{self.config.columns} and {self.config.features}",
+ )
+ return datasets.DatasetInfo(features=self.config.features)
+
+ def _split_generators(self, dl_manager):
+ """We handle string, list and dicts in datafiles"""
+ if not self.config.data_files:
+ raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}")
+ dl_manager.download_config.extract_on_the_fly = True
+ data_files = dl_manager.download_and_extract(self.config.data_files)
+ splits = []
+ for split_name, files in data_files.items():
+ if isinstance(files, str):
+ files = [files]
+ # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
+ files = [dl_manager.iter_files(file) for file in files]
+ # Infer features if they are stored in the arrow schema
+ if self.info.features is None:
+ for file in itertools.chain.from_iterable(files):
+ try:
+ with open(file, "rb") as f:
+ self.info.features = datasets.Features.from_arrow_schema(pq.read_schema(f))
+ break
+ except pa.ArrowInvalid as e:
+ if self.config.on_bad_files == "error":
+ logger.error(f"Failed to read schema from '{file}' with error {type(e).__name__}: {e}")
+ raise
+ elif self.config.on_bad_files == "warn":
+ logger.warning(f"Skipping bad schema from '{file}'. {type(e).__name__}: {e}`")
+ else:
+ logger.debug(f"Skipping bad schema from '{file}'. {type(e).__name__}: {e}`")
+ if self.info.features is None:
+ raise ValueError(
+ f"At least one valid data file must be specified, all the data_files are invalid: {self.config.data_files}"
+ )
+ splits.append(datasets.SplitGenerator(name=split_name, gen_kwargs={"files": files}))
+ if self.config.columns is not None and set(self.config.columns) != set(self.info.features):
+ self.info.features = datasets.Features(
+ {col: feat for col, feat in self.info.features.items() if col in self.config.columns}
+ )
+ return splits
+
+ def _cast_table(self, pa_table: pa.Table) -> pa.Table:
+ if self.info.features is not None:
+ # more expensive cast to support nested features with keys in a different order
+ # allows str <-> int/float or str to Audio for example
+ pa_table = table_cast(pa_table, self.info.features.arrow_schema)
+ return pa_table
+
+ def _generate_tables(self, files):
+ if self.config.features is not None and self.config.columns is not None:
+ if sorted(field.name for field in self.info.features.arrow_schema) != sorted(self.config.columns):
+ raise ValueError(
+ f"Tried to load parquet data with columns '{self.config.columns}' with mismatching features '{self.info.features}'"
+ )
+ filter_expr = (
+ pq.filters_to_expression(self.config.filters)
+ if isinstance(self.config.filters, list)
+ else self.config.filters
+ )
+ parquet_file_format = ds.ParquetFileFormat(default_fragment_scan_options=self.config.fragment_scan_options)
+ for file_idx, file in enumerate(itertools.chain.from_iterable(files)):
+ try:
+ with open(file, "rb") as f:
+ parquet_fragment = parquet_file_format.make_fragment(f)
+ if parquet_fragment.row_groups:
+ batch_size = self.config.batch_size or parquet_fragment.row_groups[0].num_rows
+ for batch_idx, record_batch in enumerate(
+ parquet_fragment.to_batches(
+ batch_size=batch_size,
+ columns=self.config.columns,
+ filter=filter_expr,
+ batch_readahead=0,
+ fragment_readahead=0,
+ )
+ ):
+ pa_table = pa.Table.from_batches([record_batch])
+ # Uncomment for debugging (will print the Arrow table size and elements)
+ # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
+ # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
+ yield f"{file_idx}_{batch_idx}", self._cast_table(pa_table)
+ except (pa.ArrowInvalid, ValueError) as e:
+ if self.config.on_bad_files == "error":
+ logger.error(f"Failed to read file '{file}' with error {type(e).__name__}: {e}")
+ raise
+ elif self.config.on_bad_files == "warn":
+ logger.warning(f"Skipping bad file '{file}'. {type(e).__name__}: {e}`")
+ else:
+ logger.debug(f"Skipping bad file '{file}'. {type(e).__name__}: {e}`")
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/pdffolder/__init__.py b/venv/lib/python3.10/site-packages/datasets/packaged_modules/pdffolder/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/pdffolder/__pycache__/__init__.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/packaged_modules/pdffolder/__pycache__/__init__.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..4ad3c26efbdb89041635eb6b054a4c2498db87f2
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/packaged_modules/pdffolder/__pycache__/__init__.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/pdffolder/__pycache__/pdffolder.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/packaged_modules/pdffolder/__pycache__/pdffolder.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..9a88dbcdf4f90a66fe70a1a5dfb2ca244fc709b9
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/packaged_modules/pdffolder/__pycache__/pdffolder.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/pdffolder/pdffolder.py b/venv/lib/python3.10/site-packages/datasets/packaged_modules/pdffolder/pdffolder.py
new file mode 100644
index 0000000000000000000000000000000000000000..5aaf9c71a4df6551a53a2625b332bc51a2f02837
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/packaged_modules/pdffolder/pdffolder.py
@@ -0,0 +1,23 @@
+import datasets
+
+from ..folder_based_builder import folder_based_builder
+
+
+logger = datasets.utils.logging.get_logger(__name__)
+
+
+class PdfFolderConfig(folder_based_builder.FolderBasedBuilderConfig):
+ """BuilderConfig for ImageFolder."""
+
+ drop_labels: bool = None
+ drop_metadata: bool = None
+
+ def __post_init__(self):
+ super().__post_init__()
+
+
+class PdfFolder(folder_based_builder.FolderBasedBuilder):
+ BASE_FEATURE = datasets.Pdf
+ BASE_COLUMN_NAME = "pdf"
+ BUILDER_CONFIG_CLASS = PdfFolderConfig
+ EXTENSIONS: list[str] = [".pdf"]
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/spark/__init__.py b/venv/lib/python3.10/site-packages/datasets/packaged_modules/spark/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/spark/__pycache__/__init__.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/packaged_modules/spark/__pycache__/__init__.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..09b1d9f7339e7640c630186257504239bd1e8d0b
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/packaged_modules/spark/__pycache__/__init__.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/spark/__pycache__/spark.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/packaged_modules/spark/__pycache__/spark.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..c1a0eb81af518557eaec5ab8e8fa8782cd7a6f72
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/packaged_modules/spark/__pycache__/spark.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/spark/spark.py b/venv/lib/python3.10/site-packages/datasets/packaged_modules/spark/spark.py
new file mode 100644
index 0000000000000000000000000000000000000000..d730bfea50277dc2c4a40604cf400f4e040c20ea
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/packaged_modules/spark/spark.py
@@ -0,0 +1,367 @@
+import os
+import posixpath
+import uuid
+from collections.abc import Iterable
+from dataclasses import dataclass
+from itertools import islice
+from typing import TYPE_CHECKING, Optional, Union
+
+import numpy as np
+import pyarrow as pa
+
+import datasets
+from datasets.arrow_writer import ArrowWriter, ParquetWriter
+from datasets.config import MAX_SHARD_SIZE
+from datasets.filesystems import (
+ is_remote_filesystem,
+ rename,
+)
+from datasets.iterable_dataset import _BaseExamplesIterable
+from datasets.utils import experimental
+from datasets.utils.py_utils import convert_file_size_to_int
+
+
+logger = datasets.utils.logging.get_logger(__name__)
+
+if TYPE_CHECKING:
+ import pyspark
+ import pyspark.sql
+
+
+@dataclass
+class SparkConfig(datasets.BuilderConfig):
+ """BuilderConfig for Spark."""
+
+ features: Optional[datasets.Features] = None
+
+ def __post_init__(self):
+ super().__post_init__()
+
+
+def _reorder_dataframe_by_partition(df: "pyspark.sql.DataFrame", new_partition_order: list[int]):
+ df_combined = df.select("*").where(f"part_id = {new_partition_order[0]}")
+ for partition_id in new_partition_order[1:]:
+ partition_df = df.select("*").where(f"part_id = {partition_id}")
+ df_combined = df_combined.union(partition_df)
+ return df_combined
+
+
+def _generate_iterable_examples(
+ df: "pyspark.sql.DataFrame",
+ partition_order: list[int],
+ state_dict: Optional[dict] = None,
+):
+ import pyspark
+
+ df_with_partition_id = df.select("*", pyspark.sql.functions.spark_partition_id().alias("part_id"))
+ partition_idx_start = state_dict["partition_idx"] if state_dict else 0
+ partition_df = _reorder_dataframe_by_partition(df_with_partition_id, partition_order[partition_idx_start:])
+ # pipeline next partition in parallel to hide latency
+ rows = partition_df.toLocalIterator(prefetchPartitions=True)
+ curr_partition = None
+ row_id = state_dict["partition_example_idx"] if state_dict else 0
+ for row in islice(rows, row_id, None):
+ row_as_dict = row.asDict()
+ part_id = row_as_dict["part_id"]
+ row_as_dict.pop("part_id")
+ if curr_partition != part_id:
+ if state_dict and curr_partition is not None:
+ state_dict["partition_idx"] += 1
+ curr_partition = part_id
+ row_id = 0
+ if state_dict:
+ state_dict["partition_example_idx"] = row_id + 1
+ yield f"{part_id}_{row_id}", row_as_dict
+ row_id += 1
+
+
+class SparkExamplesIterable(_BaseExamplesIterable):
+ def __init__(
+ self,
+ df: "pyspark.sql.DataFrame",
+ partition_order=None,
+ ):
+ super().__init__()
+ self.df = df
+ self.partition_order = partition_order or range(self.df.rdd.getNumPartitions())
+
+ def _init_state_dict(self) -> dict:
+ self._state_dict = {"partition_idx": 0, "partition_example_idx": 0}
+ return self._state_dict
+
+ @experimental
+ def load_state_dict(self, state_dict: dict) -> dict:
+ return super().load_state_dict(state_dict)
+
+ def __iter__(self):
+ yield from _generate_iterable_examples(self.df, self.partition_order, self._state_dict)
+
+ def shuffle_data_sources(self, generator: np.random.Generator) -> "SparkExamplesIterable":
+ partition_order = list(range(self.df.rdd.getNumPartitions()))
+ generator.shuffle(partition_order)
+ return SparkExamplesIterable(self.df, partition_order=partition_order)
+
+ def shard_data_sources(self, num_shards: int, index: int, contiguous=True) -> "SparkExamplesIterable":
+ partition_order = self.split_shard_indices_by_worker(num_shards=num_shards, index=index, contiguous=contiguous)
+ return SparkExamplesIterable(self.df, partition_order=partition_order)
+
+ @property
+ def num_shards(self) -> int:
+ return len(self.partition_order)
+
+
+class Spark(datasets.DatasetBuilder):
+ BUILDER_CONFIG_CLASS = SparkConfig
+
+ def __init__(
+ self,
+ df: "pyspark.sql.DataFrame",
+ cache_dir: str = None,
+ working_dir: str = None,
+ **config_kwargs,
+ ):
+ import pyspark
+
+ self._spark = pyspark.sql.SparkSession.builder.getOrCreate()
+ self.df = df
+ self._working_dir = working_dir
+
+ super().__init__(
+ cache_dir=cache_dir,
+ config_name=str(self.df.semanticHash()),
+ **config_kwargs,
+ )
+
+ def _validate_cache_dir(self):
+ # Define this so that we don't reference self in create_cache_and_write_probe, which will result in a pickling
+ # error due to pickling the SparkContext.
+ cache_dir = self._cache_dir
+
+ # Returns the path of the created file.
+ def create_cache_and_write_probe(context):
+ # makedirs with exist_ok will recursively create the directory. It will not throw an error if directories
+ # already exist.
+ os.makedirs(cache_dir, exist_ok=True)
+ probe_file = os.path.join(cache_dir, "fs_test" + uuid.uuid4().hex)
+ # Opening the file in append mode will create a new file unless it already exists, in which case it will not
+ # change the file contents.
+ open(probe_file, "a")
+ return [probe_file]
+
+ if self._spark.conf.get("spark.master", "").startswith("local"):
+ return
+
+ # If the cluster is multi-node, make sure that the user provided a cache_dir and that it is on an NFS
+ # accessible to the driver.
+ # TODO: Stream batches to the driver using ArrowCollectSerializer instead of throwing an error.
+ if self._cache_dir:
+ probe = (
+ self._spark.sparkContext.parallelize(range(1), 1).mapPartitions(create_cache_and_write_probe).collect()
+ )
+ if os.path.isfile(probe[0]):
+ return
+
+ raise ValueError(
+ "When using Dataset.from_spark on a multi-node cluster, the driver and all workers should be able to access cache_dir"
+ )
+
+ def _info(self):
+ return datasets.DatasetInfo(features=self.config.features)
+
+ def _split_generators(self, dl_manager: datasets.download.download_manager.DownloadManager):
+ return [datasets.SplitGenerator(name=datasets.Split.TRAIN)]
+
+ def _repartition_df_if_needed(self, max_shard_size):
+ import pyspark
+
+ def get_arrow_batch_size(it):
+ for batch in it:
+ yield pa.RecordBatch.from_pydict({"batch_bytes": [batch.nbytes]})
+
+ df_num_rows = self.df.count()
+ sample_num_rows = df_num_rows if df_num_rows <= 100 else 100
+ # Approximate the size of each row (in Arrow format) by averaging over a max-100-row sample.
+ approx_bytes_per_row = (
+ self.df.limit(sample_num_rows)
+ .repartition(1)
+ .mapInArrow(get_arrow_batch_size, "batch_bytes: long")
+ .agg(pyspark.sql.functions.sum("batch_bytes").alias("sample_bytes"))
+ .collect()[0]
+ .sample_bytes
+ / sample_num_rows
+ )
+ approx_total_size = approx_bytes_per_row * df_num_rows
+ if approx_total_size > max_shard_size:
+ # Make sure there is at least one row per partition.
+ new_num_partitions = min(df_num_rows, int(approx_total_size / max_shard_size))
+ self.df = self.df.repartition(new_num_partitions)
+
+ def _prepare_split_single(
+ self,
+ fpath: str,
+ file_format: str,
+ max_shard_size: int,
+ ) -> Iterable[tuple[int, bool, Union[int, tuple]]]:
+ import pyspark
+
+ writer_class = ParquetWriter if file_format == "parquet" else ArrowWriter
+ working_fpath = os.path.join(self._working_dir, os.path.basename(fpath)) if self._working_dir else fpath
+ embed_local_files = file_format == "parquet"
+
+ # Define these so that we don't reference self in write_arrow, which will result in a pickling error due to
+ # pickling the SparkContext.
+ features = self.config.features
+ writer_batch_size = self._writer_batch_size
+ storage_options = self._fs.storage_options
+
+ def write_arrow(it):
+ # Within the same SparkContext, no two task attempts will share the same attempt ID.
+ task_id = pyspark.TaskContext().taskAttemptId()
+ first_batch = next(it, None)
+ if first_batch is None:
+ # Some partitions might not receive any data.
+ return pa.RecordBatch.from_arrays(
+ [[task_id], [0], [0]],
+ names=["task_id", "num_examples", "num_bytes"],
+ )
+ shard_id = 0
+ writer = writer_class(
+ features=features,
+ path=working_fpath.replace("SSSSS", f"{shard_id:05d}").replace("TTTTT", f"{task_id:05d}"),
+ writer_batch_size=writer_batch_size,
+ storage_options=storage_options,
+ embed_local_files=embed_local_files,
+ )
+ table = pa.Table.from_batches([first_batch])
+ writer.write_table(table)
+ for batch in it:
+ if max_shard_size is not None and writer._num_bytes >= max_shard_size:
+ num_examples, num_bytes = writer.finalize()
+ writer.close()
+ yield pa.RecordBatch.from_arrays(
+ [[task_id], [num_examples], [num_bytes]],
+ names=["task_id", "num_examples", "num_bytes"],
+ )
+ shard_id += 1
+ writer = writer_class(
+ features=writer._features,
+ path=working_fpath.replace("SSSSS", f"{shard_id:05d}").replace("TTTTT", f"{task_id:05d}"),
+ writer_batch_size=writer_batch_size,
+ storage_options=storage_options,
+ embed_local_files=embed_local_files,
+ )
+ table = pa.Table.from_batches([batch])
+ writer.write_table(table)
+
+ if writer._num_bytes > 0:
+ num_examples, num_bytes = writer.finalize()
+ writer.close()
+ yield pa.RecordBatch.from_arrays(
+ [[task_id], [num_examples], [num_bytes]],
+ names=["task_id", "num_examples", "num_bytes"],
+ )
+
+ if working_fpath != fpath:
+ for file in os.listdir(os.path.dirname(working_fpath)):
+ dest = os.path.join(os.path.dirname(fpath), os.path.basename(file))
+ shutil.move(file, dest)
+
+ stats = (
+ self.df.mapInArrow(write_arrow, "task_id: long, num_examples: long, num_bytes: long")
+ .groupBy("task_id")
+ .agg(
+ pyspark.sql.functions.sum("num_examples").alias("total_num_examples"),
+ pyspark.sql.functions.sum("num_bytes").alias("total_num_bytes"),
+ pyspark.sql.functions.count("num_bytes").alias("num_shards"),
+ pyspark.sql.functions.collect_list("num_examples").alias("shard_lengths"),
+ )
+ .collect()
+ )
+ for row in stats:
+ yield row.task_id, (row.total_num_examples, row.total_num_bytes, row.num_shards, row.shard_lengths)
+
+ def _prepare_split(
+ self,
+ split_generator: "datasets.SplitGenerator",
+ file_format: str = "arrow",
+ max_shard_size: Optional[Union[str, int]] = None,
+ num_proc: Optional[int] = None,
+ **kwargs,
+ ):
+ self._validate_cache_dir()
+
+ max_shard_size = convert_file_size_to_int(max_shard_size or MAX_SHARD_SIZE)
+ self._repartition_df_if_needed(max_shard_size)
+ is_local = not is_remote_filesystem(self._fs)
+ path_join = os.path.join if is_local else posixpath.join
+
+ SUFFIX = "-TTTTT-SSSSS-of-NNNNN"
+ fname = f"{self.name}-{split_generator.name}{SUFFIX}.{file_format}"
+ fpath = path_join(self._output_dir, fname)
+
+ total_num_examples = 0
+ total_num_bytes = 0
+ total_shards = 0
+ task_id_and_num_shards = []
+ all_shard_lengths = []
+
+ for task_id, content in self._prepare_split_single(fpath, file_format, max_shard_size):
+ (
+ num_examples,
+ num_bytes,
+ num_shards,
+ shard_lengths,
+ ) = content
+ if num_bytes > 0:
+ total_num_examples += num_examples
+ total_num_bytes += num_bytes
+ total_shards += num_shards
+ task_id_and_num_shards.append((task_id, num_shards))
+ all_shard_lengths.extend(shard_lengths)
+
+ split_generator.split_info.num_examples = total_num_examples
+ split_generator.split_info.num_bytes = total_num_bytes
+
+ # should rename everything at the end
+ logger.debug(f"Renaming {total_shards} shards.")
+ if total_shards > 1:
+ split_generator.split_info.shard_lengths = all_shard_lengths
+
+ # Define fs outside of _rename_shard so that we don't reference self in the function, which will result in a
+ # pickling error due to pickling the SparkContext.
+ fs = self._fs
+
+ # use the -SSSSS-of-NNNNN pattern
+ def _rename_shard(
+ task_id: int,
+ shard_id: int,
+ global_shard_id: int,
+ ):
+ rename(
+ fs,
+ fpath.replace("SSSSS", f"{shard_id:05d}").replace("TTTTT", f"{task_id:05d}"),
+ fpath.replace("TTTTT-SSSSS", f"{global_shard_id:05d}").replace("NNNNN", f"{total_shards:05d}"),
+ )
+
+ args = []
+ global_shard_id = 0
+ for i in range(len(task_id_and_num_shards)):
+ task_id, num_shards = task_id_and_num_shards[i]
+ for shard_id in range(num_shards):
+ args.append([task_id, shard_id, global_shard_id])
+ global_shard_id += 1
+ self._spark.sparkContext.parallelize(args, len(args)).map(lambda args: _rename_shard(*args)).collect()
+ else:
+ # don't use any pattern
+ shard_id = 0
+ task_id = task_id_and_num_shards[0][0]
+ self._rename(
+ fpath.replace("SSSSS", f"{shard_id:05d}").replace("TTTTT", f"{task_id:05d}"),
+ fpath.replace(SUFFIX, ""),
+ )
+
+ def _get_examples_iterable_for_split(
+ self,
+ split_generator: "datasets.SplitGenerator",
+ ) -> SparkExamplesIterable:
+ return SparkExamplesIterable(self.df)
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/sql/__init__.py b/venv/lib/python3.10/site-packages/datasets/packaged_modules/sql/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/sql/__pycache__/__init__.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/packaged_modules/sql/__pycache__/__init__.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..467bb7ee7645cd0f6c4484cdcbad81e3f6247436
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/packaged_modules/sql/__pycache__/__init__.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/sql/__pycache__/sql.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/packaged_modules/sql/__pycache__/sql.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..c01f4a9c86d88ec6ac3aadebcf0c68e65948659f
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/packaged_modules/sql/__pycache__/sql.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/sql/sql.py b/venv/lib/python3.10/site-packages/datasets/packaged_modules/sql/sql.py
new file mode 100644
index 0000000000000000000000000000000000000000..d8e71664c8e1fde656f681249f56bb6d3ab07da8
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/packaged_modules/sql/sql.py
@@ -0,0 +1,119 @@
+import sys
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Optional, Union
+
+import pandas as pd
+import pyarrow as pa
+
+import datasets
+import datasets.config
+from datasets.features.features import require_storage_cast
+from datasets.table import table_cast
+
+
+if TYPE_CHECKING:
+ import sqlite3
+
+ import sqlalchemy
+
+
+logger = datasets.utils.logging.get_logger(__name__)
+
+
+@dataclass
+class SqlConfig(datasets.BuilderConfig):
+ """BuilderConfig for SQL."""
+
+ sql: Union[str, "sqlalchemy.sql.Selectable"] = None
+ con: Union[str, "sqlalchemy.engine.Connection", "sqlalchemy.engine.Engine", "sqlite3.Connection"] = None
+ index_col: Optional[Union[str, list[str]]] = None
+ coerce_float: bool = True
+ params: Optional[Union[list, tuple, dict]] = None
+ parse_dates: Optional[Union[list, dict]] = None
+ columns: Optional[list[str]] = None
+ chunksize: Optional[int] = 10_000
+ features: Optional[datasets.Features] = None
+
+ def __post_init__(self):
+ super().__post_init__()
+ if self.sql is None:
+ raise ValueError("sql must be specified")
+ if self.con is None:
+ raise ValueError("con must be specified")
+
+ def create_config_id(
+ self,
+ config_kwargs: dict,
+ custom_features: Optional[datasets.Features] = None,
+ ) -> str:
+ config_kwargs = config_kwargs.copy()
+ # We need to stringify the Selectable object to make its hash deterministic
+
+ # The process of stringifying is explained here: http://docs.sqlalchemy.org/en/latest/faq/sqlexpressions.html
+ sql = config_kwargs["sql"]
+ if not isinstance(sql, str):
+ if datasets.config.SQLALCHEMY_AVAILABLE and "sqlalchemy" in sys.modules:
+ import sqlalchemy
+
+ if isinstance(sql, sqlalchemy.sql.Selectable):
+ engine = sqlalchemy.create_engine(config_kwargs["con"].split("://")[0] + "://")
+ sql_str = str(sql.compile(dialect=engine.dialect))
+ config_kwargs["sql"] = sql_str
+ else:
+ raise TypeError(
+ f"Supported types for 'sql' are string and sqlalchemy.sql.Selectable but got {type(sql)}: {sql}"
+ )
+ else:
+ raise TypeError(
+ f"Supported types for 'sql' are string and sqlalchemy.sql.Selectable but got {type(sql)}: {sql}"
+ )
+ con = config_kwargs["con"]
+ if not isinstance(con, str):
+ config_kwargs["con"] = id(con)
+ logger.info(
+ f"SQL connection 'con' of type {type(con)} couldn't be hashed properly. To enable hashing, specify 'con' as URI string instead."
+ )
+
+ return super().create_config_id(config_kwargs, custom_features=custom_features)
+
+ @property
+ def pd_read_sql_kwargs(self):
+ pd_read_sql_kwargs = {
+ "index_col": self.index_col,
+ "columns": self.columns,
+ "params": self.params,
+ "coerce_float": self.coerce_float,
+ "parse_dates": self.parse_dates,
+ }
+ return pd_read_sql_kwargs
+
+
+class Sql(datasets.ArrowBasedBuilder):
+ BUILDER_CONFIG_CLASS = SqlConfig
+
+ def _info(self):
+ return datasets.DatasetInfo(features=self.config.features)
+
+ def _split_generators(self, dl_manager):
+ return [datasets.SplitGenerator(name=datasets.Split.TRAIN, gen_kwargs={})]
+
+ def _cast_table(self, pa_table: pa.Table) -> pa.Table:
+ if self.config.features is not None:
+ schema = self.config.features.arrow_schema
+ if all(not require_storage_cast(feature) for feature in self.config.features.values()):
+ # cheaper cast
+ pa_table = pa.Table.from_arrays([pa_table[field.name] for field in schema], schema=schema)
+ else:
+ # more expensive cast; allows str <-> int/float or str to Audio for example
+ pa_table = table_cast(pa_table, schema)
+ return pa_table
+
+ def _generate_tables(self):
+ chunksize = self.config.chunksize
+ sql_reader = pd.read_sql(
+ self.config.sql, self.config.con, chunksize=chunksize, **self.config.pd_read_sql_kwargs
+ )
+ sql_reader = [sql_reader] if chunksize is None else sql_reader
+ for chunk_idx, df in enumerate(sql_reader):
+ pa_table = pa.Table.from_pandas(df)
+ yield chunk_idx, self._cast_table(pa_table)
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/text/__init__.py b/venv/lib/python3.10/site-packages/datasets/packaged_modules/text/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/text/__pycache__/__init__.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/packaged_modules/text/__pycache__/__init__.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..feb379d8e6d59e34829eeed7d72ad4a913cd97b7
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/packaged_modules/text/__pycache__/__init__.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/text/__pycache__/text.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/packaged_modules/text/__pycache__/text.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..efd4ebd242d87a35cab5d22d155d57bde5fb2dae
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/packaged_modules/text/__pycache__/text.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/text/text.py b/venv/lib/python3.10/site-packages/datasets/packaged_modules/text/text.py
new file mode 100644
index 0000000000000000000000000000000000000000..a1f3ff5a744042abd6a9bbbb9ab074fa624a0541
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/packaged_modules/text/text.py
@@ -0,0 +1,112 @@
+import itertools
+from dataclasses import dataclass
+from io import StringIO
+from typing import Optional
+
+import pyarrow as pa
+
+import datasets
+from datasets.features.features import require_storage_cast
+from datasets.table import table_cast
+
+
+logger = datasets.utils.logging.get_logger(__name__)
+
+
+@dataclass
+class TextConfig(datasets.BuilderConfig):
+ """BuilderConfig for text files."""
+
+ features: Optional[datasets.Features] = None
+ encoding: str = "utf-8"
+ encoding_errors: Optional[str] = None
+ chunksize: int = 10 << 20 # 10MB
+ keep_linebreaks: bool = False
+ sample_by: str = "line"
+
+
+class Text(datasets.ArrowBasedBuilder):
+ BUILDER_CONFIG_CLASS = TextConfig
+
+ def _info(self):
+ return datasets.DatasetInfo(features=self.config.features)
+
+ def _split_generators(self, dl_manager):
+ """The `data_files` kwarg in load_dataset() can be a str, List[str], Dict[str,str], or Dict[str,List[str]].
+
+ If str or List[str], then the dataset returns only the 'train' split.
+ If dict, then keys should be from the `datasets.Split` enum.
+ """
+ if not self.config.data_files:
+ raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}")
+ dl_manager.download_config.extract_on_the_fly = True
+ data_files = dl_manager.download_and_extract(self.config.data_files)
+ splits = []
+ for split_name, files in data_files.items():
+ if isinstance(files, str):
+ files = [files]
+ files = [dl_manager.iter_files(file) for file in files]
+ splits.append(datasets.SplitGenerator(name=split_name, gen_kwargs={"files": files}))
+ return splits
+
+ def _cast_table(self, pa_table: pa.Table) -> pa.Table:
+ if self.config.features is not None:
+ schema = self.config.features.arrow_schema
+ if all(not require_storage_cast(feature) for feature in self.config.features.values()):
+ # cheaper cast
+ pa_table = pa_table.cast(schema)
+ else:
+ # more expensive cast; allows str <-> int/float or str to Audio for example
+ pa_table = table_cast(pa_table, schema)
+ return pa_table
+ else:
+ return pa_table.cast(pa.schema({"text": pa.string()}))
+
+ def _generate_tables(self, files):
+ pa_table_names = list(self.config.features) if self.config.features is not None else ["text"]
+ for file_idx, file in enumerate(itertools.chain.from_iterable(files)):
+ # open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n"
+ with open(file, encoding=self.config.encoding, errors=self.config.encoding_errors) as f:
+ if self.config.sample_by == "line":
+ batch_idx = 0
+ while True:
+ batch = f.read(self.config.chunksize)
+ if not batch:
+ break
+ batch += f.readline() # finish current line
+ # StringIO.readlines, by default splits only on "\n" (and keeps line breaks)
+ batch = StringIO(batch).readlines()
+ if not self.config.keep_linebreaks:
+ batch = [line.rstrip("\n") for line in batch]
+ pa_table = pa.Table.from_arrays([pa.array(batch)], names=pa_table_names)
+ # Uncomment for debugging (will print the Arrow table size and elements)
+ # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
+ # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
+ yield (file_idx, batch_idx), self._cast_table(pa_table)
+ batch_idx += 1
+ elif self.config.sample_by == "paragraph":
+ batch_idx = 0
+ batch = ""
+ while True:
+ new_batch = f.read(self.config.chunksize)
+ if not new_batch:
+ break
+ batch += new_batch
+ batch += f.readline() # finish current line
+ batch = batch.split("\n\n")
+ pa_table = pa.Table.from_arrays(
+ [pa.array([example for example in batch[:-1] if example])], names=pa_table_names
+ )
+ # Uncomment for debugging (will print the Arrow table size and elements)
+ # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
+ # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
+ yield (file_idx, batch_idx), self._cast_table(pa_table)
+ batch_idx += 1
+ batch = batch[-1]
+ if batch:
+ pa_table = pa.Table.from_arrays([pa.array([batch])], names=pa_table_names)
+ yield (file_idx, batch_idx), self._cast_table(pa_table)
+ elif self.config.sample_by == "document":
+ text = f.read()
+ pa_table = pa.Table.from_arrays([pa.array([text])], names=pa_table_names)
+ yield file_idx, self._cast_table(pa_table)
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/videofolder/__init__.py b/venv/lib/python3.10/site-packages/datasets/packaged_modules/videofolder/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/videofolder/__pycache__/__init__.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/packaged_modules/videofolder/__pycache__/__init__.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..fa8c5f156e05762b57ddd8a5d65dfdb1236e5322
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/packaged_modules/videofolder/__pycache__/__init__.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/videofolder/__pycache__/videofolder.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/packaged_modules/videofolder/__pycache__/videofolder.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..ac2209e80bc553bfb27f96c7f0700a0eecab662b
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/packaged_modules/videofolder/__pycache__/videofolder.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/videofolder/videofolder.py b/venv/lib/python3.10/site-packages/datasets/packaged_modules/videofolder/videofolder.py
new file mode 100644
index 0000000000000000000000000000000000000000..099a17a8d09e981b34c4a40a3f5e8c948d0efa5d
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/packaged_modules/videofolder/videofolder.py
@@ -0,0 +1,34 @@
+import datasets
+
+from ..folder_based_builder import folder_based_builder
+
+
+logger = datasets.utils.logging.get_logger(__name__)
+
+
+class VideoFolderConfig(folder_based_builder.FolderBasedBuilderConfig):
+ """BuilderConfig for ImageFolder."""
+
+ drop_labels: bool = None
+ drop_metadata: bool = None
+
+ def __post_init__(self):
+ super().__post_init__()
+
+
+class VideoFolder(folder_based_builder.FolderBasedBuilder):
+ BASE_FEATURE = datasets.Video
+ BASE_COLUMN_NAME = "video"
+ BUILDER_CONFIG_CLASS = VideoFolderConfig
+ EXTENSIONS: list[str] # definition at the bottom of the script
+
+
+# TODO: initial list, we should check the compatibility of other formats
+VIDEO_EXTENSIONS = [
+ ".mkv",
+ ".mp4",
+ ".avi",
+ ".mpeg",
+ ".mov",
+]
+VideoFolder.EXTENSIONS = VIDEO_EXTENSIONS
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/webdataset/__init__.py b/venv/lib/python3.10/site-packages/datasets/packaged_modules/webdataset/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/webdataset/__pycache__/__init__.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/packaged_modules/webdataset/__pycache__/__init__.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..7b3e3da1f2585d14544a65e6ee68a29575460c5c
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/packaged_modules/webdataset/__pycache__/__init__.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/webdataset/__pycache__/_tenbin.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/packaged_modules/webdataset/__pycache__/_tenbin.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..764e2c2fe3845643626a22dfd0e4bf2ac799721a
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/packaged_modules/webdataset/__pycache__/_tenbin.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/webdataset/__pycache__/webdataset.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/packaged_modules/webdataset/__pycache__/webdataset.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..0c5bde071668a16a58c349a53649cf552453029a
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/packaged_modules/webdataset/__pycache__/webdataset.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/webdataset/_tenbin.py b/venv/lib/python3.10/site-packages/datasets/packaged_modules/webdataset/_tenbin.py
new file mode 100644
index 0000000000000000000000000000000000000000..cd8c054842e090dc09bdf3d2fee59241a1a928c5
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/packaged_modules/webdataset/_tenbin.py
@@ -0,0 +1,285 @@
+#
+# Copyright (c) 2017-2021 NVIDIA CORPORATION. All rights reserved.
+# This file coems from the WebDataset library.
+# See the LICENSE file for licensing terms (BSD-style).
+#
+
+"""
+Binary tensor encodings for PyTorch and NumPy.
+
+This defines efficient binary encodings for tensors. The format is 8 byte
+aligned and can be used directly for computations when transmitted, say,
+via RDMA. The format is supported by WebDataset with the `.ten` filename
+extension. It is also used by Tensorcom, Tensorcom RDMA, and can be used
+for fast tensor storage with LMDB and in disk files (which can be memory
+mapped)
+
+Data is encoded as a series of chunks:
+
+- magic number (int64)
+- length in bytes (int64)
+- bytes (multiple of 64 bytes long)
+
+Arrays are a header chunk followed by a data chunk.
+Header chunks have the following structure:
+
+- dtype (int64)
+- 8 byte array name
+- ndim (int64)
+- dim[0]
+- dim[1]
+- ...
+"""
+
+import struct
+import sys
+
+import numpy as np
+
+
+def bytelen(a):
+ """Determine the length of a in bytes."""
+ if hasattr(a, "nbytes"):
+ return a.nbytes
+ elif isinstance(a, (bytearray, bytes)):
+ return len(a)
+ else:
+ raise ValueError(a, "cannot determine nbytes")
+
+
+def bytedata(a):
+ """Return a the raw data corresponding to a."""
+ if isinstance(a, (bytearray, bytes, memoryview)):
+ return a
+ elif hasattr(a, "data"):
+ return a.data
+ else:
+ raise ValueError(a, "cannot return bytedata")
+
+
+# tables for converting between long/short NumPy dtypes
+
+long_to_short = """
+float16 f2
+float32 f4
+float64 f8
+int8 i1
+int16 i2
+int32 i4
+int64 i8
+uint8 u1
+uint16 u2
+unit32 u4
+uint64 u8
+""".strip()
+long_to_short = [x.split() for x in long_to_short.split("\n")]
+long_to_short = {x[0]: x[1] for x in long_to_short}
+short_to_long = {v: k for k, v in long_to_short.items()}
+
+
+def check_acceptable_input_type(data, allow64):
+ """Check that the data has an acceptable type for tensor encoding.
+
+ :param data: array
+ :param allow64: allow 64 bit types
+ """
+ for a in data:
+ if a.dtype.name not in long_to_short:
+ raise ValueError("unsupported dataypte")
+ if not allow64 and a.dtype.name not in ["float64", "int64", "uint64"]:
+ raise ValueError("64 bit datatypes not allowed unless explicitly enabled")
+
+
+def str64(s):
+ """Convert a string to an int64."""
+ s = s + "\0" * (8 - len(s))
+ s = s.encode("ascii")
+ return struct.unpack("@q", s)[0]
+
+
+def unstr64(i):
+ """Convert an int64 to a string."""
+ b = struct.pack("@q", i)
+ return b.decode("ascii").strip("\0")
+
+
+def check_infos(data, infos, required_infos=None):
+ """Verify the info strings."""
+ if required_infos is False or required_infos is None:
+ return data
+ if required_infos is True:
+ return data, infos
+ if not isinstance(required_infos, (tuple, list)):
+ raise ValueError("required_infos must be tuple or list")
+ for required, actual in zip(required_infos, infos):
+ raise ValueError(f"actual info {actual} doesn't match required info {required}")
+ return data
+
+
+def encode_header(a, info=""):
+ """Encode an array header as a byte array."""
+ if a.ndim >= 10:
+ raise ValueError("too many dimensions")
+ if a.nbytes != np.prod(a.shape) * a.itemsize:
+ raise ValueError("mismatch between size and shape")
+ if a.dtype.name not in long_to_short:
+ raise ValueError("unsupported array type")
+ header = [str64(long_to_short[a.dtype.name]), str64(info), len(a.shape)] + list(a.shape)
+ return bytedata(np.array(header, dtype="i8"))
+
+
+def decode_header(h):
+ """Decode a byte array into an array header."""
+ h = np.frombuffer(h, dtype="i8")
+ if unstr64(h[0]) not in short_to_long:
+ raise ValueError("unsupported array type")
+ dtype = np.dtype(short_to_long[unstr64(h[0])])
+ info = unstr64(h[1])
+ rank = int(h[2])
+ shape = tuple(h[3 : 3 + rank])
+ return shape, dtype, info
+
+
+def encode_list(l, infos=None): # noqa: E741
+ """Given a list of arrays, encode them into a list of byte arrays."""
+ if infos is None:
+ infos = [""]
+ else:
+ if len(l) != len(infos):
+ raise ValueError(f"length of list {l} must muatch length of infos {infos}")
+ result = []
+ for i, a in enumerate(l):
+ header = encode_header(a, infos[i % len(infos)])
+ result += [header, bytedata(a)]
+ return result
+
+
+def decode_list(l, infos=False): # noqa: E741
+ """Given a list of byte arrays, decode them into arrays."""
+ result = []
+ infos0 = []
+ for header, data in zip(l[::2], l[1::2]):
+ shape, dtype, info = decode_header(header)
+ a = np.frombuffer(data, dtype=dtype, count=np.prod(shape)).reshape(*shape)
+ result += [a]
+ infos0 += [info]
+ return check_infos(result, infos0, infos)
+
+
+magic_str = "~TenBin~"
+magic = str64(magic_str)
+magic_bytes = unstr64(magic).encode("ascii")
+
+
+def roundup(n, k=64):
+ """Round up to the next multiple of 64."""
+ return k * ((n + k - 1) // k)
+
+
+def encode_chunks(l): # noqa: E741
+ """Encode a list of chunks into a single byte array, with lengths and magics.."""
+ size = sum(16 + roundup(b.nbytes) for b in l)
+ result = bytearray(size)
+ offset = 0
+ for b in l:
+ result[offset : offset + 8] = magic_bytes
+ offset += 8
+ result[offset : offset + 8] = struct.pack("@q", b.nbytes)
+ offset += 8
+ result[offset : offset + bytelen(b)] = b
+ offset += roundup(bytelen(b))
+ return result
+
+
+def decode_chunks(buf):
+ """Decode a byte array into a list of chunks."""
+ result = []
+ offset = 0
+ total = bytelen(buf)
+ while offset < total:
+ if magic_bytes != buf[offset : offset + 8]:
+ raise ValueError("magic bytes mismatch")
+ offset += 8
+ nbytes = struct.unpack("@q", buf[offset : offset + 8])[0]
+ offset += 8
+ b = buf[offset : offset + nbytes]
+ offset += roundup(nbytes)
+ result.append(b)
+ return result
+
+
+def encode_buffer(l, infos=None): # noqa: E741
+ """Encode a list of arrays into a single byte array."""
+ if not isinstance(l, list):
+ raise ValueError("requires list")
+ return encode_chunks(encode_list(l, infos=infos))
+
+
+def decode_buffer(buf, infos=False):
+ """Decode a byte array into a list of arrays."""
+ return decode_list(decode_chunks(buf), infos=infos)
+
+
+def write_chunk(stream, buf):
+ """Write a byte chunk to the stream with magics, length, and padding."""
+ nbytes = bytelen(buf)
+ stream.write(magic_bytes)
+ stream.write(struct.pack("@q", nbytes))
+ stream.write(bytedata(buf))
+ padding = roundup(nbytes) - nbytes
+ if padding > 0:
+ stream.write(b"\0" * padding)
+
+
+def read_chunk(stream):
+ """Read a byte chunk from a stream with magics, length, and padding."""
+ magic = stream.read(8)
+ if magic == b"":
+ return None
+ if magic != magic_bytes:
+ raise ValueError("magic number does not match")
+ nbytes = stream.read(8)
+ nbytes = struct.unpack("@q", nbytes)[0]
+ if nbytes < 0:
+ raise ValueError("negative nbytes")
+ data = stream.read(nbytes)
+ padding = roundup(nbytes) - nbytes
+ if padding > 0:
+ stream.read(padding)
+ return data
+
+
+def write(stream, l, infos=None): # noqa: E741
+ """Write a list of arrays to a stream, with magics, length, and padding."""
+ for chunk in encode_list(l, infos=infos):
+ write_chunk(stream, chunk)
+
+
+def read(stream, n=sys.maxsize, infos=False):
+ """Read a list of arrays from a stream, with magics, length, and padding."""
+ chunks = []
+ for _ in range(n):
+ header = read_chunk(stream)
+ if header is None:
+ break
+ data = read_chunk(stream)
+ if data is None:
+ raise ValueError("premature EOF")
+ chunks += [header, data]
+ return decode_list(chunks, infos=infos)
+
+
+def save(fname, *args, infos=None, nocheck=False):
+ """Save a list of arrays to a file, with magics, length, and padding."""
+ if not nocheck and not fname.endswith(".ten"):
+ raise ValueError("file name should end in .ten")
+ with open(fname, "wb") as stream:
+ write(stream, args, infos=infos)
+
+
+def load(fname, infos=False, nocheck=False):
+ """Read a list of arrays from a file, with magics, length, and padding."""
+ if not nocheck and not fname.endswith(".ten"):
+ raise ValueError("file name should end in .ten")
+ with open(fname, "rb") as stream:
+ return read(stream, infos=infos)
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/webdataset/webdataset.py b/venv/lib/python3.10/site-packages/datasets/packaged_modules/webdataset/webdataset.py
new file mode 100644
index 0000000000000000000000000000000000000000..7072fd81aea6f5063de59df0244bc7ab97da8109
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/packaged_modules/webdataset/webdataset.py
@@ -0,0 +1,347 @@
+import io
+import json
+import re
+from itertools import islice
+from typing import Any, Callable
+
+import fsspec
+import numpy as np
+import pyarrow as pa
+
+import datasets
+from datasets.features.features import cast_to_python_objects
+from datasets.utils.file_utils import SINGLE_FILE_COMPRESSION_EXTENSION_TO_PROTOCOL, xbasename
+
+
+logger = datasets.utils.logging.get_logger(__name__)
+
+
+class WebDataset(datasets.GeneratorBasedBuilder):
+ DEFAULT_WRITER_BATCH_SIZE = 100
+ IMAGE_EXTENSIONS: list[str] # definition at the bottom of the script
+ AUDIO_EXTENSIONS: list[str] # definition at the bottom of the script
+ VIDEO_EXTENSIONS: list[str] # definition at the bottom of the script
+ DECODERS: dict[str, Callable[[Any], Any]] # definition at the bottom of the script
+ NUM_EXAMPLES_FOR_FEATURES_INFERENCE = 5
+
+ @classmethod
+ def _get_pipeline_from_tar(cls, tar_path, tar_iterator):
+ current_example = {}
+ fs: fsspec.AbstractFileSystem = fsspec.filesystem("memory")
+ streaming_download_manager = datasets.StreamingDownloadManager()
+ for filename, f in tar_iterator:
+ example_key, field_name = base_plus_ext(filename)
+ if example_key is None:
+ continue
+ if current_example and current_example["__key__"] != example_key:
+ # reposition some keys in last position
+ current_example["__key__"] = current_example.pop("__key__")
+ current_example["__url__"] = current_example.pop("__url__")
+ yield current_example
+ current_example = {}
+ current_example["__key__"] = example_key
+ current_example["__url__"] = tar_path
+ current_example[field_name] = f.read()
+ if field_name.split(".")[-1].lower() in SINGLE_FILE_COMPRESSION_EXTENSION_TO_PROTOCOL:
+ fs.write_bytes(filename, current_example[field_name])
+ extracted_file_path = streaming_download_manager.extract(f"memory://{filename}")
+ with fsspec.open(extracted_file_path) as f:
+ current_example[field_name] = f.read()
+ fs.delete(filename)
+ data_extension = xbasename(extracted_file_path).split(".")[-1].lower()
+ else:
+ data_extension = field_name.split(".")[-1].lower()
+ if data_extension in cls.DECODERS:
+ current_example[field_name] = cls.DECODERS[data_extension](current_example[field_name])
+ if current_example:
+ yield current_example
+
+ def _info(self) -> datasets.DatasetInfo:
+ return datasets.DatasetInfo()
+
+ def _split_generators(self, dl_manager):
+ """We handle string, list and dicts in datafiles"""
+ # Download the data files
+ if not self.config.data_files:
+ raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}")
+ data_files = dl_manager.download(self.config.data_files)
+ splits = []
+ for split_name, tar_paths in data_files.items():
+ if isinstance(tar_paths, str):
+ tar_paths = [tar_paths]
+ tar_iterators = [dl_manager.iter_archive(tar_path) for tar_path in tar_paths]
+ splits.append(
+ datasets.SplitGenerator(
+ name=split_name, gen_kwargs={"tar_paths": tar_paths, "tar_iterators": tar_iterators}
+ )
+ )
+ if not self.info.features:
+ # Get one example to get the feature types
+ pipeline = self._get_pipeline_from_tar(tar_paths[0], tar_iterators[0])
+ first_examples = list(islice(pipeline, self.NUM_EXAMPLES_FOR_FEATURES_INFERENCE))
+ if any(example.keys() != first_examples[0].keys() for example in first_examples):
+ raise ValueError(
+ "The TAR archives of the dataset should be in WebDataset format, "
+ "but the files in the archive don't share the same prefix or the same types."
+ )
+ pa_tables = [
+ pa.Table.from_pylist(cast_to_python_objects([example], only_1d_for_numpy=True))
+ for example in first_examples
+ ]
+ inferred_arrow_schema = pa.concat_tables(pa_tables, promote_options="default").schema
+ features = datasets.Features.from_arrow_schema(inferred_arrow_schema)
+
+ for field_name in first_examples[0]:
+ extension = field_name.rsplit(".", 1)[-1].lower()
+ # Set Image types
+ if extension in self.IMAGE_EXTENSIONS:
+ features[field_name] = datasets.Image()
+ # Set Audio types
+ if extension in self.AUDIO_EXTENSIONS:
+ features[field_name] = datasets.Audio()
+ # Set Video types
+ if extension in self.VIDEO_EXTENSIONS:
+ features[field_name] = datasets.Video()
+ self.info.features = features
+
+ return splits
+
+ def _generate_examples(self, tar_paths, tar_iterators):
+ image_field_names = [
+ field_name for field_name, feature in self.info.features.items() if isinstance(feature, datasets.Image)
+ ]
+ audio_field_names = [
+ field_name for field_name, feature in self.info.features.items() if isinstance(feature, datasets.Audio)
+ ]
+ all_field_names = list(self.info.features.keys())
+ for tar_idx, (tar_path, tar_iterator) in enumerate(zip(tar_paths, tar_iterators)):
+ for example_idx, example in enumerate(self._get_pipeline_from_tar(tar_path, tar_iterator)):
+ for field_name in all_field_names:
+ if field_name not in example:
+ example[field_name] = None
+ for field_name in image_field_names + audio_field_names:
+ if example[field_name] is not None:
+ example[field_name] = {
+ "path": example["__key__"] + "." + field_name,
+ "bytes": example[field_name],
+ }
+ yield f"{tar_idx}_{example_idx}", example
+
+
+# Source: https://github.com/webdataset/webdataset/blob/87bd5aa41602d57f070f65a670893ee625702f2f/webdataset/tariterators.py#L25
+def base_plus_ext(path):
+ """Split off all file extensions.
+
+ Returns base, allext.
+ """
+ match = re.match(r"^((?:.*/|)[^.]+)[.]([^/]*)$", path)
+ if not match:
+ return None, None
+ return match.group(1), match.group(2)
+
+
+# Obtained with:
+# ```
+# import PIL.Image
+# IMAGE_EXTENSIONS = []
+# PIL.Image.init()
+# for ext, format in PIL.Image.EXTENSION.items():
+# if format in PIL.Image.OPEN:
+# IMAGE_EXTENSIONS.append(ext[1:])
+# ```
+# We intentionally do not run this code on launch because:
+# (1) Pillow is an optional dependency, so importing Pillow in global namespace is not allowed
+# (2) To ensure the list of supported extensions is deterministic
+IMAGE_EXTENSIONS = [
+ "blp",
+ "bmp",
+ "dib",
+ "bufr",
+ "cur",
+ "pcx",
+ "dcx",
+ "dds",
+ "ps",
+ "eps",
+ "fit",
+ "fits",
+ "fli",
+ "flc",
+ "ftc",
+ "ftu",
+ "gbr",
+ "gif",
+ "grib",
+ "h5",
+ "hdf",
+ "png",
+ "apng",
+ "jp2",
+ "j2k",
+ "jpc",
+ "jpf",
+ "jpx",
+ "j2c",
+ "icns",
+ "ico",
+ "im",
+ "iim",
+ "tif",
+ "tiff",
+ "jfif",
+ "jpe",
+ "jpg",
+ "jpeg",
+ "mpg",
+ "mpeg",
+ "msp",
+ "pcd",
+ "pxr",
+ "pbm",
+ "pgm",
+ "ppm",
+ "pnm",
+ "psd",
+ "bw",
+ "rgb",
+ "rgba",
+ "sgi",
+ "ras",
+ "tga",
+ "icb",
+ "vda",
+ "vst",
+ "webp",
+ "wmf",
+ "emf",
+ "xbm",
+ "xpm",
+]
+WebDataset.IMAGE_EXTENSIONS = IMAGE_EXTENSIONS
+
+
+# Obtained with:
+# ```
+# import soundfile as sf
+#
+# AUDIO_EXTENSIONS = [f".{format.lower()}" for format in sf.available_formats().keys()]
+#
+# # .opus decoding is supported if libsndfile >= 1.0.31:
+# AUDIO_EXTENSIONS.extend([".mp3", ".opus"])
+# ```
+# We intentionally did not run this code on launch because:
+# (1) Soundfile was an optional dependency, so importing it in global namespace is not allowed
+# (2) To ensure the list of supported extensions is deterministic
+# (3) We use TorchCodec now anyways instead of Soundfile
+AUDIO_EXTENSIONS = [
+ "aiff",
+ "au",
+ "avr",
+ "caf",
+ "flac",
+ "htk",
+ "svx",
+ "mat4",
+ "mat5",
+ "mpc2k",
+ "ogg",
+ "paf",
+ "pvf",
+ "raw",
+ "rf64",
+ "sd2",
+ "sds",
+ "ircam",
+ "voc",
+ "w64",
+ "wav",
+ "nist",
+ "wavex",
+ "wve",
+ "xi",
+ "mp3",
+ "opus",
+]
+WebDataset.AUDIO_EXTENSIONS = AUDIO_EXTENSIONS
+
+
+# TODO: initial list, we should check the compatibility of other formats
+VIDEO_EXTENSIONS = [
+ ".mkv",
+ ".mp4",
+ ".avi",
+ ".mpeg",
+ ".mov",
+]
+WebDataset.VIDEO_EXTENSIONS = VIDEO_EXTENSIONS
+
+
+def text_loads(data: bytes):
+ return data.decode("utf-8")
+
+
+def tenbin_loads(data: bytes):
+ from . import _tenbin
+
+ return _tenbin.decode_buffer(data)
+
+
+def msgpack_loads(data: bytes):
+ import msgpack
+
+ return msgpack.unpackb(data)
+
+
+def npy_loads(data: bytes):
+ import numpy.lib.format
+
+ stream = io.BytesIO(data)
+ return numpy.lib.format.read_array(stream, allow_pickle=False)
+
+
+def npz_loads(data: bytes):
+ return np.load(io.BytesIO(data), allow_pickle=False)
+
+
+def cbor_loads(data: bytes):
+ import cbor
+
+ return cbor.loads(data)
+
+
+def torch_loads(data: bytes):
+ import torch
+
+ return torch.load(io.BytesIO(data), weights_only=True)
+
+
+# Obtained by checking `decoders` in `webdataset.autodecode`
+# and removing unsafe extension decoders.
+# Removed Pickle decoders:
+# - "pyd": lambda data: pickle.loads(data)
+# - "pickle": lambda data: pickle.loads(data)
+# Modified NumPy decoders to fix CVE-2019-6446 (add allow_pickle=False and weights_only=True):
+# - "npy": npy_loads,
+# - "npz": lambda data: np.load(io.BytesIO(data)),
+# - "pth": lambda data: torch_loads(data)
+DECODERS = {
+ "txt": text_loads,
+ "text": text_loads,
+ "transcript": text_loads,
+ "cls": int,
+ "cls2": int,
+ "index": int,
+ "inx": int,
+ "id": int,
+ "json": json.loads,
+ "jsn": json.loads,
+ "ten": tenbin_loads,
+ "tb": tenbin_loads,
+ "mp": msgpack_loads,
+ "msg": msgpack_loads,
+ "npy": npy_loads,
+ "npz": npz_loads,
+ "cbor": cbor_loads,
+ "pth": torch_loads,
+}
+WebDataset.DECODERS = DECODERS
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/xml/__init__.py b/venv/lib/python3.10/site-packages/datasets/packaged_modules/xml/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/xml/__pycache__/__init__.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/packaged_modules/xml/__pycache__/__init__.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..fc974497c3f035efa8a18e0eb6245c812abf031d
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/packaged_modules/xml/__pycache__/__init__.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/xml/__pycache__/xml.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/packaged_modules/xml/__pycache__/xml.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..b1d32f525f4ec55e7d69ba48510fa45778f2b3c8
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/packaged_modules/xml/__pycache__/xml.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/packaged_modules/xml/xml.py b/venv/lib/python3.10/site-packages/datasets/packaged_modules/xml/xml.py
new file mode 100644
index 0000000000000000000000000000000000000000..d5009b4dd6ad3d7936eb7b96c4c0ecaf207e5915
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/packaged_modules/xml/xml.py
@@ -0,0 +1,68 @@
+import itertools
+from dataclasses import dataclass
+from typing import Optional
+
+import pyarrow as pa
+
+import datasets
+from datasets.features.features import require_storage_cast
+from datasets.table import table_cast
+
+
+logger = datasets.utils.logging.get_logger(__name__)
+
+
+@dataclass
+class XmlConfig(datasets.BuilderConfig):
+ """BuilderConfig for xml files."""
+
+ features: Optional[datasets.Features] = None
+ encoding: str = "utf-8"
+ encoding_errors: Optional[str] = None
+
+
+class Xml(datasets.ArrowBasedBuilder):
+ BUILDER_CONFIG_CLASS = XmlConfig
+
+ def _info(self):
+ return datasets.DatasetInfo(features=self.config.features)
+
+ def _split_generators(self, dl_manager):
+ """The `data_files` kwarg in load_dataset() can be a str, List[str], Dict[str,str], or Dict[str,List[str]].
+
+ If str or List[str], then the dataset returns only the 'train' split.
+ If dict, then keys should be from the `datasets.Split` enum.
+ """
+ if not self.config.data_files:
+ raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}")
+ dl_manager.download_config.extract_on_the_fly = True
+ data_files = dl_manager.download_and_extract(self.config.data_files)
+ splits = []
+ for split_name, files in data_files.items():
+ if isinstance(files, str):
+ files = [files]
+ files = [dl_manager.iter_files(file) for file in files]
+ splits.append(datasets.SplitGenerator(name=split_name, gen_kwargs={"files": files}))
+ return splits
+
+ def _cast_table(self, pa_table: pa.Table) -> pa.Table:
+ if self.config.features is not None:
+ schema = self.config.features.arrow_schema
+ if all(not require_storage_cast(feature) for feature in self.config.features.values()):
+ # cheaper cast
+ pa_table = pa_table.cast(schema)
+ else:
+ # more expensive cast; allows str <-> int/float or str to Audio for example
+ pa_table = table_cast(pa_table, schema)
+ return pa_table
+ else:
+ return pa_table.cast(pa.schema({"xml": pa.string()}))
+
+ def _generate_tables(self, files):
+ pa_table_names = list(self.config.features) if self.config.features is not None else ["xml"]
+ for file_idx, file in enumerate(itertools.chain.from_iterable(files)):
+ # open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n"
+ with open(file, encoding=self.config.encoding, errors=self.config.encoding_errors) as f:
+ xml = f.read()
+ pa_table = pa.Table.from_arrays([pa.array([xml])], names=pa_table_names)
+ yield file_idx, self._cast_table(pa_table)
diff --git a/venv/lib/python3.10/site-packages/datasets/parallel/__init__.py b/venv/lib/python3.10/site-packages/datasets/parallel/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..a8d527f4d09946b3b7b7a65ec22afa953ddfd315
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/parallel/__init__.py
@@ -0,0 +1 @@
+from .parallel import ParallelBackendConfig, parallel_backend, parallel_map
diff --git a/venv/lib/python3.10/site-packages/datasets/parallel/__pycache__/__init__.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/parallel/__pycache__/__init__.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..7c8c2f4e0f1bfd00a62b993b47bc37c6ea4f4534
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/parallel/__pycache__/__init__.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/parallel/__pycache__/parallel.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/parallel/__pycache__/parallel.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..5152b441d5c3440e531ab0b161133392371a1d90
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/parallel/__pycache__/parallel.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/parallel/parallel.py b/venv/lib/python3.10/site-packages/datasets/parallel/parallel.py
new file mode 100644
index 0000000000000000000000000000000000000000..5cad2c48ba2b0a6e71c62babd4eb052a3dd1dfd4
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/parallel/parallel.py
@@ -0,0 +1,120 @@
+import contextlib
+from multiprocessing import Pool, RLock
+
+from tqdm.auto import tqdm
+
+from ..utils import experimental, logging
+
+
+logger = logging.get_logger(__name__)
+
+
+class ParallelBackendConfig:
+ backend_name = None
+
+
+@experimental
+def parallel_map(function, iterable, num_proc, batched, batch_size, types, disable_tqdm, desc, single_map_nested_func):
+ """
+ **Experimental.** Apply a function to iterable elements in parallel, where the implementation uses either
+ multiprocessing.Pool or joblib for parallelization.
+
+ Args:
+ function (`Callable[[Any], Any]`): Function to be applied to `iterable`.
+ iterable (`list`, `tuple` or `np.ndarray`): Iterable elements to apply function to.
+ num_proc (`int`): Number of processes (if no backend specified) or jobs (using joblib).
+ types (`tuple`): Additional types (besides `dict` values) to apply `function` recursively to their elements.
+ disable_tqdm (`bool`): Whether to disable the tqdm progressbar.
+ desc (`str`): Prefix for the tqdm progressbar.
+ single_map_nested_func (`Callable`): Map function that applies `function` to an element from `iterable`.
+ Takes a tuple of function, data_struct, types, rank, disable_tqdm, desc as input, where data_struct is an
+ element of `iterable`, and `rank` is used for progress bar.
+ """
+ if ParallelBackendConfig.backend_name is None:
+ return _map_with_multiprocessing_pool(
+ function, iterable, num_proc, batched, batch_size, types, disable_tqdm, desc, single_map_nested_func
+ )
+
+ return _map_with_joblib(
+ function, iterable, num_proc, batched, batch_size, types, disable_tqdm, desc, single_map_nested_func
+ )
+
+
+def _map_with_multiprocessing_pool(
+ function, iterable, num_proc, batched, batch_size, types, disable_tqdm, desc, single_map_nested_func
+):
+ num_proc = num_proc if num_proc <= len(iterable) else len(iterable)
+ split_kwds = [] # We organize the splits ourselve (contiguous splits)
+ for index in range(num_proc):
+ div = len(iterable) // num_proc
+ mod = len(iterable) % num_proc
+ start = div * index + min(index, mod)
+ end = start + div + (1 if index < mod else 0)
+ split_kwds.append((function, iterable[start:end], batched, batch_size, types, index, disable_tqdm, desc))
+
+ if len(iterable) != sum(len(i[1]) for i in split_kwds):
+ raise ValueError(
+ f"Error dividing inputs iterable among processes. "
+ f"Total number of objects {len(iterable)}, "
+ f"length: {sum(len(i[1]) for i in split_kwds)}"
+ )
+
+ logger.info(
+ f"Spawning {num_proc} processes for {len(iterable)} objects in slices of {[len(i[1]) for i in split_kwds]}"
+ )
+ initargs, initializer = None, None
+ if not disable_tqdm:
+ initargs, initializer = (RLock(),), tqdm.set_lock
+ with Pool(num_proc, initargs=initargs, initializer=initializer) as pool:
+ mapped = pool.map(single_map_nested_func, split_kwds)
+ logger.info(f"Finished {num_proc} processes")
+ mapped = [obj for proc_res in mapped for obj in proc_res]
+ logger.info(f"Unpacked {len(mapped)} objects")
+
+ return mapped
+
+
+def _map_with_joblib(
+ function, iterable, num_proc, batched, batch_size, types, disable_tqdm, desc, single_map_nested_func
+):
+ # progress bar is not yet supported for _map_with_joblib, because tqdm couldn't accurately be applied to joblib,
+ # and it requires monkey-patching joblib internal classes which is subject to change
+ import joblib
+
+ with joblib.parallel_backend(ParallelBackendConfig.backend_name, n_jobs=num_proc):
+ return joblib.Parallel()(
+ joblib.delayed(single_map_nested_func)((function, obj, batched, batch_size, types, None, True, None))
+ for obj in iterable
+ )
+
+
+@experimental
+@contextlib.contextmanager
+def parallel_backend(backend_name: str):
+ """
+ **Experimental.** Configures the parallel backend for parallelized dataset loading, which uses the parallelization
+ implemented by joblib.
+
+ Args:
+ backend_name (str): Name of backend for parallelization implementation, has to be supported by joblib.
+
+ Example usage:
+ ```py
+ with parallel_backend('spark'):
+ dataset = load_dataset(..., num_proc=2)
+ ```
+ """
+ ParallelBackendConfig.backend_name = backend_name
+
+ if backend_name == "spark":
+ from joblibspark import register_spark
+
+ register_spark()
+
+ # TODO: call create_cache_and_write_probe if "download" in steps
+ # TODO: raise NotImplementedError when Dataset.map etc is called
+
+ try:
+ yield
+ finally:
+ ParallelBackendConfig.backend_name = None
diff --git a/venv/lib/python3.10/site-packages/datasets/search.py b/venv/lib/python3.10/site-packages/datasets/search.py
new file mode 100644
index 0000000000000000000000000000000000000000..cb994d24e41e96174386de79057441e4436cc19a
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/search.py
@@ -0,0 +1,785 @@
+import importlib.util
+import os
+import tempfile
+from pathlib import PurePath
+from typing import TYPE_CHECKING, NamedTuple, Optional, Union
+
+import fsspec
+import numpy as np
+
+from .features import List
+from .utils import logging
+from .utils import tqdm as hf_tqdm
+
+
+if TYPE_CHECKING:
+ from .arrow_dataset import Dataset # noqa: F401
+
+ try:
+ from elasticsearch import Elasticsearch # noqa: F401
+
+ except ImportError:
+ pass
+ try:
+ import faiss # noqa: F401
+
+ except ImportError:
+ pass
+
+_has_elasticsearch = importlib.util.find_spec("elasticsearch") is not None
+_has_faiss = importlib.util.find_spec("faiss") is not None
+
+
+logger = logging.get_logger(__name__)
+
+
+class MissingIndex(Exception):
+ pass
+
+
+class SearchResults(NamedTuple):
+ scores: list[float]
+ indices: list[int]
+
+
+class BatchedSearchResults(NamedTuple):
+ total_scores: list[list[float]]
+ total_indices: list[list[int]]
+
+
+class NearestExamplesResults(NamedTuple):
+ scores: list[float]
+ examples: dict
+
+
+class BatchedNearestExamplesResults(NamedTuple):
+ total_scores: list[list[float]]
+ total_examples: list[dict]
+
+
+class BaseIndex:
+ """Base class for indexing"""
+
+ def search(self, query, k: int = 10, **kwargs) -> SearchResults:
+ """
+ To implement.
+ This method has to return the scores and the indices of the retrieved examples given a certain query.
+ """
+ raise NotImplementedError
+
+ def search_batch(self, queries, k: int = 10, **kwargs) -> BatchedSearchResults:
+ """Find the nearest examples indices to the query.
+
+ Args:
+ queries (`Union[List[str], np.ndarray]`): The queries as a list of strings if `column` is a text index or as a numpy array if `column` is a vector index.
+ k (`int`): The number of examples to retrieve per query.
+
+ Output:
+ total_scores (`List[List[float]`): The retrieval scores of the retrieved examples per query.
+ total_indices (`List[List[int]]`): The indices of the retrieved examples per query.
+ """
+ total_scores, total_indices = [], []
+ for query in queries:
+ scores, indices = self.search(query, k)
+ total_scores.append(scores)
+ total_indices.append(indices)
+ return BatchedSearchResults(total_scores, total_indices)
+
+ def save(self, file: Union[str, PurePath]):
+ """Serialize the index on disk"""
+ raise NotImplementedError
+
+ @classmethod
+ def load(cls, file: Union[str, PurePath]) -> "BaseIndex":
+ """Deserialize the index from disk"""
+ raise NotImplementedError
+
+
+class ElasticSearchIndex(BaseIndex):
+ """
+ Sparse index using Elasticsearch. It is used to index text and run queries based on BM25 similarity.
+ An Elasticsearch server needs to be accessible, and a python client is declared with
+ ```
+ es_client = Elasticsearch([{'host': 'localhost', 'port': '9200'}])
+ ```
+ for example.
+ """
+
+ def __init__(
+ self,
+ host: Optional[str] = None,
+ port: Optional[int] = None,
+ es_client: Optional["Elasticsearch"] = None,
+ es_index_name: Optional[str] = None,
+ es_index_config: Optional[dict] = None,
+ ):
+ if not _has_elasticsearch:
+ raise ImportError(
+ "You must install ElasticSearch to use ElasticSearchIndex. To do so you can run `pip install elasticsearch==7.7.1 for example`"
+ )
+ if es_client is not None and (host is not None or port is not None):
+ raise ValueError("Please specify either `es_client` or `(host, port)`, but not both.")
+ host = host or "localhost"
+ port = port or 9200
+
+ import elasticsearch.helpers # noqa: F401 - need this to properly load all the es features
+ from elasticsearch import Elasticsearch # noqa: F811
+
+ self.es_client = es_client if es_client is not None else Elasticsearch([{"host": host, "port": str(port)}])
+ self.es_index_name = (
+ es_index_name
+ if es_index_name is not None
+ else "huggingface_datasets_" + os.path.basename(tempfile.NamedTemporaryFile().name)
+ )
+ self.es_index_config = (
+ es_index_config
+ if es_index_config is not None
+ else {
+ "settings": {
+ "number_of_shards": 1,
+ "analysis": {"analyzer": {"stop_standard": {"type": "standard", " stopwords": "_english_"}}},
+ },
+ "mappings": {"properties": {"text": {"type": "text", "analyzer": "standard", "similarity": "BM25"}}},
+ }
+ )
+
+ def add_documents(self, documents: Union[list[str], "Dataset"], column: Optional[str] = None):
+ """
+ Add documents to the index.
+ If the documents are inside a certain column, you can specify it using the `column` argument.
+ """
+ index_name = self.es_index_name
+ index_config = self.es_index_config
+ self.es_client.indices.create(index=index_name, body=index_config)
+ number_of_docs = len(documents)
+ progress = hf_tqdm(unit="docs", total=number_of_docs)
+ successes = 0
+
+ def passage_generator():
+ if column is not None:
+ for i, example in enumerate(documents):
+ yield {"text": example[column], "_id": i}
+ else:
+ for i, example in enumerate(documents):
+ yield {"text": example, "_id": i}
+
+ # create the ES index
+ import elasticsearch as es
+
+ for ok, action in es.helpers.streaming_bulk(
+ client=self.es_client,
+ index=index_name,
+ actions=passage_generator(),
+ ):
+ progress.update(1)
+ successes += ok
+ if successes != len(documents):
+ logger.warning(
+ f"Some documents failed to be added to ElasticSearch. Failures: {len(documents) - successes}/{len(documents)}"
+ )
+ logger.info(f"Indexed {successes:d} documents")
+
+ def search(self, query: str, k=10, **kwargs) -> SearchResults:
+ """Find the nearest examples indices to the query.
+
+ Args:
+ query (`str`): The query as a string.
+ k (`int`): The number of examples to retrieve.
+
+ Output:
+ scores (`List[List[float]`): The retrieval scores of the retrieved examples.
+ indices (`List[List[int]]`): The indices of the retrieved examples.
+ """
+ response = self.es_client.search(
+ index=self.es_index_name,
+ body={"query": {"multi_match": {"query": query, "fields": ["text"], "type": "cross_fields"}}, "size": k},
+ **kwargs,
+ )
+ hits = response["hits"]["hits"]
+ return SearchResults([hit["_score"] for hit in hits], [int(hit["_id"]) for hit in hits])
+
+ def search_batch(self, queries, k: int = 10, max_workers=10, **kwargs) -> BatchedSearchResults:
+ import concurrent.futures
+
+ total_scores, total_indices = [None] * len(queries), [None] * len(queries)
+ with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
+ future_to_index = {executor.submit(self.search, query, k, **kwargs): i for i, query in enumerate(queries)}
+ for future in concurrent.futures.as_completed(future_to_index):
+ index = future_to_index[future]
+ results: SearchResults = future.result()
+ total_scores[index] = results.scores
+ total_indices[index] = results.indices
+ return BatchedSearchResults(total_indices=total_indices, total_scores=total_scores)
+
+
+class FaissIndex(BaseIndex):
+ """
+ Dense index using Faiss. It is used to index vectors.
+ Faiss is a library for efficient similarity search and clustering of dense vectors.
+ It contains algorithms that search in sets of vectors of any size, up to ones that possibly do not fit in RAM.
+ You can find more information about Faiss here:
+ - For index types and the string factory: https://github.com/facebookresearch/faiss/wiki/The-index-factory
+ - For GPU settings: https://github.com/facebookresearch/faiss/wiki/Faiss-on-the-GPU
+ """
+
+ def __init__(
+ self,
+ device: Optional[Union[int, list[int]]] = None,
+ string_factory: Optional[str] = None,
+ metric_type: Optional[int] = None,
+ custom_index: Optional["faiss.Index"] = None,
+ ):
+ """
+ Create a Dense index using Faiss. You can specify `device` if you want to run it on GPU (`device` must be the GPU index).
+ You can find more information about Faiss here:
+ - For `string factory`: https://github.com/facebookresearch/faiss/wiki/The-index-factory
+ """
+ if string_factory is not None and custom_index is not None:
+ raise ValueError("Please specify either `string_factory` or `custom_index` but not both.")
+ if device is not None and custom_index is not None:
+ raise ValueError(
+ "Cannot pass both 'custom_index' and 'device'. "
+ "Pass 'custom_index' already transferred to the target device instead."
+ )
+ self.device = device
+ self.string_factory = string_factory
+ self.metric_type = metric_type
+ self.faiss_index = custom_index
+ if not _has_faiss:
+ raise ImportError(
+ "You must install Faiss to use FaissIndex. To do so you can run `conda install -c pytorch faiss-cpu` or `conda install -c pytorch faiss-gpu`. "
+ "A community supported package is also available on pypi: `pip install faiss-cpu` or `pip install faiss-gpu`. "
+ "Note that pip may not have the latest version of FAISS, and thus, some of the latest features and bug fixes may not be available."
+ )
+
+ def add_vectors(
+ self,
+ vectors: Union[np.array, "Dataset"],
+ column: Optional[str] = None,
+ batch_size: int = 1000,
+ train_size: Optional[int] = None,
+ faiss_verbose: Optional[bool] = None,
+ ):
+ """
+ Add vectors to the index.
+ If the arrays are inside a certain column, you can specify it using the `column` argument.
+ """
+ import faiss # noqa: F811
+
+ if column and not isinstance(vectors.features[column], List):
+ raise ValueError(
+ f"Wrong feature type for column '{column}'. Expected 1d array, got {vectors.features[column]}"
+ )
+
+ # Create index
+ if self.faiss_index is None:
+ size = len(vectors[0]) if column is None else len(vectors[0][column])
+ if self.string_factory is not None:
+ if self.metric_type is None:
+ index = faiss.index_factory(size, self.string_factory)
+ else:
+ index = faiss.index_factory(size, self.string_factory, self.metric_type)
+ else:
+ if self.metric_type is None:
+ index = faiss.IndexFlat(size)
+ else:
+ index = faiss.IndexFlat(size, self.metric_type)
+
+ self.faiss_index = self._faiss_index_to_device(index, self.device)
+ logger.info(f"Created faiss index of type {type(self.faiss_index)}")
+
+ # Set verbosity level
+ if faiss_verbose is not None:
+ self.faiss_index.verbose = faiss_verbose
+ if hasattr(self.faiss_index, "index") and self.faiss_index.index is not None:
+ self.faiss_index.index.verbose = faiss_verbose
+ if hasattr(self.faiss_index, "quantizer") and self.faiss_index.quantizer is not None:
+ self.faiss_index.quantizer.verbose = faiss_verbose
+ if hasattr(self.faiss_index, "clustering_index") and self.faiss_index.clustering_index is not None:
+ self.faiss_index.clustering_index.verbose = faiss_verbose
+
+ # Train
+ if train_size is not None:
+ train_vecs = vectors[:train_size] if column is None else vectors[:train_size][column]
+ logger.info(f"Training the index with the first {len(train_vecs)} vectors")
+ self.faiss_index.train(train_vecs)
+ else:
+ logger.info("Ignored the training step of the faiss index as `train_size` is None.")
+
+ # Add vectors
+ logger.info(f"Adding {len(vectors)} vectors to the faiss index")
+ for i in hf_tqdm(range(0, len(vectors), batch_size)):
+ vecs = vectors[i : i + batch_size] if column is None else vectors[i : i + batch_size][column]
+ self.faiss_index.add(vecs)
+
+ @staticmethod
+ def _faiss_index_to_device(index: "faiss.Index", device: Optional[Union[int, list[int]]] = None) -> "faiss.Index":
+ """
+ Sends a faiss index to a device.
+ A device can either be a positive integer (GPU id), a negative integer (all GPUs),
+ or a list of positive integers (select GPUs to use), or `None` for CPU.
+ """
+
+ # If device is not specified, then it runs on CPU.
+ if device is None:
+ return index
+
+ import faiss # noqa: F811
+
+ # If the device id is given as an integer
+ if isinstance(device, int):
+ # Positive integers are directly mapped to GPU ids
+ if device > -1:
+ faiss_res = faiss.StandardGpuResources()
+ index = faiss.index_cpu_to_gpu(faiss_res, device, index)
+ # And negative integers mean using all GPUs
+ else:
+ index = faiss.index_cpu_to_all_gpus(index)
+ # Device ids given as a list mean mapping to those devices specified.
+ elif isinstance(device, (list, tuple)):
+ index = faiss.index_cpu_to_gpus_list(index, gpus=list(device))
+ else:
+ raise TypeError(
+ f"The argument type: {type(device)} is not expected. "
+ + "Please pass in either nothing, a positive int, a negative int, or a list of positive ints."
+ )
+
+ return index
+
+ def search(self, query: np.array, k=10, **kwargs) -> SearchResults:
+ """Find the nearest examples indices to the query.
+
+ Args:
+ query (`np.array`): The query as a numpy array.
+ k (`int`): The number of examples to retrieve.
+
+ Output:
+ scores (`List[List[float]`): The retrieval scores of the retrieved examples.
+ indices (`List[List[int]]`): The indices of the retrieved examples.
+ """
+ if len(query.shape) != 1 and (len(query.shape) != 2 or query.shape[0] != 1):
+ raise ValueError("Shape of query is incorrect, it has to be either a 1D array or 2D (1, N)")
+
+ queries = query.reshape(1, -1)
+ if not queries.flags.c_contiguous:
+ queries = np.asarray(queries, order="C")
+ scores, indices = self.faiss_index.search(queries, k, **kwargs)
+ return SearchResults(scores[0], indices[0].astype(int))
+
+ def search_batch(self, queries: np.array, k=10, **kwargs) -> BatchedSearchResults:
+ """Find the nearest examples indices to the queries.
+
+ Args:
+ queries (`np.array`): The queries as a numpy array.
+ k (`int`): The number of examples to retrieve.
+
+ Output:
+ total_scores (`List[List[float]`): The retrieval scores of the retrieved examples per query.
+ total_indices (`List[List[int]]`): The indices of the retrieved examples per query.
+ """
+ if len(queries.shape) != 2:
+ raise ValueError("Shape of query must be 2D")
+ if not queries.flags.c_contiguous:
+ queries = np.asarray(queries, order="C")
+ scores, indices = self.faiss_index.search(queries, k, **kwargs)
+ return BatchedSearchResults(scores, indices.astype(int))
+
+ def save(self, file: Union[str, PurePath], storage_options: Optional[dict] = None):
+ """Serialize the FaissIndex on disk"""
+ import faiss # noqa: F811
+
+ if self.device is not None and isinstance(self.device, (int, list, tuple)):
+ index = faiss.index_gpu_to_cpu(self.faiss_index)
+ else:
+ index = self.faiss_index
+
+ with fsspec.open(str(file), "wb", **(storage_options or {})) as f:
+ faiss.write_index(index, faiss.BufferedIOWriter(faiss.PyCallbackIOWriter(f.write)))
+
+ @classmethod
+ def load(
+ cls,
+ file: Union[str, PurePath],
+ device: Optional[Union[int, list[int]]] = None,
+ storage_options: Optional[dict] = None,
+ ) -> "FaissIndex":
+ """Deserialize the FaissIndex from disk"""
+ import faiss # noqa: F811
+
+ # Instances of FaissIndex is essentially just a wrapper for faiss indices.
+ faiss_index = cls(device=device)
+ with fsspec.open(str(file), "rb", **(storage_options or {})) as f:
+ index = faiss.read_index(faiss.BufferedIOReader(faiss.PyCallbackIOReader(f.read)))
+ faiss_index.faiss_index = faiss_index._faiss_index_to_device(index, faiss_index.device)
+ return faiss_index
+
+
+class IndexableMixin:
+ """Add indexing features to `datasets.Dataset`"""
+
+ def __init__(self):
+ self._indexes: dict[str, BaseIndex] = {}
+
+ def __len__(self):
+ raise NotImplementedError
+
+ def __getitem__(self, key):
+ raise NotImplementedError
+
+ def is_index_initialized(self, index_name: str) -> bool:
+ return index_name in self._indexes
+
+ def _check_index_is_initialized(self, index_name: str):
+ if not self.is_index_initialized(index_name):
+ raise MissingIndex(
+ f"Index with index_name '{index_name}' not initialized yet. Please make sure that you call `add_faiss_index` or `add_elasticsearch_index` first."
+ )
+
+ def list_indexes(self) -> list[str]:
+ """List the `colindex_nameumns`/identifiers of all the attached indexes."""
+ return list(self._indexes)
+
+ def get_index(self, index_name: str) -> BaseIndex:
+ """List the `index_name`/identifiers of all the attached indexes.
+
+ Args:
+ index_name (`str`): Index name.
+
+ Returns:
+ [`BaseIndex`]
+ """
+ self._check_index_is_initialized(index_name)
+ return self._indexes[index_name]
+
+ def add_faiss_index(
+ self,
+ column: str,
+ index_name: Optional[str] = None,
+ device: Optional[Union[int, list[int]]] = None,
+ string_factory: Optional[str] = None,
+ metric_type: Optional[int] = None,
+ custom_index: Optional["faiss.Index"] = None,
+ batch_size: int = 1000,
+ train_size: Optional[int] = None,
+ faiss_verbose: bool = False,
+ ):
+ """Add a dense index using Faiss for fast retrieval.
+ The index is created using the vectors of the specified column.
+ You can specify `device` if you want to run it on GPU (`device` must be the GPU index, see more below).
+ You can find more information about Faiss here:
+ - For `string factory`: https://github.com/facebookresearch/faiss/wiki/The-index-factory
+
+ Args:
+ column (`str`): The column of the vectors to add to the index.
+ index_name (Optional `str`): The index_name/identifier of the index. This is the index_name that is used to call `.get_nearest` or `.search`.
+ By default it corresponds to `column`.
+ device (Optional `Union[int, List[int]]`): If positive integer, this is the index of the GPU to use. If negative integer, use all GPUs.
+ If a list of positive integers is passed in, run only on those GPUs. By default it uses the CPU.
+ string_factory (Optional `str`): This is passed to the index factory of Faiss to create the index. Default index class is IndexFlatIP.
+ metric_type (Optional `int`): Type of metric. Ex: `faiss.METRIC_INNER_PRODUCT` or `faiss.METRIC_L2`.
+ custom_index (Optional `faiss.Index`): Custom Faiss index that you already have instantiated and configured for your needs.
+ batch_size (Optional `int`): Size of the batch to use while adding vectors to the FaissIndex. Default value is 1000.
+
+ train_size (Optional `int`): If the index needs a training step, specifies how many vectors will be used to train the index.
+ faiss_verbose (`bool`, defaults to False): Enable the verbosity of the Faiss index.
+ """
+ index_name = index_name if index_name is not None else column
+ faiss_index = FaissIndex(
+ device=device, string_factory=string_factory, metric_type=metric_type, custom_index=custom_index
+ )
+ faiss_index.add_vectors(
+ self, column=column, batch_size=batch_size, train_size=train_size, faiss_verbose=faiss_verbose
+ )
+ self._indexes[index_name] = faiss_index
+
+ def add_faiss_index_from_external_arrays(
+ self,
+ external_arrays: np.array,
+ index_name: str,
+ device: Optional[Union[int, list[int]]] = None,
+ string_factory: Optional[str] = None,
+ metric_type: Optional[int] = None,
+ custom_index: Optional["faiss.Index"] = None,
+ batch_size: int = 1000,
+ train_size: Optional[int] = None,
+ faiss_verbose: bool = False,
+ ):
+ """Add a dense index using Faiss for fast retrieval.
+ The index is created using the vectors of `external_arrays`.
+ You can specify `device` if you want to run it on GPU (`device` must be the GPU index).
+ You can find more information about Faiss here:
+ - For `string factory`: https://github.com/facebookresearch/faiss/wiki/The-index-factory
+
+ Args:
+ external_arrays (`np.array`): If you want to use arrays from outside the lib for the index, you can set `external_arrays`.
+ It will use `external_arrays` to create the Faiss index instead of the arrays in the given `column`.
+ index_name (`str`): The index_name/identifier of the index. This is the index_name that is used to call `.get_nearest` or `.search`.
+ device (Optional `Union[int, List[int]]`): If positive integer, this is the index of the GPU to use. If negative integer, use all GPUs.
+ If a list of positive integers is passed in, run only on those GPUs. By default it uses the CPU.
+ string_factory (Optional `str`): This is passed to the index factory of Faiss to create the index. Default index class is IndexFlatIP.
+ metric_type (Optional `int`): Type of metric. Ex: `faiss.METRIC_INNER_PRODUCT` or `faiss.METRIC_L2`.
+ custom_index (Optional `faiss.Index`): Custom Faiss index that you already have instantiated and configured for your needs.
+ batch_size (Optional `int`): Size of the batch to use while adding vectors to the FaissIndex. Default value is 1000.
+
+ train_size (Optional `int`): If the index needs a training step, specifies how many vectors will be used to train the index.
+ faiss_verbose (`bool`, defaults to False): Enable the verbosity of the Faiss index.
+ """
+ faiss_index = FaissIndex(
+ device=device, string_factory=string_factory, metric_type=metric_type, custom_index=custom_index
+ )
+ faiss_index.add_vectors(
+ external_arrays, column=None, batch_size=batch_size, train_size=train_size, faiss_verbose=faiss_verbose
+ )
+ self._indexes[index_name] = faiss_index
+
+ def save_faiss_index(self, index_name: str, file: Union[str, PurePath], storage_options: Optional[dict] = None):
+ """Save a FaissIndex on disk.
+
+ Args:
+ index_name (`str`): The index_name/identifier of the index. This is the index_name that is used to call `.get_nearest` or `.search`.
+ file (`str`): The path to the serialized faiss index on disk or remote URI (e.g. `"s3://my-bucket/index.faiss"`).
+ storage_options (`dict`, *optional*):
+ Key/value pairs to be passed on to the file-system backend, if any.
+
+
+
+ """
+ index = self.get_index(index_name)
+ if not isinstance(index, FaissIndex):
+ raise ValueError(f"Index '{index_name}' is not a FaissIndex but a '{type(index)}'")
+ index.save(file, storage_options=storage_options)
+ logger.info(f"Saved FaissIndex {index_name} at {file}")
+
+ def load_faiss_index(
+ self,
+ index_name: str,
+ file: Union[str, PurePath],
+ device: Optional[Union[int, list[int]]] = None,
+ storage_options: Optional[dict] = None,
+ ):
+ """Load a FaissIndex from disk.
+
+ If you want to do additional configurations, you can have access to the faiss index object by doing
+ `.get_index(index_name).faiss_index` to make it fit your needs.
+
+ Args:
+ index_name (`str`): The index_name/identifier of the index. This is the index_name that is used to
+ call `.get_nearest` or `.search`.
+ file (`str`): The path to the serialized faiss index on disk or remote URI (e.g. `"s3://my-bucket/index.faiss"`).
+ device (Optional `Union[int, List[int]]`): If positive integer, this is the index of the GPU to use. If negative integer, use all GPUs.
+ If a list of positive integers is passed in, run only on those GPUs. By default it uses the CPU.
+ storage_options (`dict`, *optional*):
+ Key/value pairs to be passed on to the file-system backend, if any.
+
+
+
+ """
+ index = FaissIndex.load(file, device=device, storage_options=storage_options)
+ if index.faiss_index.ntotal != len(self):
+ raise ValueError(
+ f"Index size should match Dataset size, but Index '{index_name}' at {file} has {index.faiss_index.ntotal} elements while the dataset has {len(self)} examples."
+ )
+ self._indexes[index_name] = index
+ logger.info(f"Loaded FaissIndex {index_name} from {file}")
+
+ def add_elasticsearch_index(
+ self,
+ column: str,
+ index_name: Optional[str] = None,
+ host: Optional[str] = None,
+ port: Optional[int] = None,
+ es_client: Optional["Elasticsearch"] = None,
+ es_index_name: Optional[str] = None,
+ es_index_config: Optional[dict] = None,
+ ):
+ """Add a text index using ElasticSearch for fast retrieval.
+
+ Args:
+ column (`str`): The column of the documents to add to the index.
+ index_name (Optional `str`): The index_name/identifier of the index. This is the index name that is used to call `.get_nearest` or `.search`.
+ By default it corresponds to `column`.
+ host (Optional `str`, defaults to localhost):
+ host of where ElasticSearch is running
+ port (Optional `str`, defaults to 9200):
+ port of where ElasticSearch is running
+ es_client (Optional `elasticsearch.Elasticsearch`):
+ The elasticsearch client used to create the index if host and port are None.
+ es_index_name (Optional `str`): The elasticsearch index name used to create the index.
+ es_index_config (Optional `dict`):
+ The configuration of the elasticsearch index.
+ Default config is:
+
+ Config::
+
+ {
+ "settings": {
+ "number_of_shards": 1,
+ "analysis": {"analyzer": {"stop_standard": {"type": "standard", " stopwords": "_english_"}}},
+ },
+ "mappings": {
+ "properties": {
+ "text": {
+ "type": "text",
+ "analyzer": "standard",
+ "similarity": "BM25"
+ },
+ }
+ },
+ }
+ """
+ index_name = index_name if index_name is not None else column
+ es_index = ElasticSearchIndex(
+ host=host, port=port, es_client=es_client, es_index_name=es_index_name, es_index_config=es_index_config
+ )
+ es_index.add_documents(self, column=column)
+ self._indexes[index_name] = es_index
+
+ def load_elasticsearch_index(
+ self,
+ index_name: str,
+ es_index_name: str,
+ host: Optional[str] = None,
+ port: Optional[int] = None,
+ es_client: Optional["Elasticsearch"] = None,
+ es_index_config: Optional[dict] = None,
+ ):
+ """Load an existing text index using ElasticSearch for fast retrieval.
+
+ Args:
+ index_name (`str`):
+ The `index_name`/identifier of the index. This is the index name that is used to call `get_nearest` or `search`.
+ es_index_name (`str`):
+ The name of elasticsearch index to load.
+ host (`str`, *optional*, defaults to `localhost`):
+ Host of where ElasticSearch is running.
+ port (`str`, *optional*, defaults to `9200`):
+ Port of where ElasticSearch is running.
+ es_client (`elasticsearch.Elasticsearch`, *optional*):
+ The elasticsearch client used to create the index if host and port are `None`.
+ es_index_config (`dict`, *optional*):
+ The configuration of the elasticsearch index.
+ Default config is:
+ ```
+ {
+ "settings": {
+ "number_of_shards": 1,
+ "analysis": {"analyzer": {"stop_standard": {"type": "standard", " stopwords": "_english_"}}},
+ },
+ "mappings": {
+ "properties": {
+ "text": {
+ "type": "text",
+ "analyzer": "standard",
+ "similarity": "BM25"
+ },
+ }
+ },
+ }
+ ```
+ """
+ self._indexes[index_name] = ElasticSearchIndex(
+ host=host, port=port, es_client=es_client, es_index_name=es_index_name, es_index_config=es_index_config
+ )
+
+ def drop_index(self, index_name: str):
+ """Drop the index with the specified column.
+
+ Args:
+ index_name (`str`):
+ The `index_name`/identifier of the index.
+ """
+ del self._indexes[index_name]
+
+ def search(self, index_name: str, query: Union[str, np.array], k: int = 10, **kwargs) -> SearchResults:
+ """Find the nearest examples indices in the dataset to the query.
+
+ Args:
+ index_name (`str`):
+ The name/identifier of the index.
+ query (`Union[str, np.ndarray]`):
+ The query as a string if `index_name` is a text index or as a numpy array if `index_name` is a vector index.
+ k (`int`):
+ The number of examples to retrieve.
+
+ Returns:
+ `(scores, indices)`:
+ A tuple of `(scores, indices)` where:
+ - **scores** (`List[List[float]`): the retrieval scores from either FAISS (`IndexFlatL2` by default) or ElasticSearch of the retrieved examples
+ - **indices** (`List[List[int]]`): the indices of the retrieved examples
+ """
+ self._check_index_is_initialized(index_name)
+ return self._indexes[index_name].search(query, k, **kwargs)
+
+ def search_batch(
+ self, index_name: str, queries: Union[list[str], np.array], k: int = 10, **kwargs
+ ) -> BatchedSearchResults:
+ """Find the nearest examples indices in the dataset to the query.
+
+ Args:
+ index_name (`str`):
+ The `index_name`/identifier of the index.
+ queries (`Union[List[str], np.ndarray]`):
+ The queries as a list of strings if `index_name` is a text index or as a numpy array if `index_name` is a vector index.
+ k (`int`):
+ The number of examples to retrieve per query.
+
+ Returns:
+ `(total_scores, total_indices)`:
+ A tuple of `(total_scores, total_indices)` where:
+ - **total_scores** (`List[List[float]`): the retrieval scores from either FAISS (`IndexFlatL2` by default) or ElasticSearch of the retrieved examples per query
+ - **total_indices** (`List[List[int]]`): the indices of the retrieved examples per query
+ """
+ self._check_index_is_initialized(index_name)
+ return self._indexes[index_name].search_batch(queries, k, **kwargs)
+
+ def get_nearest_examples(
+ self, index_name: str, query: Union[str, np.array], k: int = 10, **kwargs
+ ) -> NearestExamplesResults:
+ """Find the nearest examples in the dataset to the query.
+
+ Args:
+ index_name (`str`):
+ The index_name/identifier of the index.
+ query (`Union[str, np.ndarray]`):
+ The query as a string if `index_name` is a text index or as a numpy array if `index_name` is a vector index.
+ k (`int`):
+ The number of examples to retrieve.
+
+ Returns:
+ `(scores, examples)`:
+ A tuple of `(scores, examples)` where:
+ - **scores** (`List[float]`): the retrieval scores from either FAISS (`IndexFlatL2` by default) or ElasticSearch of the retrieved examples
+ - **examples** (`dict`): the retrieved examples
+ """
+ self._check_index_is_initialized(index_name)
+ scores, indices = self.search(index_name, query, k, **kwargs)
+ top_indices = [i for i in indices if i >= 0]
+ return NearestExamplesResults(scores[: len(top_indices)], self[top_indices])
+
+ def get_nearest_examples_batch(
+ self, index_name: str, queries: Union[list[str], np.array], k: int = 10, **kwargs
+ ) -> BatchedNearestExamplesResults:
+ """Find the nearest examples in the dataset to the query.
+
+ Args:
+ index_name (`str`):
+ The `index_name`/identifier of the index.
+ queries (`Union[List[str], np.ndarray]`):
+ The queries as a list of strings if `index_name` is a text index or as a numpy array if `index_name` is a vector index.
+ k (`int`):
+ The number of examples to retrieve per query.
+
+ Returns:
+ `(total_scores, total_examples)`:
+ A tuple of `(total_scores, total_examples)` where:
+ - **total_scores** (`List[List[float]`): the retrieval scores from either FAISS (`IndexFlatL2` by default) or ElasticSearch of the retrieved examples per query
+ - **total_examples** (`List[dict]`): the retrieved examples per query
+ """
+ self._check_index_is_initialized(index_name)
+ total_scores, total_indices = self.search_batch(index_name, queries, k, **kwargs)
+ total_scores = [
+ scores_i[: len([i for i in indices_i if i >= 0])]
+ for scores_i, indices_i in zip(total_scores, total_indices)
+ ]
+ total_samples = [self[[i for i in indices if i >= 0]] for indices in total_indices]
+ return BatchedNearestExamplesResults(total_scores, total_samples)
diff --git a/venv/lib/python3.10/site-packages/datasets/splits.py b/venv/lib/python3.10/site-packages/datasets/splits.py
new file mode 100644
index 0000000000000000000000000000000000000000..02c18eef351ec3bd5270d8c6a42ca23710dd5f49
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/splits.py
@@ -0,0 +1,635 @@
+# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""Splits related API."""
+
+import abc
+import collections
+import copy
+import dataclasses
+import re
+from dataclasses import dataclass
+from typing import Optional, Union
+
+from .arrow_reader import FileInstructions, make_file_instructions
+from .naming import _split_re
+from .utils.py_utils import NonMutableDict, asdict
+
+
+@dataclass
+class SplitInfo:
+ name: str = dataclasses.field(default="", metadata={"include_in_asdict_even_if_is_default": True})
+ num_bytes: int = dataclasses.field(default=0, metadata={"include_in_asdict_even_if_is_default": True})
+ num_examples: int = dataclasses.field(default=0, metadata={"include_in_asdict_even_if_is_default": True})
+ shard_lengths: Optional[list[int]] = None
+
+ # Deprecated
+ # For backward compatibility, this field needs to always be included in files like
+ # dataset_infos.json and dataset_info.json files
+ # To do so, we always include it in the output of datasets.utils.py_utils.asdict(split_info)
+ dataset_name: Optional[str] = dataclasses.field(
+ default=None, metadata={"include_in_asdict_even_if_is_default": True}
+ )
+
+ @property
+ def file_instructions(self):
+ """Returns the list of dict(filename, take, skip)."""
+ # `self.dataset_name` is assigned in `SplitDict.add()`.
+ instructions = make_file_instructions(
+ name=self.dataset_name,
+ split_infos=[self],
+ instruction=str(self.name),
+ )
+ return instructions.file_instructions
+
+
+@dataclass
+class SubSplitInfo:
+ """Wrapper around a sub split info.
+ This class expose info on the subsplit:
+ ```
+ ds, info = datasets.load_dataset(..., split='train[75%:]', with_info=True)
+ info.splits['train[75%:]'].num_examples
+ ```
+ """
+
+ instructions: FileInstructions
+
+ @property
+ def num_examples(self):
+ """Returns the number of example in the subsplit."""
+ return self.instructions.num_examples
+
+ @property
+ def file_instructions(self):
+ """Returns the list of dict(filename, take, skip)."""
+ return self.instructions.file_instructions
+
+
+class SplitBase(metaclass=abc.ABCMeta):
+ # pylint: disable=line-too-long
+ """Abstract base class for Split compositionality.
+
+ See the
+ [guide on splits](../loading#slice-splits)
+ for more information.
+
+ There are three parts to the composition:
+ 1) The splits are composed (defined, merged, split,...) together before
+ calling the `.as_dataset()` function. This is done with the `__add__`,
+ `__getitem__`, which return a tree of `SplitBase` (whose leaf
+ are the `NamedSplit` objects)
+
+ ```
+ split = datasets.Split.TRAIN + datasets.Split.TEST.subsplit(datasets.percent[:50])
+ ```
+
+ 2) The `SplitBase` is forwarded to the `.as_dataset()` function
+ to be resolved into actual read instruction. This is done by the
+ `.get_read_instruction()` method which takes the real dataset splits
+ (name, number of shards,...) and parse the tree to return a
+ `SplitReadInstruction()` object
+
+ ```
+ read_instruction = split.get_read_instruction(self.info.splits)
+ ```
+
+ 3) The `SplitReadInstruction` is then used in the `tf.data.Dataset` pipeline
+ to define which files to read and how to skip examples within file.
+
+ """
+
+ # pylint: enable=line-too-long
+
+ @abc.abstractmethod
+ def get_read_instruction(self, split_dict):
+ """Parse the descriptor tree and compile all read instructions together.
+
+ Args:
+ split_dict: `dict`, The `dict[split_name, SplitInfo]` of the dataset
+
+ Returns:
+ split_read_instruction: `SplitReadInstruction`
+ """
+ raise NotImplementedError("Abstract method")
+
+ def __eq__(self, other):
+ """Equality: datasets.Split.TRAIN == 'train'."""
+ if isinstance(other, (NamedSplit, str)):
+ return False
+ raise NotImplementedError("Equality is not implemented between merged/sub splits.")
+
+ def __ne__(self, other):
+ """InEquality: datasets.Split.TRAIN != 'test'."""
+ return not self.__eq__(other)
+
+ def __add__(self, other):
+ """Merging: datasets.Split.TRAIN + datasets.Split.TEST."""
+ return _SplitMerged(self, other)
+
+ def subsplit(self, arg=None, k=None, percent=None, weighted=None): # pylint: disable=redefined-outer-name
+ """Divides this split into subsplits.
+
+ There are 3 ways to define subsplits, which correspond to the 3
+ arguments `k` (get `k` even subsplits), `percent` (get a slice of the
+ dataset with `datasets.percent`), and `weighted` (get subsplits with proportions
+ specified by `weighted`).
+
+ Example::
+
+ ```
+ # 50% train, 50% test
+ train, test = split.subsplit(k=2)
+ # 50% train, 25% test, 25% validation
+ train, test, validation = split.subsplit(weighted=[2, 1, 1])
+ # Extract last 20%
+ subsplit = split.subsplit(datasets.percent[-20:])
+ ```
+
+ Warning: k and weighted will be converted into percent which mean that
+ values below the percent will be rounded up or down. The final split may be
+ bigger to deal with remainders. For instance:
+
+ ```
+ train, test, valid = split.subsplit(k=3) # 33%, 33%, 34%
+ s1, s2, s3, s4 = split.subsplit(weighted=[2, 2, 1, 1]) # 33%, 33%, 16%, 18%
+ ```
+
+ Args:
+ arg: If no kwargs are given, `arg` will be interpreted as one of
+ `k`, `percent`, or `weighted` depending on the type.
+ For example:
+ ```
+ split.subsplit(10) # Equivalent to split.subsplit(k=10)
+ split.subsplit(datasets.percent[:-20]) # percent=datasets.percent[:-20]
+ split.subsplit([1, 1, 2]) # weighted=[1, 1, 2]
+ ```
+ k: `int` If set, subdivide the split into `k` equal parts.
+ percent: `datasets.percent slice`, return a single subsplit corresponding to
+ a slice of the original split. For example:
+ `split.subsplit(datasets.percent[-20:]) # Last 20% of the dataset`.
+ weighted: `list[int]`, return a list of subsplits whose proportions match
+ the normalized sum of the list. For example:
+ `split.subsplit(weighted=[1, 1, 2]) # 25%, 25%, 50%`.
+
+ Returns:
+ A subsplit or list of subsplits extracted from this split object.
+ """
+ # Note that the percent kwargs redefine the outer name datasets.percent. This
+ # is done for consistency (.subsplit(percent=datasets.percent[:40]))
+ if sum(bool(x) for x in (arg, k, percent, weighted)) != 1:
+ raise ValueError("Only one argument of subsplit should be set.")
+
+ # Auto deduce k
+ if isinstance(arg, int):
+ k = arg
+ elif isinstance(arg, slice):
+ percent = arg
+ elif isinstance(arg, list):
+ weighted = arg
+
+ if not (k or percent or weighted):
+ raise ValueError(
+ f"Invalid split argument {arg}. Only list, slice and int supported. "
+ "One of k, weighted or percent should be set to a non empty value."
+ )
+
+ def assert_slices_coverage(slices):
+ # Ensure that the expended slices cover all percents.
+ assert sum((list(range(*s.indices(100))) for s in slices), []) == list(range(100))
+
+ if k:
+ if not 0 < k <= 100:
+ raise ValueError(f"Subsplit k should be between 0 and 100, got {k}")
+ shift = 100 // k
+ slices = [slice(i * shift, (i + 1) * shift) for i in range(k)]
+ # Round up last element to ensure all elements are taken
+ slices[-1] = slice(slices[-1].start, 100)
+ # Internal check to ensure full coverage
+ assert_slices_coverage(slices)
+ return tuple(_SubSplit(self, s) for s in slices)
+ elif percent:
+ return _SubSplit(self, percent)
+ elif weighted:
+ # Normalize the weighted sum
+ total = sum(weighted)
+ weighted = [100 * x // total for x in weighted]
+ # Create the slice for each of the elements
+ start = 0
+ stop = 0
+ slices = []
+ for v in weighted:
+ stop += v
+ slices.append(slice(start, stop))
+ start = stop
+ # Round up last element to ensure all elements are taken
+ slices[-1] = slice(slices[-1].start, 100)
+ # Internal check to ensure full coverage
+ assert_slices_coverage(slices)
+ return tuple(_SubSplit(self, s) for s in slices)
+ else:
+ # Should not be possible
+ raise ValueError("Could not determine the split")
+
+
+# 2 requirements:
+# 1. datasets.percent be sliceable
+# 2. datasets.percent be documented
+#
+# Instances are not documented, so we want datasets.percent to be a class, but to
+# have it be sliceable, we need this metaclass.
+class PercentSliceMeta(type):
+ def __getitem__(cls, slice_value):
+ if not isinstance(slice_value, slice):
+ raise ValueError(f"datasets.percent should only be called with slice, not {slice_value}")
+ return slice_value
+
+
+class PercentSlice(metaclass=PercentSliceMeta):
+ # pylint: disable=line-too-long
+ """Syntactic sugar for defining slice subsplits: `datasets.percent[75:-5]`.
+
+ See the
+ [guide on splits](../loading#slice-splits)
+ for more information.
+ """
+
+ # pylint: enable=line-too-long
+ pass
+
+
+percent = PercentSlice # pylint: disable=invalid-name
+
+
+class _SplitMerged(SplitBase):
+ """Represent two split descriptors merged together."""
+
+ def __init__(self, split1, split2):
+ self._split1 = split1
+ self._split2 = split2
+
+ def get_read_instruction(self, split_dict):
+ read_instruction1 = self._split1.get_read_instruction(split_dict)
+ read_instruction2 = self._split2.get_read_instruction(split_dict)
+ return read_instruction1 + read_instruction2
+
+ def __repr__(self):
+ return f"({repr(self._split1)} + {repr(self._split2)})"
+
+
+class _SubSplit(SplitBase):
+ """Represent a sub split of a split descriptor."""
+
+ def __init__(self, split, slice_value):
+ self._split = split
+ self._slice_value = slice_value
+
+ def get_read_instruction(self, split_dict):
+ return self._split.get_read_instruction(split_dict)[self._slice_value]
+
+ def __repr__(self):
+ slice_str = "{start}:{stop}"
+ if self._slice_value.step is not None:
+ slice_str += ":{step}"
+ slice_str = slice_str.format(
+ start="" if self._slice_value.start is None else self._slice_value.start,
+ stop="" if self._slice_value.stop is None else self._slice_value.stop,
+ step=self._slice_value.step,
+ )
+ return f"{repr(self._split)}(datasets.percent[{slice_str}])"
+
+
+class NamedSplit(SplitBase):
+ """Descriptor corresponding to a named split (train, test, ...).
+
+ Example:
+ Each descriptor can be composed with other using addition or slice:
+
+ ```py
+ split = datasets.Split.TRAIN.subsplit(datasets.percent[0:25]) + datasets.Split.TEST
+ ```
+
+ The resulting split will correspond to 25% of the train split merged with
+ 100% of the test split.
+
+ A split cannot be added twice, so the following will fail:
+
+ ```py
+ split = (
+ datasets.Split.TRAIN.subsplit(datasets.percent[:25]) +
+ datasets.Split.TRAIN.subsplit(datasets.percent[75:])
+ ) # Error
+ split = datasets.Split.TEST + datasets.Split.ALL # Error
+ ```
+
+ The slices can be applied only one time. So the following are valid:
+
+ ```py
+ split = (
+ datasets.Split.TRAIN.subsplit(datasets.percent[:25]) +
+ datasets.Split.TEST.subsplit(datasets.percent[:50])
+ )
+ split = (datasets.Split.TRAIN + datasets.Split.TEST).subsplit(datasets.percent[:50])
+ ```
+
+ But this is not valid:
+
+ ```py
+ train = datasets.Split.TRAIN
+ test = datasets.Split.TEST
+ split = train.subsplit(datasets.percent[:25]).subsplit(datasets.percent[:25])
+ split = (train.subsplit(datasets.percent[:25]) + test).subsplit(datasets.percent[:50])
+ ```
+ """
+
+ def __init__(self, name):
+ self._name = name
+ split_names_from_instruction = [split_instruction.split("[")[0] for split_instruction in name.split("+")]
+ for split_name in split_names_from_instruction:
+ if not re.match(_split_re, split_name):
+ raise ValueError(f"Split name should match '{_split_re}' but got '{split_name}'.")
+
+ def __str__(self):
+ return self._name
+
+ def __repr__(self):
+ return f"NamedSplit({self._name!r})"
+
+ def __eq__(self, other):
+ """Equality: datasets.Split.TRAIN == 'train'."""
+ if isinstance(other, NamedSplit):
+ return self._name == other._name # pylint: disable=protected-access
+ elif isinstance(other, SplitBase):
+ return False
+ elif isinstance(other, str): # Other should be string
+ return self._name == other
+ else:
+ return False
+
+ def __lt__(self, other):
+ return self._name < other._name # pylint: disable=protected-access
+
+ def __hash__(self):
+ return hash(self._name)
+
+ def get_read_instruction(self, split_dict):
+ return SplitReadInstruction(split_dict[self._name])
+
+
+class NamedSplitAll(NamedSplit):
+ """Split corresponding to the union of all defined dataset splits."""
+
+ def __init__(self):
+ super().__init__("all")
+
+ def __repr__(self):
+ return "NamedSplitAll()"
+
+ def get_read_instruction(self, split_dict):
+ # Merge all dataset split together
+ read_instructions = [SplitReadInstruction(s) for s in split_dict.values()]
+ return sum(read_instructions, SplitReadInstruction())
+
+
+class Split:
+ # pylint: disable=line-too-long
+ """`Enum` for dataset splits.
+
+ Datasets are typically split into different subsets to be used at various
+ stages of training and evaluation.
+
+ - `TRAIN`: the training data.
+ - `VALIDATION`: the validation data. If present, this is typically used as
+ evaluation data while iterating on a model (e.g. changing hyperparameters,
+ model architecture, etc.).
+ - `TEST`: the testing data. This is the data to report metrics on. Typically
+ you do not want to use this during model iteration as you may overfit to it.
+ - `ALL`: the union of all defined dataset splits.
+
+ All splits, including compositions inherit from `datasets.SplitBase`.
+
+ See the [guide](../load_hub#splits) on splits for more information.
+
+ Example:
+
+ ```py
+ >>> datasets.SplitGenerator(
+ ... name=datasets.Split.TRAIN,
+ ... gen_kwargs={"split_key": "train", "files": dl_manager.download_and extract(url)},
+ ... ),
+ ... datasets.SplitGenerator(
+ ... name=datasets.Split.VALIDATION,
+ ... gen_kwargs={"split_key": "validation", "files": dl_manager.download_and extract(url)},
+ ... ),
+ ... datasets.SplitGenerator(
+ ... name=datasets.Split.TEST,
+ ... gen_kwargs={"split_key": "test", "files": dl_manager.download_and extract(url)},
+ ... )
+ ```
+ """
+
+ # pylint: enable=line-too-long
+ TRAIN = NamedSplit("train")
+ TEST = NamedSplit("test")
+ VALIDATION = NamedSplit("validation")
+ ALL = NamedSplitAll()
+
+ def __new__(cls, name):
+ """Create a custom split with datasets.Split('custom_name')."""
+ return NamedSplitAll() if name == "all" else NamedSplit(name)
+
+
+# Similar to SplitInfo, but contain an additional slice info
+SlicedSplitInfo = collections.namedtuple(
+ "SlicedSplitInfo",
+ [
+ "split_info",
+ "slice_value",
+ ],
+) # noqa: E231
+
+
+class SplitReadInstruction:
+ """Object containing the reading instruction for the dataset.
+
+ Similarly to `SplitDescriptor` nodes, this object can be composed with itself,
+ but the resolution happens instantaneously, instead of keeping track of the
+ tree, such as all instructions are compiled and flattened in a single
+ SplitReadInstruction object containing the list of files and slice to use.
+
+ Once resolved, the instructions can be accessed with:
+
+ ```
+ read_instructions.get_list_sliced_split_info() # List of splits to use
+ ```
+
+ """
+
+ def __init__(self, split_info=None):
+ self._splits = NonMutableDict(error_msg="Overlap between splits. Split {key} has been added with itself.")
+
+ if split_info:
+ self.add(SlicedSplitInfo(split_info=split_info, slice_value=None))
+
+ def add(self, sliced_split):
+ """Add a SlicedSplitInfo the read instructions."""
+ # TODO(epot): Check that the number of examples per shard % 100 == 0
+ # Otherwise the slices value may be unbalanced and not exactly reflect the
+ # requested slice.
+ self._splits[sliced_split.split_info.name] = sliced_split
+
+ def __add__(self, other):
+ """Merging split together."""
+ # Will raise error if a split has already be added (NonMutableDict)
+ # TODO(epot): If a split is already added but there is no overlap between
+ # the slices, should merge the slices (ex: [:10] + [80:])
+ split_instruction = SplitReadInstruction()
+ split_instruction._splits.update(self._splits) # pylint: disable=protected-access
+ split_instruction._splits.update(other._splits) # pylint: disable=protected-access
+ return split_instruction
+
+ def __getitem__(self, slice_value):
+ """Sub-splits."""
+ # Will raise an error if a split has already been sliced
+ split_instruction = SplitReadInstruction()
+ for v in self._splits.values():
+ if v.slice_value is not None:
+ raise ValueError(f"Trying to slice Split {v.split_info.name} which has already been sliced")
+ v = v._asdict()
+ v["slice_value"] = slice_value
+ split_instruction.add(SlicedSplitInfo(**v))
+ return split_instruction
+
+ def get_list_sliced_split_info(self):
+ return list(self._splits.values())
+
+
+class SplitDict(dict):
+ """Split info object."""
+
+ def __init__(self, *args, dataset_name=None, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.dataset_name = dataset_name
+
+ def __getitem__(self, key: Union[SplitBase, str]):
+ # 1st case: The key exists: `info.splits['train']`
+ if str(key) in self:
+ return super().__getitem__(str(key))
+ # 2nd case: Uses instructions: `info.splits['train[50%]']`
+ else:
+ instructions = make_file_instructions(
+ name=self.dataset_name,
+ split_infos=self.values(),
+ instruction=key,
+ )
+ return SubSplitInfo(instructions)
+
+ def __setitem__(self, key: Union[SplitBase, str], value: SplitInfo):
+ if key != value.name:
+ raise ValueError(f"Cannot add elem. (key mismatch: '{key}' != '{value.name}')")
+ super().__setitem__(key, value)
+
+ def add(self, split_info: SplitInfo):
+ """Add the split info."""
+ if split_info.name in self:
+ raise ValueError(f"Split {split_info.name} already present")
+ split_info.dataset_name = self.dataset_name
+ super().__setitem__(split_info.name, split_info)
+
+ @property
+ def total_num_examples(self):
+ """Return the total number of examples."""
+ return sum(s.num_examples for s in self.values())
+
+ @classmethod
+ def from_split_dict(cls, split_infos: Union[list, dict], dataset_name: Optional[str] = None):
+ """Returns a new SplitDict initialized from a Dict or List of `split_infos`."""
+ if isinstance(split_infos, dict):
+ split_infos = list(split_infos.values())
+
+ if dataset_name is None:
+ dataset_name = split_infos[0].get("dataset_name") if split_infos else None
+
+ split_dict = cls(dataset_name=dataset_name)
+
+ for split_info in split_infos:
+ if isinstance(split_info, dict):
+ split_info = SplitInfo(**split_info)
+ split_dict.add(split_info)
+
+ return split_dict
+
+ def to_split_dict(self):
+ """Returns a list of SplitInfo protos that we have."""
+ out = []
+ for split_name, split_info in self.items():
+ split_info = copy.deepcopy(split_info)
+ split_info.name = split_name
+ out.append(split_info)
+ return out
+
+ def copy(self):
+ return SplitDict.from_split_dict(self.to_split_dict(), self.dataset_name)
+
+ def _to_yaml_list(self) -> list:
+ out = [asdict(s) for s in self.to_split_dict()]
+ # we don't need the shard lengths in YAML, since it depends on max_shard_size and num_proc
+ for split_info_dict in out:
+ split_info_dict.pop("shard_lengths", None)
+ # we don't need the dataset_name attribute that is deprecated
+ for split_info_dict in out:
+ split_info_dict.pop("dataset_name", None)
+ return out
+
+ @classmethod
+ def _from_yaml_list(cls, yaml_data: list) -> "SplitDict":
+ return cls.from_split_dict(yaml_data)
+
+
+@dataclass
+class SplitGenerator:
+ """Defines the split information for the generator.
+
+ This should be used as returned value of
+ `GeneratorBasedBuilder._split_generators`.
+ See `GeneratorBasedBuilder._split_generators` for more info and example
+ of usage.
+
+ Args:
+ name (`str`):
+ Name of the `Split` for which the generator will
+ create the examples.
+ **gen_kwargs (additional keyword arguments):
+ Keyword arguments to forward to the `DatasetBuilder._generate_examples` method
+ of the builder.
+
+ Example:
+
+ ```py
+ >>> datasets.SplitGenerator(
+ ... name=datasets.Split.TRAIN,
+ ... gen_kwargs={"split_key": "train", "files": dl_manager.download_and_extract(url)},
+ ... )
+ ```
+ """
+
+ name: str
+ gen_kwargs: dict = dataclasses.field(default_factory=dict)
+ split_info: SplitInfo = dataclasses.field(init=False)
+
+ def __post_init__(self):
+ self.name = str(self.name) # Make sure we convert NamedSplits in strings
+ NamedSplit(self.name) # check that it's a valid split name
+ self.split_info = SplitInfo(name=self.name)
diff --git a/venv/lib/python3.10/site-packages/datasets/streaming.py b/venv/lib/python3.10/site-packages/datasets/streaming.py
new file mode 100644
index 0000000000000000000000000000000000000000..6e5e8ee15f9f529d93d43dc003ccdef3436fd725
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/streaming.py
@@ -0,0 +1,131 @@
+import importlib
+from functools import wraps
+from typing import TYPE_CHECKING, Optional
+
+from .download.download_config import DownloadConfig
+from .utils.file_utils import (
+ xbasename,
+ xdirname,
+ xet_parse,
+ xexists,
+ xgetsize,
+ xglob,
+ xgzip_open,
+ xisdir,
+ xisfile,
+ xjoin,
+ xlistdir,
+ xnumpy_load,
+ xopen,
+ xpandas_read_csv,
+ xpandas_read_excel,
+ xPath,
+ xpyarrow_parquet_read_table,
+ xrelpath,
+ xsio_loadmat,
+ xsplit,
+ xsplitext,
+ xwalk,
+ xxml_dom_minidom_parse,
+)
+from .utils.logging import get_logger
+from .utils.patching import patch_submodule
+
+
+logger = get_logger(__name__)
+
+
+if TYPE_CHECKING:
+ from .builder import DatasetBuilder
+
+
+def extend_module_for_streaming(module_path, download_config: Optional[DownloadConfig] = None):
+ """Extend the module to support streaming.
+
+ We patch some functions in the module to use `fsspec` to support data streaming:
+ - We use `fsspec.open` to open and read remote files. We patch the module function:
+ - `open`
+ - We use the "::" hop separator to join paths and navigate remote compressed/archive files. We patch the module
+ functions:
+ - `os.path.join`
+ - `pathlib.Path.joinpath` and `pathlib.Path.__truediv__` (called when using the "/" operator)
+
+ The patched functions are replaced with custom functions defined to work with the
+ :class:`~download.streaming_download_manager.StreamingDownloadManager`.
+
+ Args:
+ module_path: Path to the module to be extended.
+ download_config: Mainly use `token` or `storage_options` to support different platforms and auth types.
+ """
+
+ module = importlib.import_module(module_path)
+
+ # TODO(QL): always update the module to add subsequent new authentication without removing old ones
+ if hasattr(module, "_patched_for_streaming") and module._patched_for_streaming:
+ if isinstance(module._patched_for_streaming, DownloadConfig):
+ module._patched_for_streaming.token = download_config.token
+ module._patched_for_streaming.storage_options = download_config.storage_options
+ return
+
+ def wrap_auth(function):
+ @wraps(function)
+ def wrapper(*args, **kwargs):
+ return function(*args, download_config=download_config, **kwargs)
+
+ wrapper._decorator_name_ = "wrap_auth"
+ return wrapper
+
+ # open files in a streaming fashion
+ patch_submodule(module, "open", wrap_auth(xopen)).start()
+ patch_submodule(module, "os.listdir", wrap_auth(xlistdir)).start()
+ patch_submodule(module, "os.walk", wrap_auth(xwalk)).start()
+ patch_submodule(module, "glob.glob", wrap_auth(xglob)).start()
+ # allow to navigate in remote zip files
+ patch_submodule(module, "os.path.join", xjoin).start()
+ patch_submodule(module, "os.path.dirname", xdirname).start()
+ patch_submodule(module, "os.path.basename", xbasename).start()
+ patch_submodule(module, "os.path.relpath", xrelpath).start()
+ patch_submodule(module, "os.path.split", xsplit).start()
+ patch_submodule(module, "os.path.splitext", xsplitext).start()
+ # allow checks on paths
+ patch_submodule(module, "os.path.exists", wrap_auth(xexists)).start()
+ patch_submodule(module, "os.path.isdir", wrap_auth(xisdir)).start()
+ patch_submodule(module, "os.path.isfile", wrap_auth(xisfile)).start()
+ patch_submodule(module, "os.path.getsize", wrap_auth(xgetsize)).start()
+ patch_submodule(module, "pathlib.Path", xPath).start()
+ # file readers
+ patch_submodule(module, "gzip.open", wrap_auth(xgzip_open)).start()
+ patch_submodule(module, "numpy.load", wrap_auth(xnumpy_load)).start()
+ patch_submodule(module, "pandas.read_csv", wrap_auth(xpandas_read_csv), attrs=["__version__"]).start()
+ patch_submodule(module, "pandas.read_excel", wrap_auth(xpandas_read_excel), attrs=["__version__"]).start()
+ patch_submodule(module, "scipy.io.loadmat", wrap_auth(xsio_loadmat), attrs=["__version__"]).start()
+ patch_submodule(module, "xml.etree.ElementTree.parse", wrap_auth(xet_parse)).start()
+ patch_submodule(module, "xml.dom.minidom.parse", wrap_auth(xxml_dom_minidom_parse)).start()
+ # pyarrow: do not patch pyarrow attribute in packaged modules
+ if not module.__name__.startswith("datasets.packaged_modules."):
+ patch_submodule(module, "pyarrow.parquet.read_table", wrap_auth(xpyarrow_parquet_read_table)).start()
+ module._patched_for_streaming = download_config
+
+
+def extend_dataset_builder_for_streaming(builder: "DatasetBuilder"):
+ """Extend the dataset builder module and the modules imported by it to support streaming.
+
+ Args:
+ builder (:class:`DatasetBuilder`): Dataset builder instance.
+ """
+ # this extends the open and os.path.join functions for data streaming
+ download_config = DownloadConfig(storage_options=builder.storage_options, token=builder.token)
+ extend_module_for_streaming(builder.__module__, download_config=download_config)
+
+ # builders can inherit from other builders that might use streaming functionality
+ # (for example, ImageFolder and AudioFolder inherit from FolderBuilder which implements examples generation)
+ # but these parents builders are not patched automatically as they are not instantiated, so we patch them here
+ from .builder import DatasetBuilder
+
+ parent_builder_modules = [
+ cls.__module__
+ for cls in type(builder).__mro__[1:] # make sure it's not the same module we've already patched
+ if issubclass(cls, DatasetBuilder) and cls.__module__ != DatasetBuilder.__module__
+ ] # check it's not a standard builder from datasets.builder
+ for module in parent_builder_modules:
+ extend_module_for_streaming(module, download_config=download_config)
diff --git a/venv/lib/python3.10/site-packages/datasets/table.py b/venv/lib/python3.10/site-packages/datasets/table.py
new file mode 100644
index 0000000000000000000000000000000000000000..8d301afea7ad4279df8b33e69925ea76544f70c6
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/table.py
@@ -0,0 +1,2385 @@
+import copy
+import os
+from collections.abc import Iterator
+from functools import partial
+from itertools import groupby
+from typing import TYPE_CHECKING, Any, Callable, Optional, TypeVar, Union
+
+import numpy as np
+import pyarrow as pa
+import pyarrow.compute as pc
+
+from .utils.logging import get_logger
+
+
+if TYPE_CHECKING:
+ from .features.features import Features, FeatureType
+
+
+logger = get_logger(__name__)
+
+
+def inject_arrow_table_documentation(arrow_table_method):
+ def wrapper(fn):
+ fn.__doc__ = arrow_table_method.__doc__ + (fn.__doc__ if fn.__doc__ is not None else "")
+ fn.__doc__ = fn.__doc__.replace("pyarrow.Table", "Table")
+ if hasattr(arrow_table_method, "__annotations__"):
+ fn.__annotations__ = arrow_table_method.__annotations__
+ return fn
+
+ return wrapper
+
+
+def _in_memory_arrow_table_from_file(filename: str) -> pa.Table:
+ in_memory_stream = pa.input_stream(filename)
+ opened_stream = pa.ipc.open_stream(in_memory_stream)
+ pa_table = opened_stream.read_all()
+ return pa_table
+
+
+def _in_memory_arrow_table_from_buffer(buffer: pa.Buffer) -> pa.Table:
+ stream = pa.BufferReader(buffer)
+ opened_stream = pa.ipc.open_stream(stream)
+ table = opened_stream.read_all()
+ return table
+
+
+def _memory_mapped_record_batch_reader_from_file(filename: str) -> pa.RecordBatchStreamReader:
+ memory_mapped_stream = pa.memory_map(filename)
+ return pa.ipc.open_stream(memory_mapped_stream)
+
+
+def read_schema_from_file(filename: str) -> pa.Schema:
+ """
+ Infer arrow table schema from file without loading whole file into memory.
+ Useful especially while having very big files.
+ """
+ with pa.memory_map(filename) as memory_mapped_stream:
+ schema = pa.ipc.open_stream(memory_mapped_stream).schema
+ return schema
+
+
+def _memory_mapped_arrow_table_from_file(filename: str) -> pa.Table:
+ opened_stream = _memory_mapped_record_batch_reader_from_file(filename)
+ pa_table = opened_stream.read_all()
+ return pa_table
+
+
+def _deepcopy(x, memo: dict):
+ """deepcopy a regular class instance"""
+ cls = x.__class__
+ result = cls.__new__(cls)
+ memo[id(x)] = result
+ for k, v in x.__dict__.items():
+ setattr(result, k, copy.deepcopy(v, memo))
+ return result
+
+
+def _interpolation_search(arr: list[int], x: int) -> int:
+ """
+ Return the position i of a sorted array so that arr[i] <= x < arr[i+1]
+
+ Args:
+ arr (`List[int]`): non-empty sorted list of integers
+ x (`int`): query
+
+ Returns:
+ `int`: the position i so that arr[i] <= x < arr[i+1]
+
+ Raises:
+ `IndexError`: if the array is empty or if the query is outside the array values
+ """
+ i, j = 0, len(arr) - 1
+ while i < j and arr[i] <= x < arr[j]:
+ k = i + ((j - i) * (x - arr[i]) // (arr[j] - arr[i]))
+ if arr[k] <= x < arr[k + 1]:
+ return k
+ elif arr[k] < x:
+ i, j = k + 1, j
+ else:
+ i, j = i, k
+ raise IndexError(f"Invalid query '{x}' for size {arr[-1] if len(arr) else 'none'}.")
+
+
+class IndexedTableMixin:
+ def __init__(self, table: pa.Table):
+ self._schema: pa.Schema = table.schema
+ self._batches: list[pa.RecordBatch] = [
+ recordbatch for recordbatch in table.to_batches() if len(recordbatch) > 0
+ ]
+ self._offsets: np.ndarray = np.cumsum([0] + [len(b) for b in self._batches], dtype=np.int64)
+
+ def fast_gather(self, indices: Union[list[int], np.ndarray]) -> pa.Table:
+ """
+ Create a pa.Table by gathering the records at the records at the specified indices. Should be faster
+ than pa.concat_tables(table.fast_slice(int(i) % table.num_rows, 1) for i in indices) since NumPy can compute
+ the binary searches in parallel, highly optimized C
+ """
+ if not len(indices):
+ raise ValueError("Indices must be non-empty")
+ batch_indices = np.searchsorted(self._offsets, indices, side="right") - 1
+ return pa.Table.from_batches(
+ [
+ self._batches[batch_idx].slice(i - self._offsets[batch_idx], 1)
+ for batch_idx, i in zip(batch_indices, indices)
+ ],
+ schema=self._schema,
+ )
+
+ def fast_slice(self, offset=0, length=None) -> pa.Table:
+ """
+ Slice the Table using interpolation search.
+ The behavior is the same as `pyarrow.Table.slice` but it's significantly faster.
+
+ Interpolation search is used to find the start and end indexes of the batches we want to keep.
+ The batches to keep are then concatenated to form the sliced Table.
+ """
+ if offset < 0:
+ raise IndexError("Offset must be non-negative")
+ elif offset >= self._offsets[-1] or (length is not None and length <= 0):
+ return pa.Table.from_batches([], schema=self._schema)
+ i = _interpolation_search(self._offsets, offset)
+ if length is None or length + offset >= self._offsets[-1]:
+ batches = self._batches[i:]
+ batches[0] = batches[0].slice(offset - self._offsets[i])
+ else:
+ j = _interpolation_search(self._offsets, offset + length - 1)
+ batches = self._batches[i : j + 1]
+ batches[-1] = batches[-1].slice(0, offset + length - self._offsets[j])
+ batches[0] = batches[0].slice(offset - self._offsets[i])
+ return pa.Table.from_batches(batches, schema=self._schema)
+
+
+class Table(IndexedTableMixin):
+ """
+ Wraps a pyarrow Table by using composition.
+ This is the base class for `InMemoryTable`, `MemoryMappedTable` and `ConcatenationTable`.
+
+ It implements all the basic attributes/methods of the pyarrow Table class except
+ the Table transforms: `slice, filter, flatten, combine_chunks, cast, add_column,
+ append_column, remove_column, set_column, rename_columns` and `drop`.
+
+ The implementation of these methods differs for the subclasses.
+ """
+
+ def __init__(self, table: pa.Table):
+ super().__init__(table)
+ self.table = table
+
+ def __deepcopy__(self, memo: dict):
+ # arrow tables are immutable, so there's no need to copy self.table
+ # moreover calling deepcopy on a pyarrow table seems to make pa.total_allocated_bytes() decrease for some reason
+ # by adding it to the memo, self.table won't be copied
+ memo[id(self.table)] = self.table
+ # same for the recordbatches used by the index
+ memo[id(self._batches)] = list(self._batches)
+ return _deepcopy(self, memo)
+
+ def validate(self, *args, **kwargs):
+ """
+ Perform validation checks. An exception is raised if validation fails.
+
+ By default only cheap validation checks are run. Pass `full=True`
+ for thorough validation checks (potentially `O(n)`).
+
+ Args:
+ full (`bool`, defaults to `False`):
+ If `True`, run expensive checks, otherwise cheap checks only.
+
+ Raises:
+ `pa.lib.ArrowInvalid`: if validation fails
+ """
+ return self.table.validate(*args, **kwargs)
+
+ def equals(self, *args, **kwargs):
+ """
+ Check if contents of two tables are equal.
+
+ Args:
+ other ([`~datasets.table.Table`]):
+ Table to compare against.
+ check_metadata `bool`, defaults to `False`):
+ Whether schema metadata equality should be checked as well.
+
+ Returns:
+ `bool`
+ """
+ args = tuple(arg.table if isinstance(arg, Table) else arg for arg in args)
+ kwargs = {k: v.table if isinstance(v, Table) else v for k, v in kwargs}
+ return self.table.equals(*args, **kwargs)
+
+ def to_batches(self, *args, **kwargs):
+ """
+ Convert Table to list of (contiguous) `RecordBatch` objects.
+
+ Args:
+ max_chunksize (`int`, defaults to `None`):
+ Maximum size for `RecordBatch` chunks. Individual chunks may be
+ smaller depending on the chunk layout of individual columns.
+
+ Returns:
+ `List[pyarrow.RecordBatch]`
+ """
+ return self.table.to_batches(*args, **kwargs)
+
+ def to_pydict(self, *args, **kwargs):
+ """
+ Convert the Table to a `dict` or `OrderedDict`.
+
+ Returns:
+ `dict`
+ """
+ return self.table.to_pydict(*args, **kwargs)
+
+ def to_pylist(self, *args, **kwargs):
+ """
+ Convert the Table to a list
+
+ Returns:
+ `list`
+ """
+ return self.table.to_pylist(*args, **kwargs)
+
+ def to_pandas(self, *args, **kwargs):
+ """
+ Convert to a pandas-compatible NumPy array or DataFrame, as appropriate.
+
+ Args:
+ memory_pool (`MemoryPool`, defaults to `None`):
+ Arrow MemoryPool to use for allocations. Uses the default memory
+ pool is not passed.
+ strings_to_categorical (`bool`, defaults to `False`):
+ Encode string (UTF8) and binary types to `pandas.Categorical`.
+ categories (`list`, defaults to `empty`):
+ List of fields that should be returned as `pandas.Categorical`. Only
+ applies to table-like data structures.
+ zero_copy_only (`bool`, defaults to `False`):
+ Raise an `ArrowException` if this function call would require copying
+ the underlying data.
+ integer_object_nulls (`bool`, defaults to `False`):
+ Cast integers with nulls to objects.
+ date_as_object (`bool`, defaults to `True`):
+ Cast dates to objects. If `False`, convert to `datetime64[ns]` dtype.
+ timestamp_as_object (`bool`, defaults to `False`):
+ Cast non-nanosecond timestamps (`np.datetime64`) to objects. This is
+ useful if you have timestamps that don't fit in the normal date
+ range of nanosecond timestamps (1678 CE-2262 CE).
+ If `False`, all timestamps are converted to `datetime64[ns]` dtype.
+ use_threads (`bool`, defaults to `True`):
+ Whether to parallelize the conversion using multiple threads.
+ deduplicate_objects (`bool`, defaults to `False`):
+ Do not create multiple copies Python objects when created, to save
+ on memory use. Conversion will be slower.
+ ignore_metadata (`bool`, defaults to `False`):
+ If `True`, do not use the 'pandas' metadata to reconstruct the
+ DataFrame index, if present.
+ safe (`bool`, defaults to `True`):
+ For certain data types, a cast is needed in order to store the
+ data in a pandas DataFrame or Series (e.g. timestamps are always
+ stored as nanoseconds in pandas). This option controls whether it
+ is a safe cast or not.
+ split_blocks (`bool`, defaults to `False`):
+ If `True`, generate one internal "block" for each column when
+ creating a pandas.DataFrame from a `RecordBatch` or `Table`. While this
+ can temporarily reduce memory note that various pandas operations
+ can trigger "consolidation" which may balloon memory use.
+ self_destruct (`bool`, defaults to `False`):
+ EXPERIMENTAL: If `True`, attempt to deallocate the originating Arrow
+ memory while converting the Arrow object to pandas. If you use the
+ object after calling `to_pandas` with this option it will crash your
+ program.
+ types_mapper (`function`, defaults to `None`):
+ A function mapping a pyarrow DataType to a pandas `ExtensionDtype`.
+ This can be used to override the default pandas type for conversion
+ of built-in pyarrow types or in absence of `pandas_metadata` in the
+ Table schema. The function receives a pyarrow DataType and is
+ expected to return a pandas `ExtensionDtype` or `None` if the
+ default conversion should be used for that type. If you have
+ a dictionary mapping, you can pass `dict.get` as function.
+
+ Returns:
+ `pandas.Series` or `pandas.DataFrame`: `pandas.Series` or `pandas.DataFrame` depending on type of object
+ """
+ return self.table.to_pandas(*args, **kwargs)
+
+ def to_string(self, *args, **kwargs):
+ return self.table.to_string(*args, **kwargs)
+
+ def to_reader(self, max_chunksize: Optional[int] = None):
+ """
+ Convert the Table to a RecordBatchReader.
+
+ Note that this method is zero-copy, it merely exposes the same data under a different API.
+
+ Args:
+ max_chunksize (`int`, defaults to `None`)
+ Maximum size for RecordBatch chunks. Individual chunks may be smaller depending
+ on the chunk layout of individual columns.
+
+ Returns:
+ `pyarrow.RecordBatchReader`
+ """
+ return self.table.to_reader(max_chunksize=max_chunksize)
+
+ def field(self, *args, **kwargs):
+ """
+ Select a schema field by its column name or numeric index.
+
+ Args:
+ i (`Union[int, str]`):
+ The index or name of the field to retrieve.
+
+ Returns:
+ `pyarrow.Field`
+ """
+ return self.table.field(*args, **kwargs)
+
+ def column(self, *args, **kwargs):
+ """
+ Select a column by its column name, or numeric index.
+
+ Args:
+ i (`Union[int, str]`):
+ The index or name of the column to retrieve.
+
+ Returns:
+ `pyarrow.ChunkedArray`
+ """
+ return self.table.column(*args, **kwargs)
+
+ def itercolumns(self, *args, **kwargs):
+ """
+ Iterator over all columns in their numerical order.
+
+ Yields:
+ `pyarrow.ChunkedArray`
+ """
+ return self.table.itercolumns(*args, **kwargs)
+
+ @property
+ def schema(self):
+ """
+ Schema of the table and its columns.
+
+ Returns:
+ `pyarrow.Schema`
+ """
+ return self.table.schema
+
+ @property
+ def columns(self):
+ """
+ List of all columns in numerical order.
+
+ Returns:
+ `List[pa.ChunkedArray]`
+ """
+ return self.table.columns
+
+ @property
+ def num_columns(self):
+ """
+ Number of columns in this table.
+
+ Returns:
+ int
+ """
+ return self.table.num_columns
+
+ @property
+ def num_rows(self):
+ """
+ Number of rows in this table.
+
+ Due to the definition of a table, all columns have the same number of
+ rows.
+
+ Returns:
+ int
+ """
+ return self.table.num_rows
+
+ @property
+ def shape(self):
+ """
+ Dimensions of the table: (#rows, #columns).
+
+ Returns:
+ `(int, int)`: Number of rows and number of columns.
+ """
+ return self.table.shape
+
+ @property
+ def nbytes(self):
+ """
+ Total number of bytes consumed by the elements of the table.
+ """
+ return self.table.nbytes
+
+ @property
+ def column_names(self):
+ """
+ Names of the table's columns.
+ """
+ return self.table.column_names
+
+ def __eq__(self, other):
+ return self.equals(other)
+
+ def __getitem__(self, i):
+ return self.table[i]
+
+ def __len__(self):
+ return len(self.table)
+
+ def __repr__(self):
+ return self.table.__repr__().replace("pyarrow.Table", self.__class__.__name__)
+
+ def __str__(self):
+ return self.table.__str__().replace("pyarrow.Table", self.__class__.__name__)
+
+ def slice(self, *args, **kwargs):
+ """
+ Compute zero-copy slice of this Table.
+
+ Args:
+ offset (`int`, defaults to `0`):
+ Offset from start of table to slice.
+ length (`int`, defaults to `None`):
+ Length of slice (default is until end of table starting from
+ offset).
+
+ Returns:
+ `datasets.table.Table`
+ """
+ raise NotImplementedError()
+
+ def filter(self, *args, **kwargs):
+ """
+ Select records from a Table. See `pyarrow.compute.filter` for full usage.
+ """
+ raise NotImplementedError()
+
+ def flatten(self, *args, **kwargs):
+ """
+ Flatten this Table. Each column with a struct type is flattened
+ into one column per struct field. Other columns are left unchanged.
+
+ Args:
+ memory_pool (`MemoryPool`, defaults to `None`):
+ For memory allocations, if required, otherwise use default pool.
+
+ Returns:
+ `datasets.table.Table`
+ """
+ raise NotImplementedError()
+
+ def combine_chunks(self, *args, **kwargs):
+ """
+ Make a new table by combining the chunks this table has.
+
+ All the underlying chunks in the `ChunkedArray` of each column are
+ concatenated into zero or one chunk.
+
+ Args:
+ memory_pool (`MemoryPool`, defaults to `None`):
+ For memory allocations, if required, otherwise use default pool.
+
+ Returns:
+ `datasets.table.Table`
+ """
+ raise NotImplementedError()
+
+ def cast(self, *args, **kwargs):
+ """
+ Cast table values to another schema.
+
+ Args:
+ target_schema (`Schema`):
+ Schema to cast to, the names and order of fields must match.
+ safe (`bool`, defaults to `True`):
+ Check for overflows or other unsafe conversions.
+
+ Returns:
+ `datasets.table.Table`
+ """
+ raise NotImplementedError()
+
+ def replace_schema_metadata(self, *args, **kwargs):
+ """
+ EXPERIMENTAL: Create shallow copy of table by replacing schema
+ key-value metadata with the indicated new metadata (which may be None,
+ which deletes any existing metadata
+
+ Args:
+ metadata (`dict`, defaults to `None`):
+
+ Returns:
+ `datasets.table.Table`: shallow_copy
+ """
+ raise NotImplementedError()
+
+ def add_column(self, *args, **kwargs):
+ """
+ Add column to Table at position.
+
+ A new table is returned with the column added, the original table
+ object is left unchanged.
+
+ Args:
+ i (`int`):
+ Index to place the column at.
+ field_ (`Union[str, pyarrow.Field]`):
+ If a string is passed then the type is deduced from the column
+ data.
+ column (`Union[pyarrow.Array, List[pyarrow.Array]]`):
+ Column data.
+
+ Returns:
+ `datasets.table.Table`: New table with the passed column added.
+ """
+ raise NotImplementedError()
+
+ def append_column(self, *args, **kwargs):
+ """
+ Append column at end of columns.
+
+ Args:
+ field_ (`Union[str, pyarrow.Field]`):
+ If a string is passed then the type is deduced from the column
+ data.
+ column (`Union[pyarrow.Array, List[pyarrow.Array]]`):
+ Column data.
+
+ Returns:
+ `datasets.table.Table`: New table with the passed column added.
+ """
+ raise NotImplementedError()
+
+ def remove_column(self, *args, **kwargs):
+ """
+ Create new Table with the indicated column removed.
+
+ Args:
+ i (`int`):
+ Index of column to remove.
+
+ Returns:
+ `datasets.table.Table`: New table without the column.
+ """
+ raise NotImplementedError()
+
+ def set_column(self, *args, **kwargs):
+ """
+ Replace column in Table at position.
+
+ Args:
+ i (`int`):
+ Index to place the column at.
+ field_ (`Union[str, pyarrow.Field]`):
+ If a string is passed then the type is deduced from the column
+ data.
+ column (`Union[pyarrow.Array, List[pyarrow.Array]]`):
+ Column data.
+
+ Returns:
+ `datasets.table.Table`: New table with the passed column set.
+ """
+ raise NotImplementedError()
+
+ def rename_columns(self, *args, **kwargs):
+ """
+ Create new table with columns renamed to provided names.
+ """
+ raise NotImplementedError()
+
+ def drop(self, *args, **kwargs):
+ """
+ Drop one or more columns and return a new table.
+
+ Args:
+ columns (`List[str]`):
+ List of field names referencing existing columns.
+
+ Raises:
+ `KeyError` : if any of the passed columns name are not existing.
+
+ Returns:
+ `datasets.table.Table`: New table without the columns.
+ """
+ raise NotImplementedError()
+
+ def select(self, *args, **kwargs):
+ """
+ Select columns of the table.
+
+ Returns a new table with the specified columns, and metadata preserved.
+
+ Args:
+ columns (:obj:`Union[List[str], List[int]]`):
+ The column names or integer indices to select.
+
+ Returns:
+ `datasets.table.Table`: table with only a subset of the columns
+ """
+ raise NotImplementedError()
+
+
+class TableBlock(Table):
+ """
+ `TableBlock` is the allowed class inside a `ConcanetationTable`.
+ Only `MemoryMappedTable` and `InMemoryTable` are `TableBlock`.
+ This is because we don't want a `ConcanetationTable` made out of other `ConcanetationTables`.
+ """
+
+ pass
+
+
+class InMemoryTable(TableBlock):
+ """
+ The table is said in-memory when it is loaded into the user's RAM.
+
+ Pickling it does copy all the data using memory.
+ Its implementation is simple and uses the underlying pyarrow Table methods directly.
+
+ This is different from the `MemoryMapped` table, for which pickling doesn't copy all the
+ data in memory. For a `MemoryMapped`, unpickling instead reloads the table from the disk.
+
+ `InMemoryTable` must be used when data fit in memory, while `MemoryMapped` are reserved for
+ data bigger than memory or when you want the memory footprint of your application to
+ stay low.
+ """
+
+ @classmethod
+ def from_file(cls, filename: str):
+ table = _in_memory_arrow_table_from_file(filename)
+ return cls(table)
+
+ @classmethod
+ def from_buffer(cls, buffer: pa.Buffer):
+ table = _in_memory_arrow_table_from_buffer(buffer)
+ return cls(table)
+
+ @classmethod
+ def from_pandas(cls, *args, **kwargs):
+ """
+ Convert pandas.DataFrame to an Arrow Table.
+
+ The column types in the resulting Arrow Table are inferred from the
+ dtypes of the pandas.Series in the DataFrame. In the case of non-object
+ Series, the NumPy dtype is translated to its Arrow equivalent. In the
+ case of `object`, we need to guess the datatype by looking at the
+ Python objects in this Series.
+
+ Be aware that Series of the `object` dtype don't carry enough
+ information to always lead to a meaningful Arrow type. In the case that
+ we cannot infer a type, e.g. because the DataFrame is of length 0 or
+ the Series only contains `None/nan` objects, the type is set to
+ null. This behavior can be avoided by constructing an explicit schema
+ and passing it to this function.
+
+ Args:
+ df (`pandas.DataFrame`):
+ schema (`pyarrow.Schema`, *optional*):
+ The expected schema of the Arrow Table. This can be used to
+ indicate the type of columns if we cannot infer it automatically.
+ If passed, the output will have exactly this schema. Columns
+ specified in the schema that are not found in the DataFrame columns
+ or its index will raise an error. Additional columns or index
+ levels in the DataFrame which are not specified in the schema will
+ be ignored.
+ preserve_index (`bool`, *optional*):
+ Whether to store the index as an additional column in the resulting
+ `Table`. The default of None will store the index as a column,
+ except for RangeIndex which is stored as metadata only. Use
+ `preserve_index=True` to force it to be stored as a column.
+ nthreads (`int`, defaults to `None` (may use up to system CPU count threads))
+ If greater than 1, convert columns to Arrow in parallel using
+ indicated number of threads.
+ columns (`List[str]`, *optional*):
+ List of column to be converted. If `None`, use all columns.
+ safe (`bool`, defaults to `True`):
+ Check for overflows or other unsafe conversions,
+
+ Returns:
+ `datasets.table.Table`:
+
+ Examples:
+ ```python
+ >>> import pandas as pd
+ >>> import pyarrow as pa
+ >>> df = pd.DataFrame({
+ ... 'int': [1, 2],
+ ... 'str': ['a', 'b']
+ ... })
+ >>> pa.Table.from_pandas(df)
+
+ ```
+ """
+ return cls(pa.Table.from_pandas(*args, **kwargs))
+
+ @classmethod
+ def from_arrays(cls, *args, **kwargs):
+ """
+ Construct a Table from Arrow arrays.
+
+ Args:
+ arrays (`List[Union[pyarrow.Array, pyarrow.ChunkedArray]]`):
+ Equal-length arrays that should form the table.
+ names (`List[str]`, *optional*):
+ Names for the table columns. If not passed, schema must be passed.
+ schema (`Schema`, defaults to `None`):
+ Schema for the created table. If not passed, names must be passed.
+ metadata (`Union[dict, Mapping]`, defaults to `None`):
+ Optional metadata for the schema (if inferred).
+
+ Returns:
+ `datasets.table.Table`
+ """
+ return cls(pa.Table.from_arrays(*args, **kwargs))
+
+ @classmethod
+ def from_pydict(cls, *args, **kwargs):
+ """
+ Construct a Table from Arrow arrays or columns.
+
+ Args:
+ mapping (`Union[dict, Mapping]`):
+ A mapping of strings to Arrays or Python lists.
+ schema (`Schema`, defaults to `None`):
+ If not passed, will be inferred from the Mapping values
+ metadata (`Union[dict, Mapping]`, defaults to `None`):
+ Optional metadata for the schema (if inferred).
+
+ Returns:
+ `datasets.table.Table`
+ """
+ return cls(pa.Table.from_pydict(*args, **kwargs))
+
+ @classmethod
+ def from_pylist(cls, mapping, *args, **kwargs):
+ """
+ Construct a Table from list of rows / dictionaries.
+
+ Args:
+ mapping (`List[dict]`):
+ A mapping of strings to row values.
+ schema (`Schema`, defaults to `None`):
+ If not passed, will be inferred from the Mapping values
+ metadata (`Union[dict, Mapping]`, defaults to `None`):
+ Optional metadata for the schema (if inferred).
+
+ Returns:
+ `datasets.table.Table`
+ """
+ return cls(pa.Table.from_pylist(mapping, *args, **kwargs))
+
+ @classmethod
+ def from_batches(cls, *args, **kwargs):
+ """
+ Construct a Table from a sequence or iterator of Arrow `RecordBatches`.
+
+ Args:
+ batches (`Union[Sequence[pyarrow.RecordBatch], Iterator[pyarrow.RecordBatch]]`):
+ Sequence of `RecordBatch` to be converted, all schemas must be equal.
+ schema (`Schema`, defaults to `None`):
+ If not passed, will be inferred from the first `RecordBatch`.
+
+ Returns:
+ `datasets.table.Table`:
+ """
+ return cls(pa.Table.from_batches(*args, **kwargs))
+
+ def slice(self, offset=0, length=None):
+ """
+ Compute zero-copy slice of this Table.
+
+ Args:
+ offset (`int`, defaults to `0`):
+ Offset from start of table to slice.
+ length (`int`, defaults to `None`):
+ Length of slice (default is until end of table starting from
+ offset).
+
+ Returns:
+ `datasets.table.Table`
+ """
+ # Use fast slicing here
+ return InMemoryTable(self.fast_slice(offset=offset, length=length))
+
+ def filter(self, *args, **kwargs):
+ """
+ Select records from a Table. See `pyarrow.compute.filter` for full usage.
+ """
+ return InMemoryTable(self.table.filter(*args, **kwargs))
+
+ def flatten(self, *args, **kwargs):
+ """
+ Flatten this Table. Each column with a struct type is flattened
+ into one column per struct field. Other columns are left unchanged.
+
+ Args:
+ memory_pool (`MemoryPool`, defaults to `None`):
+ For memory allocations, if required, otherwise use default pool.
+
+ Returns:
+ `datasets.table.Table`
+ """
+ return InMemoryTable(table_flatten(self.table, *args, **kwargs))
+
+ def combine_chunks(self, *args, **kwargs):
+ """
+ Make a new table by combining the chunks this table has.
+
+ All the underlying chunks in the `ChunkedArray` of each column are
+ concatenated into zero or one chunk.
+
+ Args:
+ memory_pool (`MemoryPool`, defaults to `None`):
+ For memory allocations, if required, otherwise use default pool.
+
+ Returns:
+ `datasets.table.Table`
+ """
+ return InMemoryTable(self.table.combine_chunks(*args, **kwargs))
+
+ def cast(self, *args, **kwargs):
+ """
+ Cast table values to another schema.
+
+ Args:
+ target_schema (`Schema`):
+ Schema to cast to, the names and order of fields must match.
+ safe (`bool`, defaults to `True`):
+ Check for overflows or other unsafe conversions.
+
+ Returns:
+ `datasets.table.Table`
+ """
+ return InMemoryTable(table_cast(self.table, *args, **kwargs))
+
+ def replace_schema_metadata(self, *args, **kwargs):
+ """
+ EXPERIMENTAL: Create shallow copy of table by replacing schema
+ key-value metadata with the indicated new metadata (which may be `None`,
+ which deletes any existing metadata).
+
+ Args:
+ metadata (`dict`, defaults to `None`):
+
+ Returns:
+ `datasets.table.Table`: shallow_copy
+ """
+ return InMemoryTable(self.table.replace_schema_metadata(*args, **kwargs))
+
+ def add_column(self, *args, **kwargs):
+ """
+ Add column to Table at position.
+
+ A new table is returned with the column added, the original table
+ object is left unchanged.
+
+ Args:
+ i (`int`):
+ Index to place the column at.
+ field_ (`Union[str, pyarrow.Field]`):
+ If a string is passed then the type is deduced from the column
+ data.
+ column (`Union[pyarrow.Array, List[pyarrow.Array]]`):
+ Column data.
+
+ Returns:
+ `datasets.table.Table`: New table with the passed column added.
+ """
+ return InMemoryTable(self.table.add_column(*args, **kwargs))
+
+ def append_column(self, *args, **kwargs):
+ """
+ Append column at end of columns.
+
+ Args:
+ field_ (`Union[str, pyarrow.Field]`):
+ If a string is passed then the type is deduced from the column
+ data.
+ column (`Union[pyarrow.Array, List[pyarrow.Array]]`):
+ Column data.
+
+ Returns:
+ `datasets.table.Table`:
+ New table with the passed column added.
+ """
+ return InMemoryTable(self.table.append_column(*args, **kwargs))
+
+ def remove_column(self, *args, **kwargs):
+ """
+ Create new Table with the indicated column removed.
+
+ Args:
+ i (`int`):
+ Index of column to remove.
+
+ Returns:
+ `datasets.table.Table`:
+ New table without the column.
+ """
+ return InMemoryTable(self.table.remove_column(*args, **kwargs))
+
+ def set_column(self, *args, **kwargs):
+ """
+ Replace column in Table at position.
+
+ Args:
+ i (`int`):
+ Index to place the column at.
+ field_ (`Union[str, pyarrow.Field]`):
+ If a string is passed then the type is deduced from the column
+ data.
+ column (`Union[pyarrow.Array, List[pyarrow.Array]]`):
+ Column data.
+
+ Returns:
+ `datasets.table.Table`:
+ New table with the passed column set.
+ """
+ return InMemoryTable(self.table.set_column(*args, **kwargs))
+
+ def rename_columns(self, *args, **kwargs):
+ """
+ Create new table with columns renamed to provided names.
+ """
+ return InMemoryTable(self.table.rename_columns(*args, **kwargs))
+
+ def drop(self, *args, **kwargs):
+ """
+ Drop one or more columns and return a new table.
+
+ Args:
+ columns (`List[str]`):
+ List of field names referencing existing columns.
+
+ Raises:
+ `KeyError` : if any of the passed columns name are not existing.
+
+ Returns:
+ `datasets.table.Table`:
+ New table without the columns.
+ """
+ return InMemoryTable(self.table.drop(*args, **kwargs))
+
+ def select(self, *args, **kwargs):
+ """
+ Select columns of the table.
+
+ Returns a new table with the specified columns, and metadata preserved.
+
+ Args:
+ columns (:obj:`Union[List[str], List[int]]`):
+ The column names or integer indices to select.
+
+ Returns:
+ :class:`datasets.table.Table`: New table with the specified columns, and metadata preserved.
+ """
+ return InMemoryTable(self.table.select(*args, **kwargs))
+
+
+# The MemoryMappedTable needs replays to properly reload tables from the disk
+Replay = tuple[str, tuple, dict]
+
+
+class MemoryMappedTable(TableBlock):
+ """
+ The table is said memory mapped when it doesn't use the user's RAM but loads the data
+ from the disk instead.
+
+ Pickling it doesn't copy the data into memory.
+ Instead, only the path to the memory mapped arrow file is pickled, as well as the list
+ of transforms to "replay" when reloading the table from the disk.
+
+ Its implementation requires to store an history of all the transforms that were applied
+ to the underlying pyarrow Table, so that they can be "replayed" when reloading the Table
+ from the disk.
+
+ This is different from the `InMemoryTable` table, for which pickling does copy all the
+ data in memory.
+
+ `InMemoryTable` must be used when data fit in memory, while `MemoryMapped` are reserved for
+ data bigger than memory or when you want the memory footprint of your application to
+ stay low.
+ """
+
+ def __init__(self, table: pa.Table, path: str, replays: Optional[list[Replay]] = None):
+ super().__init__(table)
+ self.path = os.path.abspath(path)
+ self.replays: list[Replay] = replays if replays is not None else []
+
+ @classmethod
+ def from_file(cls, filename: str, replays=None):
+ table = _memory_mapped_arrow_table_from_file(filename)
+ table = cls._apply_replays(table, replays)
+ return cls(table, filename, replays)
+
+ def __getstate__(self):
+ return {"path": self.path, "replays": self.replays}
+
+ def __setstate__(self, state):
+ path = state["path"]
+ replays = state["replays"]
+ table = _memory_mapped_arrow_table_from_file(path)
+ table = self._apply_replays(table, replays)
+ MemoryMappedTable.__init__(self, table, path=path, replays=replays)
+
+ @staticmethod
+ def _apply_replays(table: pa.Table, replays: Optional[list[Replay]] = None) -> pa.Table:
+ if replays is not None:
+ for name, args, kwargs in replays:
+ if name == "cast":
+ table = table_cast(table, *args, **kwargs)
+ elif name == "flatten":
+ table = table_flatten(table, *args, **kwargs)
+ else:
+ table = getattr(table, name)(*args, **kwargs)
+ return table
+
+ def _append_replay(self, replay: Replay) -> list[Replay]:
+ replays = copy.deepcopy(self.replays)
+ replays.append(replay)
+ return replays
+
+ def slice(self, offset=0, length=None):
+ """
+ Compute zero-copy slice of this Table.
+
+ Args:
+ offset (`int`, defaults to `0`):
+ Offset from start of table to slice.
+ length (`int`, defaults to `None`):
+ Length of slice (default is until end of table starting from
+ offset).
+
+ Returns:
+ `datasets.table.Table`
+ """
+ replay = ("slice", (offset, length), {})
+ replays = self._append_replay(replay)
+ # Use fast slicing here
+ return MemoryMappedTable(self.fast_slice(offset=offset, length=length), self.path, replays)
+
+ def filter(self, *args, **kwargs):
+ """
+ Select records from a Table. See `pyarrow.compute.filter` for full usage.
+ """
+ replay = ("filter", copy.deepcopy(args), copy.deepcopy(kwargs))
+ replays = self._append_replay(replay)
+ return MemoryMappedTable(self.table.filter(*args, **kwargs), self.path, replays)
+
+ def flatten(self, *args, **kwargs):
+ """
+ Flatten this Table. Each column with a struct type is flattened
+ into one column per struct field. Other columns are left unchanged.
+
+ Args:
+ memory_pool (`MemoryPool`, defaults to `None`):
+ For memory allocations, if required, otherwise use default pool.
+
+ Returns:
+ `datasets.table.Table`
+ """
+ replay = ("flatten", copy.deepcopy(args), copy.deepcopy(kwargs))
+ replays = self._append_replay(replay)
+ return MemoryMappedTable(table_flatten(self.table, *args, **kwargs), self.path, replays)
+
+ def combine_chunks(self, *args, **kwargs):
+ """
+ Make a new table by combining the chunks this table has.
+
+ All the underlying chunks in the ChunkedArray of each column are
+ concatenated into zero or one chunk.
+
+ Args:
+ memory_pool (`MemoryPool`, defaults to `None`):
+ For memory allocations, if required, otherwise use default pool.
+
+ Returns:
+ `datasets.table.Table`
+ """
+ replay = ("combine_chunks", copy.deepcopy(args), copy.deepcopy(kwargs))
+ replays = self._append_replay(replay)
+ return MemoryMappedTable(self.table.combine_chunks(*args, **kwargs), self.path, replays)
+
+ def cast(self, *args, **kwargs):
+ """
+ Cast table values to another schema
+
+ Args:
+ target_schema (`Schema`):
+ Schema to cast to, the names and order of fields must match.
+ safe (`bool`, defaults to `True`):
+ Check for overflows or other unsafe conversions.
+
+ Returns:
+ `datasets.table.Table`
+ """
+ replay = ("cast", copy.deepcopy(args), copy.deepcopy(kwargs))
+ replays = self._append_replay(replay)
+ return MemoryMappedTable(table_cast(self.table, *args, **kwargs), self.path, replays)
+
+ def replace_schema_metadata(self, *args, **kwargs):
+ """
+ EXPERIMENTAL: Create shallow copy of table by replacing schema
+ key-value metadata with the indicated new metadata (which may be None,
+ which deletes any existing metadata.
+
+ Args:
+ metadata (`dict`, defaults to `None`):
+
+ Returns:
+ `datasets.table.Table`: shallow_copy
+ """
+ replay = ("replace_schema_metadata", copy.deepcopy(args), copy.deepcopy(kwargs))
+ replays = self._append_replay(replay)
+ return MemoryMappedTable(self.table.replace_schema_metadata(*args, **kwargs), self.path, replays)
+
+ def add_column(self, *args, **kwargs):
+ """
+ Add column to Table at position.
+
+ A new table is returned with the column added, the original table
+ object is left unchanged.
+
+ Args:
+ i (`int`):
+ Index to place the column at.
+ field_ (`Union[str, pyarrow.Field]`):
+ If a string is passed then the type is deduced from the column
+ data.
+ column (`Union[pyarrow.Array, List[pyarrow.Array]]`):
+ Column data.
+
+ Returns:
+ `datasets.table.Table`: New table with the passed column added.
+ """
+ replay = ("add_column", copy.deepcopy(args), copy.deepcopy(kwargs))
+ replays = self._append_replay(replay)
+ return MemoryMappedTable(self.table.add_column(*args, **kwargs), self.path, replays)
+
+ def append_column(self, *args, **kwargs):
+ """
+ Append column at end of columns.
+
+ Args:
+ field_ (`Union[str, pyarrow.Field]`):
+ If a string is passed then the type is deduced from the column
+ data.
+ column (`Union[pyarrow.Array, List[pyarrow.Array]]`):
+ Column data.
+
+ Returns:
+ `datasets.table.Table`:
+ New table with the passed column added.
+ """
+ replay = ("append_column", copy.deepcopy(args), copy.deepcopy(kwargs))
+ replays = self._append_replay(replay)
+ return MemoryMappedTable(self.table.append_column(*args, **kwargs), self.path, replays)
+
+ def remove_column(self, *args, **kwargs):
+ """
+ Create new Table with the indicated column removed.
+
+ Args:
+ i (`int`):
+ Index of column to remove.
+
+ Returns:
+ `datasets.table.Table`:
+ New table without the column.
+ """
+ replay = ("remove_column", copy.deepcopy(args), copy.deepcopy(kwargs))
+ replays = self._append_replay(replay)
+ return MemoryMappedTable(self.table.remove_column(*args, **kwargs), self.path, replays)
+
+ def set_column(self, *args, **kwargs):
+ """
+ Replace column in Table at position.
+
+ Args:
+ i (`int`):
+ Index to place the column at.
+ field_ (`Union[str, pyarrow.Field]`):
+ If a string is passed then the type is deduced from the column
+ data.
+ column (`Union[pyarrow.Array, List[pyarrow.Array]]`):
+ Column data.
+
+ Returns:
+ `datasets.table.Table`:
+ New table with the passed column set.
+ """
+ replay = ("set_column", copy.deepcopy(args), copy.deepcopy(kwargs))
+ replays = self._append_replay(replay)
+ return MemoryMappedTable(self.table.set_column(*args, **kwargs), self.path, replays)
+
+ def rename_columns(self, *args, **kwargs):
+ """
+ Create new table with columns renamed to provided names.
+ """
+ replay = ("rename_columns", copy.deepcopy(args), copy.deepcopy(kwargs))
+ replays = self._append_replay(replay)
+ return MemoryMappedTable(self.table.rename_columns(*args, **kwargs), self.path, replays)
+
+ def drop(self, *args, **kwargs):
+ """
+ Drop one or more columns and return a new table.
+
+ Args:
+ columns (`List[str]`):
+ List of field names referencing existing columns.
+
+ Raises:
+ `KeyError` : if any of the passed columns name are not existing.
+
+ Returns:
+ `datasets.table.Table`:
+ New table without the columns.
+ """
+ replay = ("drop", copy.deepcopy(args), copy.deepcopy(kwargs))
+ replays = self._append_replay(replay)
+ return MemoryMappedTable(self.table.drop(*args, **kwargs), self.path, replays)
+
+ def select(self, *args, **kwargs):
+ """
+ Select columns of the table.
+
+ Returns a new table with the specified columns, and metadata preserved.
+
+ Args:
+ columns (:obj:`Union[List[str], List[int]]`):
+ The column names or integer indices to select.
+
+ Returns:
+ :class:`datasets.table.Table`: New table with the specified columns, and metadata preserved.
+ """
+ replay = ("select", copy.deepcopy(args), copy.deepcopy(kwargs))
+ replays = self._append_replay(replay)
+ return MemoryMappedTable(self.table.select(*args, **kwargs), self.path, replays)
+
+
+# A ConcatenationTable is the concatenation of several tables.
+# The ``blocks`` attributes stores a list of list of blocks.
+# The first axis concatenates the tables along the axis 0 (it appends rows),
+# while the second axis concatenates tables along the axis 1 (it appends columns).
+TableBlockContainer = TypeVar("TableBlockContainer", TableBlock, list[TableBlock], list[list[TableBlock]])
+
+
+class ConcatenationTable(Table):
+ """
+ The table comes from the concatenation of several tables called blocks.
+ It enables concatenation on both axis 0 (append rows) and axis 1 (append columns).
+
+ The underlying tables are called "blocks" and can be either `InMemoryTable`
+ or `MemoryMappedTable` objects.
+ This allows to combine tables that come from memory or that are memory mapped.
+ When a `ConcatenationTable` is pickled, then each block is pickled:
+ - the `InMemoryTable` objects are pickled by copying all the data in memory.
+ - the MemoryMappedTable objects are pickled without copying the data into memory.
+ Instead, only the path to the memory mapped arrow file is pickled, as well as the list
+ of transforms to "replays" when reloading the table from the disk.
+
+ Its implementation requires to store each block separately.
+ The `blocks` attributes stores a list of list of blocks.
+ The first axis concatenates the tables along the axis 0 (it appends rows),
+ while the second axis concatenates tables along the axis 1 (it appends columns).
+
+ If some columns are missing when concatenating on axis 0, they are filled with null values.
+ This is done using `pyarrow.concat_tables(tables, promote=True)`.
+
+ You can access the fully combined table by accessing the `ConcatenationTable.table` attribute,
+ and the blocks by accessing the `ConcatenationTable.blocks` attribute.
+ """
+
+ def __init__(self, table: pa.Table, blocks: list[list[TableBlock]]):
+ super().__init__(table)
+ self.blocks = blocks
+ # Check that all the blocks have the right type.
+ # Only InMemoryTable and MemoryMappedTable are allowed.
+ for subtables in blocks:
+ for subtable in subtables:
+ if not isinstance(subtable, TableBlock):
+ raise TypeError(
+ "The blocks of a ConcatenationTable must be InMemoryTable or MemoryMappedTable objects"
+ f", but got {_short_str(subtable)}."
+ )
+
+ def __getstate__(self):
+ return {"blocks": self.blocks, "schema": self.table.schema}
+
+ def __setstate__(self, state):
+ blocks = state["blocks"]
+ schema = state["schema"]
+ table = self._concat_blocks_horizontally_and_vertically(blocks)
+ if schema is not None and table.schema != schema:
+ # We fix the columns by concatenating with an empty table with the right columns
+ empty_table = pa.Table.from_batches([], schema=schema)
+ # We set promote_options="default" to fill missing columns with null values
+ table = pa.concat_tables([table, empty_table], promote_options="default")
+ ConcatenationTable.__init__(self, table, blocks=blocks)
+
+ @staticmethod
+ def _concat_blocks(blocks: list[Union[TableBlock, pa.Table]], axis: int = 0) -> pa.Table:
+ pa_tables = [table.table if hasattr(table, "table") else table for table in blocks]
+ if axis == 0:
+ # We set promote_options="default" to fill missing columns with null values
+ return pa.concat_tables(pa_tables, promote_options="default")
+ elif axis == 1:
+ for i, table in enumerate(pa_tables):
+ if i == 0:
+ pa_table = table
+ else:
+ for name, col in zip(table.column_names, table.columns):
+ pa_table = pa_table.append_column(name, col)
+ return pa_table
+ else:
+ raise ValueError("'axis' must be either 0 or 1")
+
+ @classmethod
+ def _concat_blocks_horizontally_and_vertically(cls, blocks: list[list[TableBlock]]) -> pa.Table:
+ pa_tables_to_concat_vertically = []
+ for i, tables in enumerate(blocks):
+ if not tables:
+ continue
+ pa_table_horizontally_concatenated = cls._concat_blocks(tables, axis=1)
+ pa_tables_to_concat_vertically.append(pa_table_horizontally_concatenated)
+ return cls._concat_blocks(pa_tables_to_concat_vertically, axis=0)
+
+ @classmethod
+ def _merge_blocks(cls, blocks: TableBlockContainer, axis: Optional[int] = None) -> TableBlockContainer:
+ if axis is not None:
+ merged_blocks = []
+ for is_in_memory, block_group in groupby(blocks, key=lambda x: isinstance(x, InMemoryTable)):
+ if is_in_memory:
+ block_group = [InMemoryTable(cls._concat_blocks(list(block_group), axis=axis))]
+ merged_blocks += list(block_group)
+ else: # both
+ merged_blocks = [cls._merge_blocks(row_block, axis=1) for row_block in blocks]
+ if all(len(row_block) == 1 for row_block in merged_blocks):
+ merged_blocks = cls._merge_blocks(
+ [block for row_block in merged_blocks for block in row_block], axis=0
+ )
+ return merged_blocks
+
+ @classmethod
+ def _consolidate_blocks(cls, blocks: TableBlockContainer) -> TableBlockContainer:
+ if isinstance(blocks, TableBlock):
+ return blocks
+ elif isinstance(blocks[0], TableBlock):
+ return cls._merge_blocks(blocks, axis=0)
+ else:
+ return cls._merge_blocks(blocks)
+
+ @classmethod
+ def from_blocks(cls, blocks: TableBlockContainer) -> "ConcatenationTable":
+ blocks = cls._consolidate_blocks(blocks)
+ if isinstance(blocks, TableBlock):
+ table = blocks
+ return cls(table.table, [[table]])
+ elif isinstance(blocks[0], TableBlock):
+ table = cls._concat_blocks(blocks, axis=0)
+ blocks = [[t] for t in blocks]
+ return cls(table, blocks)
+ else:
+ table = cls._concat_blocks_horizontally_and_vertically(blocks)
+ return cls(table, blocks)
+
+ @classmethod
+ def from_tables(cls, tables: list[Union[pa.Table, Table]], axis: int = 0) -> "ConcatenationTable":
+ """Create `ConcatenationTable` from list of tables.
+
+ Args:
+ tables (list of `Table` or list of `pyarrow.Table`):
+ List of tables.
+ axis (`{0, 1}`, defaults to `0`, meaning over rows):
+ Axis to concatenate over, where `0` means over rows (vertically) and `1` means over columns
+ (horizontally).
+
+
+ """
+
+ def to_blocks(table: Union[pa.Table, Table]) -> list[list[TableBlock]]:
+ if isinstance(table, pa.Table):
+ return [[InMemoryTable(table)]]
+ elif isinstance(table, ConcatenationTable):
+ return copy.deepcopy(table.blocks)
+ else:
+ return [[table]]
+
+ def _slice_row_block(row_block: list[TableBlock], length: int) -> tuple[list[TableBlock], list[TableBlock]]:
+ sliced = [table.slice(0, length) for table in row_block]
+ remainder = [table.slice(length, len(row_block[0]) - length) for table in row_block]
+ return sliced, remainder
+
+ def _split_both_like(
+ result: list[list[TableBlock]], blocks: list[list[TableBlock]]
+ ) -> tuple[list[list[TableBlock]], list[list[TableBlock]]]:
+ """
+ Make sure each row_block contain the same num_rows to be able to concatenate them on axis=1.
+
+ To do so, we modify both blocks sets to have the same row_blocks boundaries.
+ For example, if `result` has 2 row_blocks of 3 rows and `blocks` has 3 row_blocks of 2 rows,
+ we modify both to have 4 row_blocks of size 2, 1, 1 and 2:
+
+ [ x x x | x x x ]
+ + [ y y | y y | y y ]
+ -----------------------------
+ = [ x x | x | x | x x ]
+ [ y y | y | y | y y ]
+
+ """
+ result, blocks = list(result), list(blocks)
+ new_result, new_blocks = [], []
+ while result and blocks:
+ # we slice the longest row block to save two row blocks of same length
+ # and we replace the long row block by its remainder if necessary
+ if len(result[0][0]) > len(blocks[0][0]):
+ new_blocks.append(blocks[0])
+ sliced, result[0] = _slice_row_block(result[0], len(blocks.pop(0)[0]))
+ new_result.append(sliced)
+ elif len(result[0][0]) < len(blocks[0][0]):
+ new_result.append(result[0])
+ sliced, blocks[0] = _slice_row_block(blocks[0], len(result.pop(0)[0]))
+ new_blocks.append(sliced)
+ else:
+ new_result.append(result.pop(0))
+ new_blocks.append(blocks.pop(0))
+ if result or blocks:
+ raise ValueError("Failed to concatenate on axis=1 because tables don't have the same number of rows")
+ return new_result, new_blocks
+
+ def _extend_blocks(
+ result: list[list[TableBlock]], blocks: list[list[TableBlock]], axis: int = 0
+ ) -> list[list[TableBlock]]:
+ if axis == 0:
+ result.extend(blocks)
+ elif axis == 1:
+ # We make sure each row_block have the same num_rows
+ result, blocks = _split_both_like(result, blocks)
+ for i, row_block in enumerate(blocks):
+ result[i].extend(row_block)
+ return result
+
+ blocks = to_blocks(tables[0])
+ for table in tables[1:]:
+ table_blocks = to_blocks(table)
+ blocks = _extend_blocks(blocks, table_blocks, axis=axis)
+ return cls.from_blocks(blocks)
+
+ @property
+ def _slices(self):
+ offset = 0
+ for tables in self.blocks:
+ length = len(tables[0])
+ yield (offset, length)
+ offset += length
+
+ def slice(self, offset=0, length=None):
+ """
+ Compute zero-copy slice of this Table.
+
+ Args:
+ offset (`int`, defaults to `0`):
+ Offset from start of table to slice.
+ length (`int`, defaults to `None`):
+ Length of slice (default is until end of table starting from
+ offset).
+
+ Returns:
+ `datasets.table.Table`
+ """
+ table = self.table.slice(offset, length=length)
+ length = length if length is not None else self.num_rows - offset
+ blocks = []
+ for tables in self.blocks:
+ n_rows = len(tables[0])
+ if length == 0:
+ break
+ elif n_rows <= offset:
+ offset = offset - n_rows
+ elif n_rows <= offset + length:
+ blocks.append([t.slice(offset) for t in tables])
+ length, offset = length + offset - n_rows, 0
+ else:
+ blocks.append([t.slice(offset, length) for t in tables])
+ length, offset = 0, 0
+ return ConcatenationTable(table, blocks)
+
+ def filter(self, mask, *args, **kwargs):
+ """
+ Select records from a Table. See `pyarrow.compute.filter` for full usage.
+ """
+ table = self.table.filter(mask, *args, **kwargs)
+ blocks = []
+ for (offset, length), tables in zip(self._slices, self.blocks):
+ submask = mask.slice(offset, length)
+ blocks.append([t.filter(submask, *args, **kwargs) for t in tables])
+ return ConcatenationTable(table, blocks)
+
+ def flatten(self, *args, **kwargs):
+ """
+ Flatten this Table. Each column with a struct type is flattened
+ into one column per struct field. Other columns are left unchanged.
+
+ Args:
+ memory_pool (`MemoryPool`, defaults to `None`):
+ For memory allocations, if required, otherwise use default pool.
+
+ Returns:
+ `datasets.table.Table`
+ """
+ table = table_flatten(self.table, *args, **kwargs)
+ blocks = []
+ for tables in self.blocks:
+ blocks.append([t.flatten(*args, **kwargs) for t in tables])
+ return ConcatenationTable(table, blocks)
+
+ def combine_chunks(self, *args, **kwargs):
+ """
+ Make a new table by combining the chunks this table has.
+
+ All the underlying chunks in the `ChunkedArray` of each column are
+ concatenated into zero or one chunk.
+
+ Args:
+ memory_pool (`MemoryPool`, defaults to `None`):
+ For memory allocations, if required, otherwise use default pool.
+
+ Returns:
+ `datasets.table.Table`
+ """
+ table = self.table.combine_chunks(*args, **kwargs)
+ blocks = []
+ for tables in self.blocks:
+ blocks.append([t.combine_chunks(*args, **kwargs) for t in tables])
+ return ConcatenationTable(table, blocks)
+
+ def cast(self, target_schema, *args, **kwargs):
+ """
+ Cast table values to another schema.
+
+ Args:
+ target_schema (`Schema`):
+ Schema to cast to, the names and order of fields must match.
+ safe (`bool`, defaults to `True`):
+ Check for overflows or other unsafe conversions.
+
+ Returns:
+ `datasets.table.Table`
+ """
+ from .features import Features
+
+ table = table_cast(self.table, target_schema, *args, **kwargs)
+ target_features = Features.from_arrow_schema(target_schema)
+ blocks = []
+ for subtables in self.blocks:
+ new_tables = []
+ fields = list(target_schema)
+ for subtable in subtables:
+ subfields = []
+ for name in subtable.column_names:
+ subfields.append(fields.pop(next(i for i, field in enumerate(fields) if field.name == name)))
+ subfeatures = Features({subfield.name: target_features[subfield.name] for subfield in subfields})
+ subschema = subfeatures.arrow_schema
+ new_tables.append(subtable.cast(subschema, *args, **kwargs))
+ blocks.append(new_tables)
+ return ConcatenationTable(table, blocks)
+
+ def replace_schema_metadata(self, *args, **kwargs):
+ """
+ EXPERIMENTAL: Create shallow copy of table by replacing schema
+ key-value metadata with the indicated new metadata (which may be `None`,
+ which deletes any existing metadata).
+
+ Args:
+ metadata (`dict`, defaults to `None`):
+
+ Returns:
+ `datasets.table.Table`: shallow_copy
+ """
+ table = self.table.replace_schema_metadata(*args, **kwargs)
+ blocks = []
+ for tables in self.blocks:
+ blocks.append([t.replace_schema_metadata(*args, **kwargs) for t in tables])
+ return ConcatenationTable(table, self.blocks)
+
+ def add_column(self, *args, **kwargs):
+ """
+ Add column to Table at position.
+
+ A new table is returned with the column added, the original table
+ object is left unchanged.
+
+ Args:
+ i (`int`):
+ Index to place the column at.
+ field_ (`Union[str, pyarrow.Field]`):
+ If a string is passed then the type is deduced from the column
+ data.
+ column (`Union[pyarrow.Array, List[pyarrow.Array]]`):
+ Column data.
+
+ Returns:
+ `datasets.table.Table`: New table with the passed column added.
+ """
+ raise NotImplementedError()
+
+ def append_column(self, *args, **kwargs):
+ """
+ Append column at end of columns.
+
+ Args:
+ field_ (`Union[str, pyarrow.Field]`):
+ If a string is passed then the type is deduced from the column
+ data.
+ column (`Union[pyarrow.Array, List[pyarrow.Array]]`):
+ Column data.
+
+ Returns:
+ `datasets.table.Table`:
+ New table with the passed column added.
+ """
+ raise NotImplementedError()
+
+ def remove_column(self, i, *args, **kwargs):
+ """
+ Create new Table with the indicated column removed.
+
+ Args:
+ i (`int`):
+ Index of column to remove.
+
+ Returns:
+ `datasets.table.Table`:
+ New table without the column.
+ """
+ table = self.table.remove_column(i, *args, **kwargs)
+ name = self.table.column_names[i]
+ blocks = []
+ for tables in self.blocks:
+ blocks.append(
+ [
+ t.remove_column(t.column_names.index(name), *args, **kwargs) if name in t.column_names else t
+ for t in tables
+ ]
+ )
+ return ConcatenationTable(table, blocks)
+
+ def set_column(self, *args, **kwargs):
+ """
+ Replace column in Table at position.
+
+ Args:
+ i (`int`):
+ Index to place the column at.
+ field_ (`Union[str, pyarrow.Field]`):
+ If a string is passed then the type is deduced from the column
+ data.
+ column (`Union[pyarrow.Array, List[pyarrow.Array]]`):
+ Column data.
+
+ Returns:
+ `datasets.table.Table`:
+ New table with the passed column set.
+ """
+ raise NotImplementedError()
+
+ def rename_columns(self, names, *args, **kwargs):
+ """
+ Create new table with columns renamed to provided names.
+ """
+ table = self.table.rename_columns(names, *args, **kwargs)
+ names = dict(zip(self.table.column_names, names))
+ blocks = []
+ for tables in self.blocks:
+ blocks.append(
+ [t.rename_columns([names[name] for name in t.column_names], *args, **kwargs) for t in tables]
+ )
+ return ConcatenationTable(table, blocks)
+
+ def drop(self, columns, *args, **kwargs):
+ """
+ Drop one or more columns and return a new table.
+
+ Args:
+ columns (`List[str]`):
+ List of field names referencing existing columns.
+
+ Raises:
+ `KeyError` : if any of the passed columns name are not existing.
+
+ Returns:
+ `datasets.table.Table`:
+ New table without the columns.
+ """
+ table = self.table.drop(columns, *args, **kwargs)
+ blocks = []
+ for tables in self.blocks:
+ blocks.append([t.drop([c for c in columns if c in t.column_names], *args, **kwargs) for t in tables])
+ return ConcatenationTable(table, blocks)
+
+ def select(self, columns, *args, **kwargs):
+ """
+ Select columns of the table.
+
+ Returns a new table with the specified columns, and metadata preserved.
+
+ Args:
+ columns (:obj:`Union[List[str], List[int]]`):
+ The column names or integer indices to select.
+
+ Returns:
+ :class:`datasets.table.Table`: New table with the specified columns, and metadata preserved.
+ """
+ table = self.table.select(columns, *args, **kwargs)
+ blocks = []
+ for tables in self.blocks:
+ blocks.append([t.select([c for c in columns if c in t.column_names], *args, **kwargs) for t in tables])
+ return ConcatenationTable(table, blocks)
+
+
+def concat_tables(tables: list[Table], axis: int = 0) -> Table:
+ """
+ Concatenate tables.
+
+ Args:
+ tables (list of `Table`):
+ List of tables to be concatenated.
+ axis (`{0, 1}`, defaults to `0`, meaning over rows):
+ Axis to concatenate over, where `0` means over rows (vertically) and `1` means over columns
+ (horizontally).
+
+
+ Returns:
+ `datasets.table.Table`:
+ If the number of input tables is > 1, then the returned table is a `datasets.table.ConcatenationTable`.
+ Otherwise if there's only one table, it is returned as is.
+ """
+ tables = list(tables)
+ if len(tables) == 1:
+ return tables[0]
+ return ConcatenationTable.from_tables(tables, axis=axis)
+
+
+def list_table_cache_files(table: Table) -> list[str]:
+ """
+ Get the cache files that are loaded by the table.
+ Cache file are used when parts of the table come from the disk via memory mapping.
+
+ Returns:
+ `List[str]`:
+ A list of paths to the cache files loaded by the table.
+ """
+ if isinstance(table, ConcatenationTable):
+ cache_files = []
+ for subtables in table.blocks:
+ for subtable in subtables:
+ cache_files += list_table_cache_files(subtable)
+ return cache_files
+ elif isinstance(table, MemoryMappedTable):
+ return [table.path]
+ else:
+ return []
+
+
+def _wrap_for_chunked_arrays(func):
+ """Apply the function on each chunk of a `pyarrow.ChunkedArray`, or on the array directly"""
+
+ def wrapper(array, *args, **kwargs):
+ if isinstance(array, pa.ChunkedArray):
+ return pa.chunked_array([func(chunk, *args, **kwargs) for chunk in array.chunks])
+ else:
+ return func(array, *args, **kwargs)
+
+ return wrapper
+
+
+def _are_list_values_of_length(array: pa.ListArray, length: int) -> bool:
+ """Check if all the sub-lists of a `pa.ListArray` have the specified length."""
+ return pc.all(pc.equal(array.value_lengths(), length)).as_py() or array.null_count == len(array)
+
+
+def _combine_list_array_offsets_with_mask(array: pa.ListArray) -> pa.Array:
+ """Add the null bitmap to the offsets of a `pa.ListArray`."""
+ offsets = array.offsets
+ if array.null_count > 0:
+ offsets = pa.concat_arrays(
+ [
+ pc.replace_with_mask(offsets[:-1], array.is_null(), pa.nulls(len(array), pa.int32())),
+ offsets[-1:],
+ ]
+ )
+ return offsets
+
+
+def _storage_type(type: pa.DataType) -> pa.DataType:
+ """Convert a (possibly nested) `pa.ExtensionType` to its storage type."""
+ if isinstance(type, pa.ExtensionType):
+ return _storage_type(type.storage_type)
+ elif isinstance(type, pa.StructType):
+ return pa.struct([pa.field(field.name, _storage_type(field.type)) for field in type])
+ elif isinstance(type, pa.ListType):
+ return pa.list_(_storage_type(type.value_type))
+ elif isinstance(type, pa.FixedSizeListType):
+ return pa.list_(_storage_type(type.value_type), type.list_size)
+ return type
+
+
+def _short_str(value: Any) -> str:
+ out = str(value)
+ if len(out) > 3000:
+ out = out[:1500] + "\n...\n" + out[-1500:]
+ return out
+
+
+@_wrap_for_chunked_arrays
+def array_cast(
+ array: pa.Array, pa_type: pa.DataType, allow_primitive_to_str: bool = True, allow_decimal_to_str: bool = True
+) -> Union[pa.Array, pa.FixedSizeListArray, pa.ListArray, pa.StructArray, pa.ExtensionArray]:
+ """Improved version of `pa.Array.cast`
+
+ It supports casting `pa.StructArray` objects to re-order the fields.
+ It also let you control certain aspects of the casting, e.g. whether
+ to disable casting primitives (`booleans`, `floats` or `ints`) or
+ disable casting decimals to strings.
+
+ Args:
+ array (`pa.Array`):
+ PyArrow array to cast
+ pa_type (`pa.DataType`):
+ Target PyArrow type
+ allow_primitive_to_str (`bool`, defaults to `True`):
+ Whether to allow casting primitives to strings.
+ Defaults to `True`.
+ allow_decimal_to_str (`bool`, defaults to `True`):
+ Whether to allow casting decimals to strings.
+ Defaults to `True`.
+
+ Raises:
+ `pa.ArrowInvalidError`: if the arrow data casting fails
+ `TypeError`: if the target type is not supported according, e.g.
+
+ - if a field is missing
+ - if casting from primitives to strings and `allow_primitive_to_str` is `False`
+ - if casting from decimals to strings and `allow_decimal_to_str` is `False`
+
+ Returns:
+ `List[pyarrow.Array]`: the casted array
+ """
+ _c = partial(array_cast, allow_primitive_to_str=allow_primitive_to_str, allow_decimal_to_str=allow_decimal_to_str)
+ if isinstance(array, pa.ExtensionArray):
+ array = array.storage
+ if isinstance(pa_type, pa.ExtensionType):
+ return pa_type.wrap_array(_c(array, pa_type.storage_type))
+ elif array.type == pa_type:
+ return array
+ elif pa.types.is_struct(array.type):
+ if pa.types.is_struct(pa_type) and ({field.name for field in pa_type} == {field.name for field in array.type}):
+ if array.type.num_fields == 0:
+ return array
+ arrays = [_c(array.field(field.name), field.type) for field in pa_type]
+ return pa.StructArray.from_arrays(arrays, fields=list(pa_type), mask=array.is_null())
+ elif pa.types.is_list(array.type) or pa.types.is_large_list(array.type):
+ if pa.types.is_fixed_size_list(pa_type):
+ if _are_list_values_of_length(array, pa_type.list_size):
+ if array.null_count > 0:
+ # Ensure each null value in the array translates to [null] * pa_type.list_size in the array's values array
+ array_type = array.type
+ storage_type = _storage_type(array_type)
+ if array_type != storage_type:
+ # Temporarily convert to the storage type to support extension types in the slice operation
+ array = _c(array, storage_type)
+ array = pc.list_slice(array, 0, pa_type.list_size, return_fixed_size_list=True)
+ array = _c(array, array_type)
+ else:
+ array = pc.list_slice(array, 0, pa_type.list_size, return_fixed_size_list=True)
+ array_values = array.values
+ return pa.FixedSizeListArray.from_arrays(
+ _c(array_values, pa_type.value_type), pa_type.list_size, mask=array.is_null()
+ )
+ else:
+ array_values = array.values[
+ array.offset * pa_type.list_size : (array.offset + len(array)) * pa_type.list_size
+ ]
+ return pa.FixedSizeListArray.from_arrays(_c(array_values, pa_type.value_type), pa_type.list_size)
+ elif pa.types.is_list(pa_type):
+ # Merge offsets with the null bitmap to avoid the "Null bitmap with offsets slice not supported" ArrowNotImplementedError
+ array_offsets = _combine_list_array_offsets_with_mask(array)
+ return pa.ListArray.from_arrays(array_offsets, _c(array.values, pa_type.value_type))
+ elif pa.types.is_large_list(pa_type):
+ # Merge offsets with the null bitmap to avoid the "Null bitmap with offsets slice not supported" ArrowNotImplementedError
+ array_offsets = _combine_list_array_offsets_with_mask(array)
+ return pa.LargeListArray.from_arrays(array_offsets, _c(array.values, pa_type.value_type))
+ elif pa.types.is_fixed_size_list(array.type):
+ if pa.types.is_fixed_size_list(pa_type):
+ if pa_type.list_size == array.type.list_size:
+ array_values = array.values[
+ array.offset * array.type.list_size : (array.offset + len(array)) * array.type.list_size
+ ]
+ return pa.FixedSizeListArray.from_arrays(
+ _c(array_values, pa_type.value_type), pa_type.list_size, mask=array.is_null()
+ )
+ elif pa.types.is_list(pa_type):
+ array_offsets = (np.arange(len(array) + 1) + array.offset) * array.type.list_size
+ return pa.ListArray.from_arrays(array_offsets, _c(array.values, pa_type.value_type), mask=array.is_null())
+ elif pa.types.is_large_list(pa_type):
+ array_offsets = (np.arange(len(array) + 1) + array.offset) * array.type.list_size
+ return pa.LargeListArray.from_arrays(
+ array_offsets, _c(array.values, pa_type.value_type), mask=array.is_null()
+ )
+ else:
+ if pa.types.is_string(pa_type):
+ if not allow_primitive_to_str and pa.types.is_primitive(array.type):
+ raise TypeError(
+ f"Couldn't cast array of type {_short_str(array.type)} to {_short_str(pa_type)} "
+ f"since allow_primitive_to_str is set to {allow_primitive_to_str} "
+ )
+ if not allow_decimal_to_str and pa.types.is_decimal(array.type):
+ raise TypeError(
+ f"Couldn't cast array of type {_short_str(array.type)} to {_short_str(pa_type)} "
+ f"and allow_decimal_to_str is set to {allow_decimal_to_str}"
+ )
+ if pa.types.is_null(pa_type) and not pa.types.is_null(array.type):
+ raise TypeError(f"Couldn't cast array of type {_short_str(array.type)} to {_short_str(pa_type)}")
+ return array.cast(pa_type)
+ raise TypeError(f"Couldn't cast array of type {_short_str(array.type)} to {_short_str(pa_type)}")
+
+
+@_wrap_for_chunked_arrays
+def cast_array_to_feature(
+ array: pa.Array, feature: "FeatureType", allow_primitive_to_str: bool = True, allow_decimal_to_str: bool = True
+) -> pa.Array:
+ """Cast an array to the arrow type that corresponds to the requested feature type.
+ For custom features like [`Audio`] or [`Image`], it takes into account the "cast_storage" methods
+ they defined to enable casting from other arrow types.
+
+ Args:
+ array (`pa.Array`):
+ The PyArrow array to cast.
+ feature (`datasets.features.FeatureType`):
+ The target feature type.
+ allow_primitive_to_str (`bool`, defaults to `True`):
+ Whether to allow casting primitives to strings.
+ Defaults to `True`.
+ allow_decimal_to_str (`bool`, defaults to `True`):
+ Whether to allow casting decimals to strings.
+ Defaults to `True`.
+
+ Raises:
+ `pa.ArrowInvalidError`: if the arrow data casting fails
+ `TypeError`: if the target type is not supported according, e.g.
+
+ - if a field is missing
+ - if casting from primitives and `allow_primitive_to_str` is `False`
+ - if casting from decimals and `allow_decimal_to_str` is `False`
+
+ Returns:
+ array (`pyarrow.Array`): the casted array
+ """
+ from .features.features import LargeList, List, get_nested_type
+
+ _c = partial(
+ cast_array_to_feature,
+ allow_primitive_to_str=allow_primitive_to_str,
+ allow_decimal_to_str=allow_decimal_to_str,
+ )
+
+ if isinstance(array, pa.ExtensionArray):
+ array = array.storage
+ if hasattr(feature, "cast_storage"):
+ return feature.cast_storage(array)
+
+ if pa.types.is_struct(array.type):
+ # feature must be a dict
+ if isinstance(feature, dict) and (array_fields := {field.name for field in array.type}) <= set(feature):
+ null_array = pa.array([None] * len(array))
+ arrays = [
+ _c(array.field(name) if name in array_fields else null_array, subfeature)
+ for name, subfeature in feature.items()
+ ]
+ return pa.StructArray.from_arrays(arrays, names=list(feature), mask=array.is_null())
+ elif pa.types.is_list(array.type) or pa.types.is_large_list(array.type):
+ # feature must be either List(subfeature) or LargeList(subfeature)
+ if isinstance(feature, LargeList):
+ casted_array_values = _c(array.values, feature.feature)
+ if pa.types.is_large_list(array.type) and casted_array_values.type == array.values.type:
+ # Both array and feature have equal large_list type and values (within the list) type
+ return array
+ else:
+ # Merge offsets with the null bitmap to avoid the "Null bitmap with offsets slice not supported" ArrowNotImplementedError
+ array_offsets = _combine_list_array_offsets_with_mask(array)
+ return pa.LargeListArray.from_arrays(array_offsets, casted_array_values)
+ elif isinstance(feature, List):
+ if feature.length > -1:
+ if _are_list_values_of_length(array, feature.length):
+ if array.null_count > 0:
+ # Ensure each null value in the array translates to [null] * pa_type.list_size in the array's values array
+ array_type = array.type
+ storage_type = _storage_type(array_type)
+ if array_type != storage_type:
+ # Temporarily convert to the storage type to support extension types in the slice operation
+ array = array_cast(
+ array,
+ storage_type,
+ allow_primitive_to_str=allow_primitive_to_str,
+ allow_decimal_to_str=allow_decimal_to_str,
+ )
+ array = pc.list_slice(array, 0, feature.length, return_fixed_size_list=True)
+ array = array_cast(
+ array,
+ array_type,
+ allow_primitive_to_str=allow_primitive_to_str,
+ allow_decimal_to_str=allow_decimal_to_str,
+ )
+ else:
+ array = pc.list_slice(array, 0, feature.length, return_fixed_size_list=True)
+ array_values = array.values
+ casted_array_values = _c(array_values, feature.feature)
+ return pa.FixedSizeListArray.from_arrays(
+ casted_array_values, feature.length, mask=array.is_null()
+ )
+ else:
+ array_values = array.values[
+ array.offset * feature.length : (array.offset + len(array)) * feature.length
+ ]
+ return pa.FixedSizeListArray.from_arrays(_c(array_values, feature.feature), feature.length)
+ else:
+ casted_array_values = _c(array.values, feature.feature)
+ if pa.types.is_list(array.type) and casted_array_values.type == array.values.type:
+ # Both array and feature have equal list type and values (within the list) type
+ return array
+ else:
+ # Merge offsets with the null bitmap to avoid the "Null bitmap with offsets slice not supported" ArrowNotImplementedError
+ array_offsets = _combine_list_array_offsets_with_mask(array)
+ return pa.ListArray.from_arrays(array_offsets, casted_array_values)
+ elif pa.types.is_fixed_size_list(array.type):
+ # feature must be List(subfeature)
+ if isinstance(feature, LargeList):
+ array_offsets = (np.arange(len(array) + 1) + array.offset) * array.type.list_size
+ return pa.LargeListArray.from_arrays(
+ array_offsets, _c(array.values, feature.feature), mask=array.is_null()
+ )
+ elif isinstance(feature, List):
+ if feature.length > -1:
+ if feature.length == array.type.list_size:
+ array_values = array.values[
+ array.offset * array.type.list_size : (array.offset + len(array)) * array.type.list_size
+ ]
+ casted_array_values = _c(array_values, feature.feature)
+ return pa.FixedSizeListArray.from_arrays(casted_array_values, feature.length, mask=array.is_null())
+ else:
+ array_offsets = (np.arange(len(array) + 1) + array.offset) * array.type.list_size
+ return pa.ListArray.from_arrays(array_offsets, _c(array.values, feature.feature), mask=array.is_null())
+ if pa.types.is_null(array.type):
+ return array_cast(
+ array,
+ get_nested_type(feature),
+ allow_primitive_to_str=allow_primitive_to_str,
+ allow_decimal_to_str=allow_decimal_to_str,
+ )
+ elif not isinstance(feature, (List, LargeList, dict)):
+ return array_cast(
+ array,
+ feature(),
+ allow_primitive_to_str=allow_primitive_to_str,
+ allow_decimal_to_str=allow_decimal_to_str,
+ )
+ raise TypeError(f"Couldn't cast array of type\n{_short_str(array.type)}\nto\n{_short_str(feature)}")
+
+
+@_wrap_for_chunked_arrays
+def embed_array_storage(array: pa.Array, feature: "FeatureType", token_per_repo_id=None):
+ """Embed data into an arrays's storage.
+ For custom features like Audio or Image, it takes into account the "embed_storage" methods
+ they define to embed external data (e.g. an image file) into an array.
+
+
+
+ Args:
+ array (`pa.Array`):
+ The PyArrow array in which to embed data.
+ feature (`datasets.features.FeatureType`):
+ Array features.
+
+ Raises:
+ `TypeError`: if the target type is not supported according, e.g.
+
+ - if a field is missing
+
+ Returns:
+ array (`pyarrow.Array`): the casted array
+ """
+ from .features import LargeList, List
+
+ _e = partial(embed_array_storage, token_per_repo_id=token_per_repo_id)
+
+ if isinstance(array, pa.ExtensionArray):
+ array = array.storage
+ if hasattr(feature, "embed_storage"):
+ return feature.embed_storage(array, token_per_repo_id=token_per_repo_id)
+ elif pa.types.is_struct(array.type):
+ # feature must be a dict
+ if isinstance(feature, dict):
+ arrays = [_e(array.field(name), subfeature) for name, subfeature in feature.items()]
+ return pa.StructArray.from_arrays(arrays, names=list(feature), mask=array.is_null())
+ elif pa.types.is_list(array.type):
+ # feature must be either List(subfeature)
+ # Merge offsets with the null bitmap to avoid the "Null bitmap with offsets slice not supported" ArrowNotImplementedError
+ array_offsets = _combine_list_array_offsets_with_mask(array)
+ if isinstance(feature, List) and feature.length == -1:
+ return pa.ListArray.from_arrays(array_offsets, _e(array.values, feature.feature))
+ elif pa.types.is_large_list(array.type):
+ # feature must be LargeList(subfeature)
+ # Merge offsets with the null bitmap to avoid the "Null bitmap with offsets slice not supported" ArrowNotImplementedError
+ array_offsets = _combine_list_array_offsets_with_mask(array)
+ return pa.LargeListArray.from_arrays(array_offsets, _e(array.values, feature.feature))
+ elif pa.types.is_fixed_size_list(array.type):
+ # feature must be List(subfeature)
+ if isinstance(feature, List) and feature.length > -1:
+ array_values = array.values[
+ array.offset * array.type.list_size : (array.offset + len(array)) * array.type.list_size
+ ]
+ embedded_array_values = _e(array_values, feature.feature)
+ return pa.FixedSizeListArray.from_arrays(embedded_array_values, feature.length, mask=array.is_null())
+ if not isinstance(feature, (List, LargeList, dict)):
+ return array
+ raise TypeError(f"Couldn't embed array of type\n{_short_str(array.type)}\nwith\n{_short_str(feature)}")
+
+
+class CastError(ValueError):
+ """When it's not possible to cast an Arrow table to a specific schema or set of features"""
+
+ def __init__(self, *args, table_column_names: list[str], requested_column_names: list[str]) -> None:
+ super().__init__(*args)
+ self.table_column_names = table_column_names
+ self.requested_column_names = requested_column_names
+
+ def __reduce__(self):
+ # Fix unpickling: TypeError: __init__() missing 2 required keyword-only arguments: 'table_column_names' and 'requested_column_names'
+ return partial(
+ CastError, table_column_names=self.table_column_names, requested_column_names=self.requested_column_names
+ ), ()
+
+ def details(self):
+ new_columns = set(self.table_column_names) - set(self.requested_column_names)
+ missing_columns = set(self.requested_column_names) - set(self.table_column_names)
+ if new_columns and missing_columns:
+ return f"there are {len(new_columns)} new columns ({_short_str(new_columns)}) and {len(missing_columns)} missing columns ({_short_str(missing_columns)})."
+ elif new_columns:
+ return f"there are {len(new_columns)} new columns ({_short_str(new_columns)})"
+ else:
+ return f"there are {len(missing_columns)} missing columns ({_short_str(missing_columns)})"
+
+
+def cast_table_to_features(table: pa.Table, features: "Features"):
+ """Cast a table to the arrow schema that corresponds to the requested features.
+
+ Args:
+ table (`pyarrow.Table`):
+ PyArrow table to cast.
+ features ([`Features`]):
+ Target features.
+
+ Returns:
+ table (`pyarrow.Table`): the casted table
+ """
+ if sorted(table.column_names) != sorted(features):
+ raise CastError(
+ f"Couldn't cast\n{_short_str(table.schema)}\nto\n{_short_str(features)}\nbecause column names don't match",
+ table_column_names=table.column_names,
+ requested_column_names=list(features),
+ )
+ arrays = [cast_array_to_feature(table[name], feature) for name, feature in features.items()]
+ return pa.Table.from_arrays(arrays, schema=features.arrow_schema)
+
+
+def cast_table_to_schema(table: pa.Table, schema: pa.Schema):
+ """Cast a table to the arrow schema. Different from `cast_table_to_features`, this method can preserve nullability.
+
+ Args:
+ table (`pa.Table`):
+ PyArrow table to cast.
+ features ([`Features`]):
+ Target features.
+
+ Returns:
+ `pa.Table`: the casted table
+ """
+ from .features import Features
+
+ features = Features.from_arrow_schema(schema)
+ table_column_names = set(table.column_names)
+ if not table_column_names <= set(schema.names):
+ raise CastError(
+ f"Couldn't cast\n{_short_str(table.schema)}\nto\n{_short_str(features)}\nbecause column names don't match",
+ table_column_names=table.column_names,
+ requested_column_names=list(features),
+ )
+ arrays = [
+ cast_array_to_feature(
+ table[name] if name in table_column_names else pa.array([None] * len(table), type=schema.field(name).type),
+ feature,
+ )
+ for name, feature in features.items()
+ ]
+ return pa.Table.from_arrays(arrays, schema=schema)
+
+
+def embed_table_storage(table: pa.Table, token_per_repo_id=None):
+ """Embed external data into a table's storage.
+
+
+
+ Args:
+ table (`pyarrow.Table`):
+ PyArrow table in which to embed data.
+
+ Returns:
+ table (`pyarrow.Table`): the table with embedded data
+ """
+ from .features.features import Features, require_storage_embed
+
+ features = Features.from_arrow_schema(table.schema)
+ arrays = [
+ embed_array_storage(table[name], feature, token_per_repo_id=token_per_repo_id)
+ if require_storage_embed(feature)
+ else table[name]
+ for name, feature in features.items()
+ ]
+ return pa.Table.from_arrays(arrays, schema=features.arrow_schema)
+
+
+def table_cast(table: pa.Table, schema: pa.Schema):
+ """Improved version of `pa.Table.cast`.
+
+ It supports casting to feature types stored in the schema metadata.
+
+ Args:
+ table (`pyarrow.Table`):
+ PyArrow table to cast.
+ schema (`pyarrow.Schema`):
+ Target PyArrow schema.
+
+ Returns:
+ table (`pyarrow.Table`): the casted table
+ """
+ if table.schema != schema:
+ return cast_table_to_schema(table, schema)
+ elif table.schema.metadata != schema.metadata:
+ return table.replace_schema_metadata(schema.metadata)
+ else:
+ return table
+
+
+def table_flatten(table: pa.Table):
+ """Improved version of `pa.Table.flatten`.
+
+ It behaves as `pa.Table.flatten` in a sense it does 1-step flatten of the columns with a struct type into one column per struct field,
+ but updates the metadata and skips decodable features unless the `decode` attribute of these features is set to False.
+
+ Args:
+ table (`pa.Table`):
+ PyArrow table to flatten.
+
+ Returns:
+ `Table`: the flattened table
+ """
+ from .features import Features
+
+ features = Features.from_arrow_schema(table.schema)
+ if any(hasattr(subfeature, "flatten") and subfeature.flatten() == subfeature for subfeature in features.values()):
+ flat_arrays = []
+ flat_column_names = []
+ for field in table.schema:
+ array = table.column(field.name)
+ subfeature = features[field.name]
+ if pa.types.is_struct(field.type) and (
+ not hasattr(subfeature, "flatten") or subfeature.flatten() != subfeature
+ ):
+ flat_arrays.extend(array.flatten())
+ flat_column_names.extend([f"{field.name}.{subfield.name}" for subfield in field.type])
+ else:
+ flat_arrays.append(array)
+ flat_column_names.append(field.name)
+ flat_table = pa.Table.from_arrays(
+ flat_arrays,
+ names=flat_column_names,
+ )
+ else:
+ flat_table = table.flatten()
+ # Preserve complex types in the metadata
+ flat_features = features.flatten(max_depth=2)
+ flat_features = Features({column_name: flat_features[column_name] for column_name in flat_table.column_names})
+ return flat_table.replace_schema_metadata(flat_features.arrow_schema.metadata)
+
+
+def table_visitor(table: pa.Table, function: Callable[[pa.Array], None]):
+ """Visit all arrays in a table and apply a function to them.
+
+ Args:
+ table (`pyarrow.Table`):
+ PyArrow table to visit.
+ function (`Callable[[pa.Array], None]`):
+ Function to apply to each array.
+ """
+ from .features import Features, LargeList, List
+
+ features = Features.from_arrow_schema(table.schema)
+
+ def _visit(array, feature):
+ if isinstance(array, pa.ChunkedArray):
+ for chunk in array.chunks:
+ _visit(chunk, feature)
+ else:
+ if isinstance(array, pa.ExtensionArray):
+ array = array.storage
+ function(array, feature)
+ if pa.types.is_struct(array.type) and not hasattr(feature, "cast_storage"):
+ for name, subfeature in feature.items():
+ _visit(array.field(name), subfeature)
+ elif pa.types.is_list(array.type):
+ if isinstance(feature, (LargeList, List)):
+ _visit(array.values, feature.feature)
+
+ for name, feature in features.items():
+ _visit(table[name], feature)
+
+
+def table_iter(table: Table, batch_size: int, drop_last_batch=False) -> Iterator[pa.Table]:
+ """Iterate over sub-tables of size `batch_size`.
+
+ Args:
+ table (`pyarrow.Table`):
+ PyArrow table to iterate over.
+ batch_size (`int`):
+ Size of each sub-table to yield.
+ drop_last_batch (`bool`, defaults to `False`):
+ Drop the last batch if it is smaller than `batch_size`.
+ """
+ chunks_buffer = []
+ chunks_buffer_size = 0
+ for chunk in table.to_reader(max_chunksize=batch_size):
+ if len(chunk) == 0:
+ continue
+ elif chunks_buffer_size + len(chunk) < batch_size:
+ chunks_buffer.append(chunk)
+ chunks_buffer_size += len(chunk)
+ continue
+ elif chunks_buffer_size + len(chunk) == batch_size:
+ chunks_buffer.append(chunk)
+ yield pa.Table.from_batches(chunks_buffer)
+ chunks_buffer = []
+ chunks_buffer_size = 0
+ else:
+ cropped_chunk_length = batch_size - chunks_buffer_size
+ chunks_buffer.append(chunk.slice(0, cropped_chunk_length))
+ yield pa.Table.from_batches(chunks_buffer)
+ chunks_buffer = [chunk.slice(cropped_chunk_length, len(chunk) - cropped_chunk_length)]
+ chunks_buffer_size = len(chunk) - cropped_chunk_length
+ if not drop_last_batch and chunks_buffer:
+ yield pa.Table.from_batches(chunks_buffer)
diff --git a/venv/lib/python3.10/site-packages/datasets/utils/__init__.py b/venv/lib/python3.10/site-packages/datasets/utils/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..a1c3d2935994956e230a441f3f2d65695ebaaf2d
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/utils/__init__.py
@@ -0,0 +1,25 @@
+# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from . import tqdm as _tqdm # _tqdm is the module
+from .experimental import experimental
+from .info_utils import VerificationMode
+from .logging import disable_progress_bar, enable_progress_bar, is_progress_bar_enabled
+from .tqdm import (
+ are_progress_bars_disabled,
+ disable_progress_bars,
+ enable_progress_bars,
+ tqdm,
+)
+from .version import Version
diff --git a/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/__init__.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/__init__.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..b88569e41e072cd773203d3ef44a84ee1ba71194
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/__init__.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/_dataset_viewer.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/_dataset_viewer.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..ab87e3d15682e8d6aa25e3b047a79b645067356a
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/_dataset_viewer.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/_dill.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/_dill.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..e305c050a2922f8a3af876d47d17be7ce2a79533
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/_dill.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/_filelock.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/_filelock.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..a6922be5fa95f40e2c5c7a6200c34c5b96d38631
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/_filelock.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/deprecation_utils.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/deprecation_utils.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..d6b5cdd75901e80d1fc6e0858160ee854c068ac1
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/deprecation_utils.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/doc_utils.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/doc_utils.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..b23308a7c7c492012bcd0a1233b73dbf09ddfede
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/doc_utils.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/experimental.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/experimental.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..21ff66302904d3a428725a3ae413efd53ce46297
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/experimental.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/extract.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/extract.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..d09d6349dddfa4a73dd6ceaeac0294c580ba57df
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/extract.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/file_utils.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/file_utils.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..4df48c7569abb232e2be2f6d3d8e3d08fc83731b
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/file_utils.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/filelock.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/filelock.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..3892b2b66d93e436ee8fb5d84cdc2d6d97217481
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/filelock.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/hub.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/hub.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..f15f4b8cf12fb2e0b5b73d38360feebeeb846a16
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/hub.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/info_utils.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/info_utils.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..f3f8e778bc51d021061fe1b1422793d647a53cb0
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/info_utils.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/logging.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/logging.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..9624cf5969e4b19ea541ff70ec25f1441e032eb0
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/logging.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/metadata.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/metadata.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..7de80600dce32d952032e01462c615a914aa2804
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/metadata.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/patching.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/patching.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..d3470bdeed00c1aacf138ff7310679d67edbd015
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/patching.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/py_utils.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/py_utils.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..f01b38b04ec3472e398f7fc90b83a9212c7f24f5
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/py_utils.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/sharding.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/sharding.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..826b5f6a8d88adfb4e994e2129476cb31f2bfca8
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/sharding.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/stratify.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/stratify.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..cdf6eebf0b0c8692b01b62d68cd640460ba48a76
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/stratify.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/tf_utils.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/tf_utils.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..6f913bcf5d9fcbc9520cbd3e1ce11432765110ac
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/tf_utils.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/tqdm.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/tqdm.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..9dbf7fbc9085d81bca6aebe273a56ccdcfca0866
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/tqdm.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/track.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/track.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..93c0466048873461d16a77319ff352e999afac59
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/track.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/typing.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/typing.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..7771916417294bed1f62742737ae00bacccc7785
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/typing.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/version.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/version.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..57b94b475bc1cdefd9d00d06c9978991d2a6eded
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/utils/__pycache__/version.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/utils/_dataset_viewer.py b/venv/lib/python3.10/site-packages/datasets/utils/_dataset_viewer.py
new file mode 100644
index 0000000000000000000000000000000000000000..0f4c298646c5534a93ecfd217b84987b4786cec6
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/utils/_dataset_viewer.py
@@ -0,0 +1,95 @@
+from typing import Any, Optional, Union
+
+from huggingface_hub.utils import get_session
+
+from .. import config
+from ..exceptions import DatasetsError
+from .file_utils import (
+ get_authentication_headers_for_url,
+)
+from .logging import get_logger
+
+
+logger = get_logger(__name__)
+
+
+class DatasetViewerError(DatasetsError):
+ """Dataset viewer error.
+
+ Raised when trying to use the dataset viewer HTTP API and when trying to access:
+ - a missing dataset, or
+ - a private/gated dataset and the user is not authenticated.
+ - unavailable /parquet or /info responses
+ """
+
+
+def get_exported_parquet_files(
+ dataset: str, commit_hash: str, token: Optional[Union[str, bool]]
+) -> list[dict[str, Any]]:
+ """
+ Get the dataset exported parquet files
+ Docs: https://huggingface.co/docs/datasets-server/parquet
+ """
+ dataset_viewer_parquet_url = config.HF_ENDPOINT.replace("://", "://datasets-server.") + "/parquet?dataset="
+ try:
+ parquet_data_files_response = get_session().get(
+ url=dataset_viewer_parquet_url + dataset,
+ headers=get_authentication_headers_for_url(config.HF_ENDPOINT + f"datasets/{dataset}", token=token),
+ timeout=100.0,
+ )
+ parquet_data_files_response.raise_for_status()
+ if "X-Revision" in parquet_data_files_response.headers:
+ if parquet_data_files_response.headers["X-Revision"] == commit_hash or commit_hash is None:
+ parquet_data_files_response_json = parquet_data_files_response.json()
+ if (
+ parquet_data_files_response_json.get("partial") is False
+ and not parquet_data_files_response_json.get("pending", True)
+ and not parquet_data_files_response_json.get("failed", True)
+ and "parquet_files" in parquet_data_files_response_json
+ ):
+ return parquet_data_files_response_json["parquet_files"]
+ else:
+ logger.debug(f"Parquet export for {dataset} is not completely ready yet.")
+ else:
+ logger.debug(
+ f"Parquet export for {dataset} is available but outdated (commit_hash='{parquet_data_files_response.headers['X-Revision']}')"
+ )
+ except Exception as e: # noqa catch any exception of the dataset viewer API and consider the parquet export doesn't exist
+ logger.debug(f"No parquet export for {dataset} available ({type(e).__name__}: {e})")
+ raise DatasetViewerError("No exported Parquet files available.")
+
+
+def get_exported_dataset_infos(
+ dataset: str, commit_hash: str, token: Optional[Union[str, bool]]
+) -> dict[str, dict[str, Any]]:
+ """
+ Get the dataset information, can be useful to get e.g. the dataset features.
+ Docs: https://huggingface.co/docs/datasets-server/info
+ """
+ dataset_viewer_info_url = config.HF_ENDPOINT.replace("://", "://datasets-server.") + "/info?dataset="
+ try:
+ info_response = get_session().get(
+ url=dataset_viewer_info_url + dataset,
+ headers=get_authentication_headers_for_url(config.HF_ENDPOINT + f"datasets/{dataset}", token=token),
+ timeout=100.0,
+ )
+ info_response.raise_for_status()
+ if "X-Revision" in info_response.headers:
+ if info_response.headers["X-Revision"] == commit_hash or commit_hash is None:
+ info_response = info_response.json()
+ if (
+ info_response.get("partial") is False
+ and not info_response.get("pending", True)
+ and not info_response.get("failed", True)
+ and "dataset_info" in info_response
+ ):
+ return info_response["dataset_info"]
+ else:
+ logger.debug(f"Dataset info for {dataset} is not completely ready yet.")
+ else:
+ logger.debug(
+ f"Dataset info for {dataset} is available but outdated (commit_hash='{info_response.headers['X-Revision']}')"
+ )
+ except Exception as e: # noqa catch any exception of the dataset viewer API and consider the dataset info doesn't exist
+ logger.debug(f"No dataset info for {dataset} available ({type(e).__name__}: {e})")
+ raise DatasetViewerError("No exported dataset infos available.")
diff --git a/venv/lib/python3.10/site-packages/datasets/utils/_dill.py b/venv/lib/python3.10/site-packages/datasets/utils/_dill.py
new file mode 100644
index 0000000000000000000000000000000000000000..f3a4baba6811499996813b954158f78f7e0c6b18
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/utils/_dill.py
@@ -0,0 +1,467 @@
+# Copyright 2023 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Extends `dill` to support pickling more types and produce more consistent dumps."""
+
+import os
+import sys
+from io import BytesIO
+from types import CodeType, FunctionType
+
+import dill
+from packaging import version
+
+from .. import config
+
+
+class Pickler(dill.Pickler):
+ dispatch = dill._dill.MetaCatchingDict(dill.Pickler.dispatch.copy())
+ _legacy_no_dict_keys_sorting = False
+
+ def save(self, obj, save_persistent_id=True):
+ obj_type = type(obj)
+ if obj_type not in self.dispatch:
+ if "regex" in sys.modules:
+ import regex # type: ignore
+
+ if obj_type is regex.Pattern:
+ pklregister(obj_type)(_save_regexPattern)
+ if "spacy" in sys.modules:
+ import spacy # type: ignore
+
+ if issubclass(obj_type, spacy.Language):
+ pklregister(obj_type)(_save_spacyLanguage)
+ if "tiktoken" in sys.modules:
+ import tiktoken # type: ignore
+
+ if obj_type is tiktoken.Encoding:
+ pklregister(obj_type)(_save_tiktokenEncoding)
+ if "torch" in sys.modules:
+ import torch # type: ignore
+
+ if issubclass(obj_type, torch.Tensor):
+ pklregister(obj_type)(_save_torchTensor)
+
+ if obj_type is torch.Generator:
+ pklregister(obj_type)(_save_torchGenerator)
+
+ # Unwrap `torch.compile`-ed modules
+ if issubclass(obj_type, torch.nn.Module):
+ obj = getattr(obj, "_orig_mod", obj)
+ if "transformers" in sys.modules:
+ import transformers # type: ignore
+
+ if issubclass(obj_type, transformers.PreTrainedTokenizerBase):
+ pklregister(obj_type)(_save_transformersPreTrainedTokenizerBase)
+
+ # Unwrap `torch.compile`-ed functions
+ if obj_type is FunctionType:
+ obj = getattr(obj, "_torchdynamo_orig_callable", obj)
+ dill.Pickler.save(self, obj, save_persistent_id=save_persistent_id)
+
+ def _batch_setitems(self, items, *args, **kwargs):
+ # Ignore the order of keys in a dict
+ try:
+ # Faster, but fails for unorderable elements
+ items = sorted(items)
+ except Exception: # TypeError, decimal.InvalidOperation, etc.
+ from datasets.fingerprint import Hasher
+
+ items = sorted(items, key=lambda x: Hasher.hash(x[0]))
+ return super()._batch_setitems(items, *args, **kwargs)
+
+ def memoize(self, obj):
+ # Don't memoize strings since two identical strings can have different Python ids
+ if type(obj) is not str: # noqa: E721
+ dill.Pickler.memoize(self, obj)
+
+
+def pklregister(t):
+ """Register a custom reducer for the type."""
+
+ def proxy(func):
+ Pickler.dispatch[t] = func
+ return func
+
+ return proxy
+
+
+def _is_supported_dill_version():
+ """Check if the current dill version is in the supported range."""
+ return config.DILL_VERSION.release[:3] in [
+ version.parse("0.3.6").release,
+ version.parse("0.3.7").release,
+ version.parse("0.3.8").release,
+ version.parse("0.3.9").release,
+ version.parse("0.4.0").release,
+ ]
+
+
+def dump(obj, file):
+ """Pickle an object to a file."""
+ Pickler(file, recurse=True).dump(obj)
+
+
+def dumps(obj):
+ """Pickle an object to a string."""
+ file = BytesIO()
+ dump(obj, file)
+ return file.getvalue()
+
+
+if config.DILL_VERSION < version.parse("0.3.6"):
+
+ def log(pickler, msg):
+ dill._dill.log.info(msg)
+
+elif _is_supported_dill_version():
+
+ def log(pickler, msg):
+ dill._dill.logger.trace(pickler, msg)
+
+
+@pklregister(set)
+def _save_set(pickler, obj):
+ log(pickler, f"Se: {obj}")
+ try:
+ # Faster, but fails for unorderable elements
+ args = (sorted(obj),)
+ except Exception: # TypeError, decimal.InvalidOperation, etc.
+ from datasets.fingerprint import Hasher
+
+ args = (sorted(obj, key=Hasher.hash),)
+
+ pickler.save_reduce(set, args, obj=obj)
+ log(pickler, "# Se")
+
+
+def _save_regexPattern(pickler, obj):
+ import regex # type: ignore
+
+ log(pickler, f"Re: {obj}")
+ args = (obj.pattern, obj.flags)
+ pickler.save_reduce(regex.compile, args, obj=obj)
+ log(pickler, "# Re")
+
+
+def _save_tiktokenEncoding(pickler, obj):
+ import tiktoken # type: ignore
+
+ log(pickler, f"Enc: {obj}")
+ args = (obj.name, obj._pat_str, obj._mergeable_ranks, obj._special_tokens)
+ pickler.save_reduce(tiktoken.Encoding, args, obj=obj)
+ log(pickler, "# Enc")
+
+
+def _save_torchTensor(pickler, obj):
+ import torch # type: ignore
+
+ # `torch.from_numpy` is not picklable in `torch>=1.11.0`
+ def create_torchTensor(np_array, dtype=None):
+ tensor = torch.from_numpy(np_array)
+ if dtype:
+ tensor = tensor.type(dtype)
+ return tensor
+
+ log(pickler, f"To: {obj}")
+ if obj.dtype == torch.bfloat16:
+ args = (obj.detach().to(torch.float).cpu().numpy(), torch.bfloat16)
+ else:
+ args = (obj.detach().cpu().numpy(),)
+ pickler.save_reduce(create_torchTensor, args, obj=obj)
+ log(pickler, "# To")
+
+
+def _save_torchGenerator(pickler, obj):
+ import torch # type: ignore
+
+ def create_torchGenerator(state):
+ generator = torch.Generator()
+ generator.set_state(state)
+ return generator
+
+ log(pickler, f"Ge: {obj}")
+ args = (obj.get_state(),)
+ pickler.save_reduce(create_torchGenerator, args, obj=obj)
+ log(pickler, "# Ge")
+
+
+def _save_spacyLanguage(pickler, obj):
+ import spacy # type: ignore
+
+ def create_spacyLanguage(config, bytes):
+ lang_cls = spacy.util.get_lang_class(config["nlp"]["lang"])
+ lang_inst = lang_cls.from_config(config)
+ return lang_inst.from_bytes(bytes)
+
+ log(pickler, f"Sp: {obj}")
+ args = (obj.config, obj.to_bytes())
+ pickler.save_reduce(create_spacyLanguage, args, obj=obj)
+ log(pickler, "# Sp")
+
+
+def _save_transformersPreTrainedTokenizerBase(pickler, obj):
+ log(pickler, f"Tok: {obj}")
+ # Ignore the `cache` attribute
+ state = obj.__dict__
+ if "cache" in state and isinstance(state["cache"], dict):
+ state["cache"] = {}
+ pickler.save_reduce(type(obj), (), state=state, obj=obj)
+ log(pickler, "# Tok")
+
+
+if config.DILL_VERSION < version.parse("0.3.6"):
+
+ @pklregister(CodeType)
+ def _save_code(pickler, obj):
+ """
+ From dill._dill.save_code
+ This is a modified version that removes the origin (filename + line no.)
+ of functions created in notebooks or shells for example.
+ """
+ dill._dill.log.info(f"Co: {obj}")
+ # The filename of a function is the .py file where it is defined.
+ # Filenames of functions created in notebooks or shells start with '<'
+ # ex: for ipython, and for shell
+ # Filenames of functions created in ipykernel the filename
+ # look like f"{tempdir}/ipykernel_{id1}/{id2}.py"
+ # Moreover lambda functions have a special name: ''
+ # ex: (lambda x: x).__code__.co_name == "" # True
+ #
+ # For the hashing mechanism we ignore where the function has been defined
+ # More specifically:
+ # - we ignore the filename of special functions (filename starts with '<')
+ # - we always ignore the line number
+ # - we only use the base name of the file instead of the whole path,
+ # to be robust in case a script is moved for example.
+ #
+ # Only those two lines are different from the original implementation:
+ co_filename = (
+ ""
+ if obj.co_filename.startswith("<")
+ or (
+ len(obj.co_filename.split(os.path.sep)) > 1
+ and obj.co_filename.split(os.path.sep)[-2].startswith("ipykernel_")
+ )
+ or obj.co_name == ""
+ else os.path.basename(obj.co_filename)
+ )
+ co_firstlineno = 1
+ # The rest is the same as in the original dill implementation (with also a version check for 3.10)
+ if dill._dill.PY3:
+ if hasattr(obj, "co_posonlyargcount"): # python 3.8 (16 args)
+ args = (
+ obj.co_argcount,
+ obj.co_posonlyargcount,
+ obj.co_kwonlyargcount,
+ obj.co_nlocals,
+ obj.co_stacksize,
+ obj.co_flags,
+ obj.co_code,
+ obj.co_consts,
+ obj.co_names,
+ obj.co_varnames,
+ co_filename,
+ obj.co_name,
+ co_firstlineno,
+ obj.co_linetable if sys.version_info >= (3, 10) else obj.co_lnotab,
+ obj.co_freevars,
+ obj.co_cellvars,
+ )
+ else: # python 3.7 (15 args)
+ args = (
+ obj.co_argcount,
+ obj.co_kwonlyargcount,
+ obj.co_nlocals,
+ obj.co_stacksize,
+ obj.co_flags,
+ obj.co_code,
+ obj.co_consts,
+ obj.co_names,
+ obj.co_varnames,
+ co_filename,
+ obj.co_name,
+ co_firstlineno,
+ obj.co_lnotab,
+ obj.co_freevars,
+ obj.co_cellvars,
+ )
+ else:
+ args = (
+ obj.co_argcount,
+ obj.co_nlocals,
+ obj.co_stacksize,
+ obj.co_flags,
+ obj.co_code,
+ obj.co_consts,
+ obj.co_names,
+ obj.co_varnames,
+ co_filename,
+ obj.co_name,
+ co_firstlineno,
+ obj.co_lnotab,
+ obj.co_freevars,
+ obj.co_cellvars,
+ )
+ pickler.save_reduce(CodeType, args, obj=obj)
+ dill._dill.log.info("# Co")
+ return
+
+elif _is_supported_dill_version():
+ # From: https://github.com/uqfoundation/dill/blob/dill-0.3.6/dill/_dill.py#L1104
+ @pklregister(CodeType)
+ def save_code(pickler, obj):
+ dill._dill.logger.trace(pickler, "Co: %s", obj)
+
+ ############################################################################################################
+ # Modification here for huggingface/datasets
+ # The filename of a function is the .py file where it is defined.
+ # Filenames of functions created in notebooks or shells start with '<'
+ # ex: for ipython, and for shell
+ # Filenames of functions created in ipykernel the filename
+ # look like f"{tempdir}/ipykernel_{id1}/{id2}.py"
+ # Moreover lambda functions have a special name: ''
+ # ex: (lambda x: x).__code__.co_name == "" # True
+ #
+ # For the hashing mechanism we ignore where the function has been defined
+ # More specifically:
+ # - we ignore the filename of special functions (filename starts with '<')
+ # - we always ignore the line number
+ # - we only use the base name of the file instead of the whole path,
+ # to be robust in case a script is moved for example.
+ #
+ # Only those two lines are different from the original implementation:
+ co_filename = (
+ ""
+ if obj.co_filename.startswith("<")
+ or (
+ len(obj.co_filename.split(os.path.sep)) > 1
+ and obj.co_filename.split(os.path.sep)[-2].startswith("ipykernel_")
+ )
+ or obj.co_name == ""
+ else os.path.basename(obj.co_filename)
+ )
+ co_firstlineno = 1
+ # The rest is the same as in the original dill implementation, except for the replacements:
+ # - obj.co_filename => co_filename
+ # - obj.co_firstlineno => co_firstlineno
+ # - obj.co_lnotab => obj.co_linetable for >= 3.10 since co_lnotab was deprecated
+ ############################################################################################################
+
+ if hasattr(obj, "co_endlinetable"): # python 3.11a (20 args)
+ args = (
+ obj.co_linetable, # Modification for huggingface/datasets ############################################
+ obj.co_argcount,
+ obj.co_posonlyargcount,
+ obj.co_kwonlyargcount,
+ obj.co_nlocals,
+ obj.co_stacksize,
+ obj.co_flags,
+ obj.co_code,
+ obj.co_consts,
+ obj.co_names,
+ obj.co_varnames,
+ co_filename, # Modification for huggingface/datasets ############################################
+ obj.co_name,
+ obj.co_qualname,
+ co_firstlineno, # Modification for huggingface/datasets #########################################
+ obj.co_linetable,
+ obj.co_endlinetable,
+ obj.co_columntable,
+ obj.co_exceptiontable,
+ obj.co_freevars,
+ obj.co_cellvars,
+ )
+ elif hasattr(obj, "co_exceptiontable"): # python 3.11 (18 args)
+ args = (
+ obj.co_linetable, # Modification for huggingface/datasets #######################################
+ obj.co_argcount,
+ obj.co_posonlyargcount,
+ obj.co_kwonlyargcount,
+ obj.co_nlocals,
+ obj.co_stacksize,
+ obj.co_flags,
+ obj.co_code,
+ obj.co_consts,
+ obj.co_names,
+ obj.co_varnames,
+ co_filename, # Modification for huggingface/datasets ############################################
+ obj.co_name,
+ obj.co_qualname,
+ co_firstlineno, # Modification for huggingface/datasets #########################################
+ obj.co_linetable,
+ obj.co_exceptiontable,
+ obj.co_freevars,
+ obj.co_cellvars,
+ )
+ elif hasattr(obj, "co_linetable"): # python 3.10 (16 args)
+ args = (
+ obj.co_linetable, # Modification for huggingface/datasets #######################################
+ obj.co_argcount,
+ obj.co_posonlyargcount,
+ obj.co_kwonlyargcount,
+ obj.co_nlocals,
+ obj.co_stacksize,
+ obj.co_flags,
+ obj.co_code,
+ obj.co_consts,
+ obj.co_names,
+ obj.co_varnames,
+ co_filename, # Modification for huggingface/datasets ############################################
+ obj.co_name,
+ co_firstlineno, # Modification for huggingface/datasets #########################################
+ obj.co_linetable,
+ obj.co_freevars,
+ obj.co_cellvars,
+ )
+ elif hasattr(obj, "co_posonlyargcount"): # python 3.8 (16 args)
+ args = (
+ obj.co_argcount,
+ obj.co_posonlyargcount,
+ obj.co_kwonlyargcount,
+ obj.co_nlocals,
+ obj.co_stacksize,
+ obj.co_flags,
+ obj.co_code,
+ obj.co_consts,
+ obj.co_names,
+ obj.co_varnames,
+ co_filename, # Modification for huggingface/datasets ############################################
+ obj.co_name,
+ co_firstlineno, # Modification for huggingface/datasets #########################################
+ obj.co_lnotab,
+ obj.co_freevars,
+ obj.co_cellvars,
+ )
+ else: # python 3.7 (15 args)
+ args = (
+ obj.co_argcount,
+ obj.co_kwonlyargcount,
+ obj.co_nlocals,
+ obj.co_stacksize,
+ obj.co_flags,
+ obj.co_code,
+ obj.co_consts,
+ obj.co_names,
+ obj.co_varnames,
+ co_filename, # Modification for huggingface/datasets ############################################
+ obj.co_name,
+ co_firstlineno, # Modification for huggingface/datasets #########################################
+ obj.co_lnotab,
+ obj.co_freevars,
+ obj.co_cellvars,
+ )
+
+ pickler.save_reduce(dill._dill._create_code, args, obj=obj)
+ dill._dill.logger.trace(pickler, "# Co")
+ return
diff --git a/venv/lib/python3.10/site-packages/datasets/utils/_filelock.py b/venv/lib/python3.10/site-packages/datasets/utils/_filelock.py
new file mode 100644
index 0000000000000000000000000000000000000000..803ce77f5a4168ac9023e1d4a4548db2bd154126
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/utils/_filelock.py
@@ -0,0 +1,58 @@
+#!/usr/bin/env python
+# Copyright 2023 The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+"""Utilities to handle file locking in `datasets`."""
+
+import os
+
+from filelock import FileLock as FileLock_
+from filelock import UnixFileLock
+from filelock import __version__ as _filelock_version
+from packaging import version
+
+
+class FileLock(FileLock_):
+ """
+ A `filelock.FileLock` initializer that handles long paths.
+ It also uses the current umask for lock files.
+ """
+
+ MAX_FILENAME_LENGTH = 255
+
+ def __init__(self, lock_file, *args, **kwargs):
+ # The "mode" argument is required if we want to use the current umask in filelock >= 3.10
+ # In previous previous it was already using the current umask.
+ if "mode" not in kwargs and version.parse(_filelock_version) >= version.parse("3.10.0"):
+ umask = os.umask(0o666)
+ os.umask(umask)
+ kwargs["mode"] = 0o666 & ~umask
+ lock_file = self.hash_filename_if_too_long(lock_file)
+ super().__init__(lock_file, *args, **kwargs)
+
+ @classmethod
+ def hash_filename_if_too_long(cls, path: str) -> str:
+ path = os.path.abspath(os.path.expanduser(path))
+ filename = os.path.basename(path)
+ max_filename_length = cls.MAX_FILENAME_LENGTH
+ if issubclass(cls, UnixFileLock):
+ max_filename_length = min(max_filename_length, os.statvfs(os.path.dirname(path)).f_namemax)
+ if len(filename) > max_filename_length:
+ dirname = os.path.dirname(path)
+ hashed_filename = str(hash(filename))
+ new_filename = (
+ filename[: max_filename_length - len(hashed_filename) - 8] + "..." + hashed_filename + ".lock"
+ )
+ return os.path.join(dirname, new_filename)
+ else:
+ return path
diff --git a/venv/lib/python3.10/site-packages/datasets/utils/deprecation_utils.py b/venv/lib/python3.10/site-packages/datasets/utils/deprecation_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..f05ecbeaa3eae5476e99c461dbede9ebfa111eb0
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/utils/deprecation_utils.py
@@ -0,0 +1,105 @@
+import enum
+import inspect
+import warnings
+from functools import wraps
+from typing import Callable, Optional
+
+from .logging import get_logger
+
+
+_emitted_deprecation_warnings = set()
+logger = get_logger(__name__)
+
+
+def deprecated(help_message: Optional[str] = None):
+ """Decorator to mark a class or a function as deprecated.
+
+ Args:
+ help_message (:obj:`str`, optional): An optional message to guide the user on how to
+ switch to non-deprecated usage of the library.
+ """
+
+ def decorator(deprecated_class_or_function: Callable):
+ global _emitted_deprecation_warnings
+
+ if inspect.isclass(deprecated_class_or_function):
+ deprecated_function = deprecated_class_or_function.__init__
+ name = deprecated_class_or_function.__name__
+ else:
+ deprecated_function = deprecated_class_or_function
+ name = deprecated_function.__name__
+ # Support deprecating __init__ class method: class name instead
+ name = name if name != "__init__" else deprecated_function.__qualname__.split(".")[-2]
+
+ warning_msg = (
+ f"{name} is deprecated and will be removed in the next major version of datasets." + f" {help_message}"
+ if help_message
+ else ""
+ )
+
+ @wraps(deprecated_function)
+ def wrapper(*args, **kwargs):
+ func_hash = hash(deprecated_function)
+ if func_hash not in _emitted_deprecation_warnings:
+ warnings.warn(warning_msg, category=FutureWarning, stacklevel=2)
+ _emitted_deprecation_warnings.add(func_hash)
+ return deprecated_function(*args, **kwargs)
+
+ wrapper._decorator_name_ = "deprecated"
+
+ if inspect.isclass(deprecated_class_or_function):
+ deprecated_class_or_function.__init__ = wrapper
+ return deprecated_class_or_function
+ else:
+ return wrapper
+
+ return decorator
+
+
+class OnAccess(enum.EnumMeta):
+ """
+ Enum metaclass that calls a user-specified function whenever a member is accessed.
+ """
+
+ def __getattribute__(cls, name):
+ obj = super().__getattribute__(name)
+ if isinstance(obj, enum.Enum) and obj._on_access:
+ obj._on_access()
+ return obj
+
+ def __getitem__(cls, name):
+ member = super().__getitem__(name)
+ if member._on_access:
+ member._on_access()
+ return member
+
+ def __call__(cls, value, names=None, *, module=None, qualname=None, type=None, start=1):
+ obj = super().__call__(value, names, module=module, qualname=qualname, type=type, start=start)
+ if isinstance(obj, enum.Enum) and obj._on_access:
+ obj._on_access()
+ return obj
+
+
+class DeprecatedEnum(enum.Enum, metaclass=OnAccess):
+ """
+ Enum class that calls `deprecate` method whenever a member is accessed.
+ """
+
+ def __new__(cls, value):
+ member = object.__new__(cls)
+ member._value_ = value
+ member._on_access = member.deprecate
+ return member
+
+ @property
+ def help_message(self):
+ return ""
+
+ def deprecate(self):
+ help_message = f" {self.help_message}" if self.help_message else ""
+ warnings.warn(
+ f"'{self.__objclass__.__name__}' is deprecated and will be removed in the next major version of datasets."
+ + help_message,
+ FutureWarning,
+ stacklevel=3,
+ )
diff --git a/venv/lib/python3.10/site-packages/datasets/utils/doc_utils.py b/venv/lib/python3.10/site-packages/datasets/utils/doc_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..6ef8bcb4e70725ad086cb817e0ec4551d1c0966e
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/utils/doc_utils.py
@@ -0,0 +1,15 @@
+from typing import Callable
+
+
+def is_documented_by(function_with_docstring: Callable):
+ """Decorator to share docstrings across common functions.
+
+ Args:
+ function_with_docstring (`Callable`): Name of the function with the docstring.
+ """
+
+ def wrapper(target_function):
+ target_function.__doc__ = function_with_docstring.__doc__
+ return target_function
+
+ return wrapper
diff --git a/venv/lib/python3.10/site-packages/datasets/utils/experimental.py b/venv/lib/python3.10/site-packages/datasets/utils/experimental.py
new file mode 100644
index 0000000000000000000000000000000000000000..cc406154e9347f4df83b1f7b08c32a961d469f6a
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/utils/experimental.py
@@ -0,0 +1,43 @@
+"""Contains utilities to flag a feature as "experimental" in datasets."""
+
+import warnings
+from functools import wraps
+from typing import Callable
+
+
+def experimental(fn: Callable) -> Callable:
+ """Decorator to flag a feature as experimental.
+
+ An experimental feature trigger a warning when used as it might be subject to breaking changes in the future.
+
+ Args:
+ fn (`Callable`):
+ The function to flag as experimental.
+
+ Returns:
+ `Callable`: The decorated function.
+
+ Example:
+
+ ```python
+ >>> from datasets.utils import experimental
+
+ >>> @experimental
+ ... def my_function():
+ ... print("Hello world!")
+
+ >>> my_function()
+ UserWarning: 'my_function' is experimental and might be subject to breaking changes in the future.
+ Hello world!
+ ```
+ """
+
+ @wraps(fn)
+ def _inner_fn(*args, **kwargs):
+ warnings.warn(
+ (f"'{fn.__name__}' is experimental and might be subject to breaking changes in the future."),
+ UserWarning,
+ )
+ return fn(*args, **kwargs)
+
+ return _inner_fn
diff --git a/venv/lib/python3.10/site-packages/datasets/utils/extract.py b/venv/lib/python3.10/site-packages/datasets/utils/extract.py
new file mode 100644
index 0000000000000000000000000000000000000000..1e87c617217b2b93fd6273804c2ba553ce42eb68
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/utils/extract.py
@@ -0,0 +1,332 @@
+import bz2
+import gzip
+import lzma
+import os
+import shutil
+import struct
+import tarfile
+import warnings
+import zipfile
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import Optional, Union
+
+from .. import config
+from ._filelock import FileLock
+from .logging import get_logger
+
+
+logger = get_logger(__name__)
+
+
+class ExtractManager:
+ def __init__(self, cache_dir: Optional[str] = None):
+ self.extract_dir = (
+ os.path.join(cache_dir, config.EXTRACTED_DATASETS_DIR) if cache_dir else config.EXTRACTED_DATASETS_PATH
+ )
+ self.extractor = Extractor
+
+ def _get_output_path(self, path: str) -> str:
+ from .file_utils import hash_url_to_filename
+
+ # Path where we extract compressed archives
+ # We extract in the cache dir, and get the extracted path name by hashing the original path"
+ abs_path = os.path.abspath(path)
+ return os.path.join(self.extract_dir, hash_url_to_filename(abs_path))
+
+ def _do_extract(self, output_path: str, force_extract: bool) -> bool:
+ return force_extract or (
+ not os.path.isfile(output_path) and not (os.path.isdir(output_path) and os.listdir(output_path))
+ )
+
+ def extract(self, input_path: str, force_extract: bool = False) -> str:
+ extractor_format = self.extractor.infer_extractor_format(input_path)
+ if not extractor_format:
+ return input_path
+ output_path = self._get_output_path(input_path)
+ if self._do_extract(output_path, force_extract):
+ self.extractor.extract(input_path, output_path, extractor_format)
+ return output_path
+
+
+class BaseExtractor(ABC):
+ @classmethod
+ @abstractmethod
+ def is_extractable(cls, path: Union[Path, str], **kwargs) -> bool: ...
+
+ @staticmethod
+ @abstractmethod
+ def extract(input_path: Union[Path, str], output_path: Union[Path, str]) -> None: ...
+
+
+class MagicNumberBaseExtractor(BaseExtractor, ABC):
+ magic_numbers: list[bytes] = []
+
+ @staticmethod
+ def read_magic_number(path: Union[Path, str], magic_number_length: int):
+ with open(path, "rb") as f:
+ return f.read(magic_number_length)
+
+ @classmethod
+ def is_extractable(cls, path: Union[Path, str], magic_number: bytes = b"") -> bool:
+ if not magic_number:
+ magic_number_length = max(len(cls_magic_number) for cls_magic_number in cls.magic_numbers)
+ try:
+ magic_number = cls.read_magic_number(path, magic_number_length)
+ except OSError:
+ return False
+ return any(magic_number.startswith(cls_magic_number) for cls_magic_number in cls.magic_numbers)
+
+
+class TarExtractor(BaseExtractor):
+ @classmethod
+ def is_extractable(cls, path: Union[Path, str], **kwargs) -> bool:
+ return tarfile.is_tarfile(path)
+
+ @staticmethod
+ def safemembers(members, output_path):
+ """
+ Fix for CVE-2007-4559
+ Desc:
+ Directory traversal vulnerability in the (1) extract and (2) extractall functions in the tarfile
+ module in Python allows user-assisted remote attackers to overwrite arbitrary files via a .. (dot dot)
+ sequence in filenames in a TAR archive, a related issue to CVE-2001-1267.
+ See: https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2007-4559
+ From: https://stackoverflow.com/a/10077309
+ """
+
+ def resolved(path: str) -> str:
+ return os.path.realpath(os.path.abspath(path))
+
+ def badpath(path: str, base: str) -> bool:
+ # joinpath will ignore base if path is absolute
+ return not resolved(os.path.join(base, path)).startswith(base)
+
+ def badlink(info, base: str) -> bool:
+ # Links are interpreted relative to the directory containing the link
+ tip = resolved(os.path.join(base, os.path.dirname(info.name)))
+ return badpath(info.linkname, base=tip)
+
+ base = resolved(output_path)
+
+ for finfo in members:
+ if badpath(finfo.name, base):
+ logger.error(f"Extraction of {finfo.name} is blocked (illegal path)")
+ elif finfo.issym() and badlink(finfo, base):
+ logger.error(f"Extraction of {finfo.name} is blocked: Symlink to {finfo.linkname}")
+ elif finfo.islnk() and badlink(finfo, base):
+ logger.error(f"Extraction of {finfo.name} is blocked: Hard link to {finfo.linkname}")
+ else:
+ yield finfo
+
+ @staticmethod
+ def extract(input_path: Union[Path, str], output_path: Union[Path, str]) -> None:
+ os.makedirs(output_path, exist_ok=True)
+ tar_file = tarfile.open(input_path)
+ tar_file.extractall(output_path, members=TarExtractor.safemembers(tar_file, output_path))
+ tar_file.close()
+
+
+class GzipExtractor(MagicNumberBaseExtractor):
+ magic_numbers = [b"\x1f\x8b"]
+
+ @staticmethod
+ def extract(input_path: Union[Path, str], output_path: Union[Path, str]) -> None:
+ with gzip.open(input_path, "rb") as gzip_file:
+ with open(output_path, "wb") as extracted_file:
+ shutil.copyfileobj(gzip_file, extracted_file)
+
+
+class ZipExtractor(MagicNumberBaseExtractor):
+ magic_numbers = [
+ b"PK\x03\x04",
+ b"PK\x05\x06", # empty archive
+ b"PK\x07\x08", # spanned archive
+ ]
+
+ @classmethod
+ def is_extractable(cls, path: Union[Path, str], magic_number: bytes = b"") -> bool:
+ if super().is_extractable(path, magic_number=magic_number):
+ return True
+ try:
+ # Alternative version of zipfile.is_zipfile that has less false positives, but misses executable zip archives.
+ # From: https://github.com/python/cpython/pull/5053
+ from zipfile import (
+ _CD_SIGNATURE,
+ _ECD_DISK_NUMBER,
+ _ECD_DISK_START,
+ _ECD_ENTRIES_TOTAL,
+ _ECD_OFFSET,
+ _ECD_SIZE,
+ _EndRecData,
+ sizeCentralDir,
+ stringCentralDir,
+ structCentralDir,
+ )
+
+ with open(path, "rb") as fp:
+ endrec = _EndRecData(fp)
+ if endrec:
+ if endrec[_ECD_ENTRIES_TOTAL] == 0 and endrec[_ECD_SIZE] == 0 and endrec[_ECD_OFFSET] == 0:
+ return True # Empty zipfiles are still zipfiles
+ elif endrec[_ECD_DISK_NUMBER] == endrec[_ECD_DISK_START]:
+ fp.seek(endrec[_ECD_OFFSET]) # Central directory is on the same disk
+ if fp.tell() == endrec[_ECD_OFFSET] and endrec[_ECD_SIZE] >= sizeCentralDir:
+ data = fp.read(sizeCentralDir) # CD is where we expect it to be
+ if len(data) == sizeCentralDir:
+ centdir = struct.unpack(structCentralDir, data) # CD is the right size
+ if centdir[_CD_SIGNATURE] == stringCentralDir:
+ return True # First central directory entry has correct magic number
+ return False
+ except Exception: # catch all errors in case future python versions change the zipfile internals
+ return False
+
+ @staticmethod
+ def extract(input_path: Union[Path, str], output_path: Union[Path, str]) -> None:
+ os.makedirs(output_path, exist_ok=True)
+ with zipfile.ZipFile(input_path, "r") as zip_file:
+ zip_file.extractall(output_path)
+ zip_file.close()
+
+
+class XzExtractor(MagicNumberBaseExtractor):
+ magic_numbers = [b"\xfd\x37\x7a\x58\x5a\x00"]
+
+ @staticmethod
+ def extract(input_path: Union[Path, str], output_path: Union[Path, str]) -> None:
+ with lzma.open(input_path) as compressed_file:
+ with open(output_path, "wb") as extracted_file:
+ shutil.copyfileobj(compressed_file, extracted_file)
+
+
+class RarExtractor(MagicNumberBaseExtractor):
+ magic_numbers = [b"Rar!\x1a\x07\x00", b"Rar!\x1a\x07\x01\x00"] # RAR_ID # RAR5_ID
+
+ @staticmethod
+ def extract(input_path: Union[Path, str], output_path: Union[Path, str]) -> None:
+ if not config.RARFILE_AVAILABLE:
+ raise ImportError("Please pip install rarfile")
+ import rarfile
+
+ os.makedirs(output_path, exist_ok=True)
+ rf = rarfile.RarFile(input_path)
+ rf.extractall(output_path)
+ rf.close()
+
+
+class ZstdExtractor(MagicNumberBaseExtractor):
+ magic_numbers = [b"\x28\xb5\x2f\xfd"]
+
+ @staticmethod
+ def extract(input_path: Union[Path, str], output_path: Union[Path, str]) -> None:
+ if not config.ZSTANDARD_AVAILABLE:
+ raise ImportError("Please pip install zstandard")
+ import zstandard as zstd
+
+ dctx = zstd.ZstdDecompressor()
+ with open(input_path, "rb") as ifh, open(output_path, "wb") as ofh:
+ dctx.copy_stream(ifh, ofh)
+
+
+class Bzip2Extractor(MagicNumberBaseExtractor):
+ magic_numbers = [b"\x42\x5a\x68"]
+
+ @staticmethod
+ def extract(input_path: Union[Path, str], output_path: Union[Path, str]) -> None:
+ with bz2.open(input_path, "rb") as compressed_file:
+ with open(output_path, "wb") as extracted_file:
+ shutil.copyfileobj(compressed_file, extracted_file)
+
+
+class SevenZipExtractor(MagicNumberBaseExtractor):
+ magic_numbers = [b"\x37\x7a\xbc\xaf\x27\x1c"]
+
+ @staticmethod
+ def extract(input_path: Union[Path, str], output_path: Union[Path, str]) -> None:
+ if not config.PY7ZR_AVAILABLE:
+ raise ImportError("Please pip install py7zr")
+ import py7zr
+
+ os.makedirs(output_path, exist_ok=True)
+ with py7zr.SevenZipFile(input_path, "r") as archive:
+ archive.extractall(output_path)
+
+
+class Lz4Extractor(MagicNumberBaseExtractor):
+ magic_numbers = [b"\x04\x22\x4d\x18"]
+
+ @staticmethod
+ def extract(input_path: Union[Path, str], output_path: Union[Path, str]) -> None:
+ if not config.LZ4_AVAILABLE:
+ raise ImportError("Please pip install lz4")
+ import lz4.frame
+
+ with lz4.frame.open(input_path, "rb") as compressed_file:
+ with open(output_path, "wb") as extracted_file:
+ shutil.copyfileobj(compressed_file, extracted_file)
+
+
+class Extractor:
+ # Put zip file to the last, b/c it is possible wrongly detected as zip (I guess it means: as tar or gzip)
+ extractors: dict[str, type[BaseExtractor]] = {
+ "tar": TarExtractor,
+ "gzip": GzipExtractor,
+ "zip": ZipExtractor,
+ "xz": XzExtractor,
+ "rar": RarExtractor,
+ "zstd": ZstdExtractor,
+ "bz2": Bzip2Extractor,
+ "7z": SevenZipExtractor, #
+ "lz4": Lz4Extractor, #
+ }
+
+ @classmethod
+ def _get_magic_number_max_length(cls):
+ return max(
+ len(extractor_magic_number)
+ for extractor in cls.extractors.values()
+ if issubclass(extractor, MagicNumberBaseExtractor)
+ for extractor_magic_number in extractor.magic_numbers
+ )
+
+ @staticmethod
+ def _read_magic_number(path: Union[Path, str], magic_number_length: int):
+ try:
+ return MagicNumberBaseExtractor.read_magic_number(path, magic_number_length=magic_number_length)
+ except OSError:
+ return b""
+
+ @classmethod
+ def is_extractable(cls, path: Union[Path, str], return_extractor: bool = False) -> bool:
+ warnings.warn(
+ "Method 'is_extractable' was deprecated in version 2.4.0 and will be removed in 3.0.0. "
+ "Use 'infer_extractor_format' instead.",
+ category=FutureWarning,
+ )
+ extractor_format = cls.infer_extractor_format(path)
+ if extractor_format:
+ return True if not return_extractor else (True, cls.extractors[extractor_format])
+ return False if not return_extractor else (False, None)
+
+ @classmethod
+ def infer_extractor_format(cls, path: Union[Path, str]) -> Optional[str]: #
+ magic_number_max_length = cls._get_magic_number_max_length()
+ magic_number = cls._read_magic_number(path, magic_number_max_length)
+ for extractor_format, extractor in cls.extractors.items():
+ if extractor.is_extractable(path, magic_number=magic_number):
+ return extractor_format
+
+ @classmethod
+ def extract(
+ cls,
+ input_path: Union[Path, str],
+ output_path: Union[Path, str],
+ extractor_format: str,
+ ) -> None:
+ os.makedirs(os.path.dirname(output_path), exist_ok=True)
+ # Prevent parallel extractions
+ lock_path = str(Path(output_path).with_suffix(".lock"))
+ with FileLock(lock_path):
+ shutil.rmtree(output_path, ignore_errors=True)
+ extractor = cls.extractors[extractor_format]
+ return extractor.extract(input_path, output_path)
diff --git a/venv/lib/python3.10/site-packages/datasets/utils/file_utils.py b/venv/lib/python3.10/site-packages/datasets/utils/file_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..b57a37845472807506a5615a635f1c4e6bb20465
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/utils/file_utils.py
@@ -0,0 +1,1402 @@
+"""
+Utilities for working with the local dataset cache.
+This file is adapted from the AllenNLP library at https://github.com/allenai/allennlp
+Copyright by the AllenNLP authors.
+"""
+
+import asyncio
+import glob
+import io
+import json
+import multiprocessing
+import os
+import posixpath
+import re
+import shutil
+import tarfile
+import time
+import xml.dom.minidom
+import zipfile
+from collections.abc import Generator
+from io import BytesIO
+from itertools import chain
+from pathlib import Path, PurePosixPath
+from typing import Any, Optional, TypeVar, Union
+from unittest.mock import patch
+from urllib.parse import urlparse
+from xml.etree import ElementTree as ET
+
+import fsspec
+import httpx
+import huggingface_hub
+import huggingface_hub.errors
+import requests
+from fsspec.core import strip_protocol, url_to_fs
+from fsspec.utils import can_be_local
+from huggingface_hub.utils import get_session, insecure_hashlib
+from packaging import version
+
+from .. import __version__, config
+from ..download.download_config import DownloadConfig
+from ..filesystems import COMPRESSION_FILESYSTEMS
+from . import _tqdm, logging
+from ._filelock import FileLock
+from .extract import ExtractManager
+from .track import TrackedIterableFromGenerator
+
+
+try:
+ from aiohttp.client_exceptions import ClientError as _AiohttpClientError
+except ImportError:
+ # aiohttp is not available; synthesize an exception type
+ # that will never be raised by any actual code for use in the `except`
+ # clause only.
+ class _AiohttpClientError(Exception):
+ pass
+
+
+logger = logging.get_logger(__name__) # pylint: disable=invalid-name
+
+INCOMPLETE_SUFFIX = ".incomplete"
+
+T = TypeVar("T", str, Path)
+
+CONNECTION_ERRORS_TO_RETRY = (
+ _AiohttpClientError,
+ asyncio.TimeoutError,
+ requests.exceptions.ConnectionError,
+ requests.exceptions.Timeout,
+ httpx.RequestError,
+)
+SERVER_UNAVAILABLE_CODE = 504
+RATE_LIMIT_CODE = 429
+
+
+def is_remote_url(url_or_filename: str) -> bool:
+ return urlparse(url_or_filename).scheme != "" and not os.path.ismount(urlparse(url_or_filename).scheme + ":/")
+
+
+def is_local_path(url_or_filename: str) -> bool:
+ # On unix the scheme of a local path is empty (for both absolute and relative),
+ # while on windows the scheme is the drive name (ex: "c") for absolute paths.
+ # for details on the windows behavior, see https://bugs.python.org/issue42215
+ return urlparse(url_or_filename).scheme == "" or os.path.ismount(urlparse(url_or_filename).scheme + ":/")
+
+
+def is_relative_path(url_or_filename: str) -> bool:
+ return urlparse(url_or_filename).scheme == "" and not os.path.isabs(url_or_filename)
+
+
+def relative_to_absolute_path(path: T) -> T:
+ """Convert relative path to absolute path."""
+ abs_path_str = os.path.abspath(os.path.expanduser(os.path.expandvars(str(path))))
+ return Path(abs_path_str) if isinstance(path, Path) else abs_path_str
+
+
+def url_or_path_join(base_name: str, *pathnames: str) -> str:
+ if is_remote_url(base_name):
+ return posixpath.join(base_name, *(str(pathname).replace(os.sep, "/").lstrip("/") for pathname in pathnames))
+ else:
+ return Path(base_name, *pathnames).as_posix()
+
+
+def url_or_path_parent(url_or_path: str) -> str:
+ if is_remote_url(url_or_path):
+ return url_or_path[: url_or_path.rindex("/")]
+ else:
+ return os.path.dirname(url_or_path)
+
+
+def hash_url_to_filename(url, etag=None):
+ """
+ Convert `url` into a hashed filename in a repeatable way.
+ If `etag` is specified, append its hash to the url's, delimited
+ by a period.
+ If the url ends with .h5 (Keras HDF5 weights) adds '.h5' to the name
+ so that TF 2.0 can identify it as a HDF5 file
+ (see https://github.com/tensorflow/tensorflow/blob/00fad90125b18b80fe054de1055770cfb8fe4ba3/tensorflow/python/keras/engine/network.py#L1380)
+ """
+ url_bytes = url.encode("utf-8")
+ url_hash = insecure_hashlib.sha256(url_bytes)
+ filename = url_hash.hexdigest()
+
+ if etag:
+ etag_bytes = etag.encode("utf-8")
+ etag_hash = insecure_hashlib.sha256(etag_bytes)
+ filename += "." + etag_hash.hexdigest()
+
+ if url.endswith(".py"):
+ filename += ".py"
+
+ return filename
+
+
+def cached_path(
+ url_or_filename,
+ download_config=None,
+ **download_kwargs,
+) -> str:
+ """
+ Given something that might be a URL (or might be a local path),
+ determine which. If it's a URL, download the file and cache it, and
+ return the path to the cached file. If it's already a local path,
+ make sure the file exists and then return the path.
+
+ Return:
+ Local path (string)
+
+ Raises:
+ FileNotFoundError: in case of non-recoverable file
+ (non-existent or no cache on disk)
+ ConnectionError: in case of unreachable url
+ and no cache on disk
+ ValueError: if it couldn't parse the url or filename correctly
+ httpx.NetworkError or requests.exceptions.ConnectionError: in case of internet connection issue
+ """
+ if download_config is None:
+ download_config = DownloadConfig(**download_kwargs)
+
+ cache_dir = download_config.cache_dir or config.DOWNLOADED_DATASETS_PATH
+ if isinstance(cache_dir, Path):
+ cache_dir = str(cache_dir)
+ if isinstance(url_or_filename, Path):
+ url_or_filename = str(url_or_filename)
+
+ # Convert fsspec URL in the format "file://local/path" to "local/path"
+ if can_be_local(url_or_filename):
+ url_or_filename = strip_protocol(url_or_filename)
+
+ if is_remote_url(url_or_filename):
+ # URL, so get it from the cache (downloading if necessary)
+ url_or_filename, storage_options = _prepare_path_and_storage_options(
+ url_or_filename, download_config=download_config
+ )
+ # Download files from Hugging Face.
+ # Note: no need to check for https://huggingface.co file URLs since _prepare_path_and_storage_options
+ # prepares Hugging Face HTTP URLs as hf:// paths already
+ if url_or_filename.startswith("hf://"):
+ resolved_path = huggingface_hub.HfFileSystem(
+ endpoint=config.HF_ENDPOINT, token=download_config.token
+ ).resolve_path(url_or_filename)
+ try:
+ output_path = huggingface_hub.HfApi(
+ endpoint=config.HF_ENDPOINT,
+ token=download_config.token,
+ library_name="datasets",
+ library_version=__version__,
+ user_agent=get_datasets_user_agent(download_config.user_agent),
+ ).hf_hub_download(
+ repo_id=resolved_path.repo_id,
+ repo_type=resolved_path.repo_type,
+ revision=resolved_path.revision,
+ filename=resolved_path.path_in_repo,
+ force_download=download_config.force_download,
+ proxies=download_config.proxies,
+ )
+ except (
+ huggingface_hub.utils.RepositoryNotFoundError,
+ huggingface_hub.utils.EntryNotFoundError,
+ huggingface_hub.utils.RevisionNotFoundError,
+ huggingface_hub.utils.GatedRepoError,
+ ) as e:
+ raise FileNotFoundError(str(e)) from e
+ # Download external files
+ else:
+ output_path = get_from_cache(
+ url_or_filename,
+ cache_dir=cache_dir,
+ force_download=download_config.force_download,
+ user_agent=download_config.user_agent,
+ use_etag=download_config.use_etag,
+ token=download_config.token,
+ storage_options=storage_options,
+ download_desc=download_config.download_desc,
+ disable_tqdm=download_config.disable_tqdm,
+ )
+ elif os.path.exists(url_or_filename):
+ # File, and it exists.
+ output_path = url_or_filename
+ elif is_local_path(url_or_filename):
+ # File, but it doesn't exist.
+ raise FileNotFoundError(f"Local file {url_or_filename} doesn't exist")
+ else:
+ # Something unknown
+ raise ValueError(f"unable to parse {url_or_filename} as a URL or as a local path")
+
+ if output_path is None:
+ return output_path
+
+ if download_config.extract_compressed_file:
+ if download_config.extract_on_the_fly:
+ # Add a compression prefix to the compressed file so that it can be extracted
+ # as it's being read using xopen.
+ protocol = _get_extraction_protocol(output_path, download_config=download_config)
+ extension = _get_path_extension(url_or_filename.split("::")[0])
+ if (
+ protocol
+ and extension not in ["tgz", "tar"]
+ and not url_or_filename.split("::")[0].endswith((".tar.gz", ".tar.bz2", ".tar.xz"))
+ ):
+ output_path = relative_to_absolute_path(output_path)
+ if protocol in SINGLE_FILE_COMPRESSION_PROTOCOLS:
+ # there is one single file which is the uncompressed file
+ inner_file = os.path.basename(output_path)
+ inner_file = inner_file[: inner_file.rindex(".")] if "." in inner_file else inner_file
+ output_path = f"{protocol}://{inner_file}::{output_path}"
+ else:
+ output_path = f"{protocol}://::{output_path}"
+ return output_path
+
+ # Eager extraction
+ output_path = ExtractManager(cache_dir=download_config.cache_dir).extract(
+ output_path, force_extract=download_config.force_extract
+ )
+ return relative_to_absolute_path(output_path)
+
+
+def get_datasets_user_agent(user_agent: Optional[Union[str, dict]] = None) -> str:
+ ua = f"datasets/{__version__}"
+ ua += f"; python/{config.PY_VERSION}"
+ ua += f"; hf_hub/{huggingface_hub.__version__}"
+ ua += f"; pyarrow/{config.PYARROW_VERSION}"
+ if config.TORCH_AVAILABLE:
+ ua += f"; torch/{config.TORCH_VERSION}"
+ if config.TF_AVAILABLE:
+ ua += f"; tensorflow/{config.TF_VERSION}"
+ if config.JAX_AVAILABLE:
+ ua += f"; jax/{config.JAX_VERSION}"
+ if isinstance(user_agent, dict):
+ ua += f"; {'; '.join(f'{k}/{v}' for k, v in user_agent.items())}"
+ elif isinstance(user_agent, str):
+ ua += "; " + user_agent
+ return ua
+
+
+def get_authentication_headers_for_url(url: str, token: Optional[Union[str, bool]] = None) -> dict:
+ """Handle the HF authentication"""
+ if url.startswith(config.HF_ENDPOINT):
+ return huggingface_hub.utils.build_hf_headers(
+ token=token, library_name="datasets", library_version=__version__
+ )
+ else:
+ return {}
+
+
+def _raise_if_offline_mode_is_enabled(msg: Optional[str] = None):
+ """Raise an OfflineModeIsEnabled error (subclass of ConnectionError) if HF_HUB_OFFLINE is True."""
+ if config.HF_HUB_OFFLINE:
+ raise huggingface_hub.errors.OfflineModeIsEnabled(
+ "Offline mode is enabled." if msg is None else "Offline mode is enabled. " + str(msg)
+ )
+
+
+def fsspec_head(url, storage_options=None):
+ _raise_if_offline_mode_is_enabled(f"Tried to reach {url}")
+ fs, path = url_to_fs(url, **(storage_options or {}))
+ return fs.info(path)
+
+
+def stack_multiprocessing_download_progress_bars():
+ # Stack downloads progress bars automatically using HF_DATASETS_STACK_MULTIPROCESSING_DOWNLOAD_PROGRESS_BARS=1
+ # We use environment variables since the download may happen in a subprocess
+ return patch.dict(os.environ, {"HF_DATASETS_STACK_MULTIPROCESSING_DOWNLOAD_PROGRESS_BARS": "1"})
+
+
+class TqdmCallback(fsspec.callbacks.TqdmCallback):
+ def __init__(self, tqdm_kwargs=None, *args, **kwargs):
+ if config.FSSPEC_VERSION < version.parse("2024.2.0"):
+ super().__init__(tqdm_kwargs, *args, **kwargs)
+ self._tqdm = _tqdm # replace tqdm module by datasets.utils.tqdm module
+ else:
+ kwargs["tqdm_cls"] = _tqdm.tqdm
+ super().__init__(tqdm_kwargs, *args, **kwargs)
+
+
+def fsspec_get(url, temp_file, storage_options=None, desc=None, disable_tqdm=False):
+ _raise_if_offline_mode_is_enabled(f"Tried to reach {url}")
+ fs, path = url_to_fs(url, **(storage_options or {}))
+ callback = TqdmCallback(
+ tqdm_kwargs={
+ "desc": desc or "Downloading",
+ "unit": "B",
+ "unit_scale": True,
+ "position": multiprocessing.current_process()._identity[-1] # contains the ranks of subprocesses
+ if os.environ.get("HF_DATASETS_STACK_MULTIPROCESSING_DOWNLOAD_PROGRESS_BARS") == "1"
+ and multiprocessing.current_process()._identity
+ else None,
+ "disable": disable_tqdm,
+ }
+ )
+ fs.get_file(path, temp_file.name, callback=callback)
+
+
+def get_from_cache(
+ url,
+ cache_dir=None,
+ force_download=False,
+ user_agent=None,
+ use_etag=True,
+ token=None,
+ storage_options=None,
+ download_desc=None,
+ disable_tqdm=False,
+) -> str:
+ """
+ Given a URL, look for the corresponding file in the local cache.
+ If it's not there, download it. Then return the path to the cached file.
+
+ Return:
+ Local path (string)
+
+ Raises:
+ FileNotFoundError: in case of non-recoverable file
+ (non-existent or no cache on disk)
+ ConnectionError: in case of unreachable url
+ and no cache on disk
+ """
+ if storage_options is None:
+ storage_options = {}
+ if cache_dir is None:
+ cache_dir = config.HF_DATASETS_CACHE
+ if isinstance(cache_dir, Path):
+ cache_dir = str(cache_dir)
+
+ os.makedirs(cache_dir, exist_ok=True)
+
+ response = None
+ etag = None
+
+ # Try a first time to file the file on the local file system without eTag (None)
+ # if we don't ask for 'force_download' then we spare a request
+ filename = hash_url_to_filename(url, etag=None)
+ cache_path = os.path.join(cache_dir, filename)
+
+ if os.path.exists(cache_path) and not force_download and not use_etag:
+ return cache_path
+
+ # Prepare headers for authentication
+ headers = get_authentication_headers_for_url(url, token=token)
+ if user_agent is not None:
+ headers["user-agent"] = user_agent
+
+ response = fsspec_head(url, storage_options=storage_options)
+ etag = (response.get("ETag", None) or response.get("etag", None)) if use_etag else None
+
+ # Try a second time
+ filename = hash_url_to_filename(url, etag)
+ cache_path = os.path.join(cache_dir, filename)
+
+ if os.path.exists(cache_path) and not force_download:
+ return cache_path
+
+ # Prevent parallel downloads of the same file with a lock.
+ lock_path = cache_path + ".lock"
+ with FileLock(lock_path):
+ # Retry in case previously locked processes just enter after the precedent process releases the lock
+ if os.path.exists(cache_path) and not force_download:
+ return cache_path
+
+ incomplete_path = cache_path + ".incomplete"
+
+ # Download to temporary file, then copy to cache path once finished.
+ # Otherwise, you get corrupt cache entries if the download gets interrupted.
+ with open(incomplete_path, "w+b") as temp_file:
+ logger.info(f"{url} not found in cache or force_download set to True, downloading to {temp_file.name}")
+ # GET file object
+ fsspec_get(url, temp_file, storage_options=storage_options, desc=download_desc, disable_tqdm=disable_tqdm)
+
+ logger.info(f"storing {url} in cache at {cache_path}")
+ shutil.move(temp_file.name, cache_path)
+
+ logger.info(f"creating metadata file for {cache_path}")
+ meta = {"url": url, "etag": etag}
+ meta_path = cache_path + ".json"
+ with open(meta_path, "w", encoding="utf-8") as meta_file:
+ json.dump(meta, meta_file)
+
+ return cache_path
+
+
+def add_start_docstrings(*docstr):
+ def docstring_decorator(fn):
+ fn.__doc__ = "".join(docstr) + "\n\n" + (fn.__doc__ if fn.__doc__ is not None else "")
+ return fn
+
+ return docstring_decorator
+
+
+def add_end_docstrings(*docstr):
+ def docstring_decorator(fn):
+ fn.__doc__ = (fn.__doc__ if fn.__doc__ is not None else "") + "\n\n" + "".join(docstr)
+ return fn
+
+ return docstring_decorator
+
+
+def estimate_dataset_size(paths):
+ return sum(path.stat().st_size for path in paths)
+
+
+def readline(f: io.RawIOBase):
+ # From: https://github.com/python/cpython/blob/d27e2f4d118e7a9909b6a3e5da06c5ff95806a85/Lib/_pyio.py#L525
+ res = bytearray()
+ while True:
+ b = f.read(1)
+ if not b:
+ break
+ res += b
+ if res.endswith(b"\n"):
+ break
+ return bytes(res)
+
+
+#######################
+# Streaming utilities #
+#######################
+
+BASE_KNOWN_EXTENSIONS = [
+ "txt",
+ "csv",
+ "json",
+ "jsonl",
+ "tsv",
+ "conll",
+ "conllu",
+ "orig",
+ "parquet",
+ "pkl",
+ "pickle",
+ "rel",
+ "xml",
+ "arrow",
+]
+COMPRESSION_EXTENSION_TO_PROTOCOL = {
+ # single file compression
+ **{
+ extension.lstrip("."): fs_class.protocol
+ for fs_class in COMPRESSION_FILESYSTEMS
+ for extension in fs_class.extensions
+ },
+ # archive compression
+ "zip": "zip",
+}
+SINGLE_FILE_COMPRESSION_EXTENSION_TO_PROTOCOL = {
+ extension.lstrip("."): fs_class.protocol
+ for fs_class in COMPRESSION_FILESYSTEMS
+ for extension in fs_class.extensions
+}
+SINGLE_FILE_COMPRESSION_PROTOCOLS = {fs_class.protocol for fs_class in COMPRESSION_FILESYSTEMS}
+SINGLE_SLASH_AFTER_PROTOCOL_PATTERN = re.compile(r"(? str:
+ # Get extension: https://foo.bar/train.json.gz -> gz
+ extension = path.split(".")[-1]
+ # Remove query params ("dl=1", "raw=true"): gz?dl=1 -> gz
+ # Remove shards infos (".txt_1", ".txt-00000-of-00100"): txt_1 -> txt
+ for symb in "?-_":
+ extension = extension.split(symb)[0]
+ return extension
+
+
+def _get_extraction_protocol_with_magic_number(f) -> Optional[str]:
+ """read the magic number from a file-like object and return the compression protocol"""
+ # Check if the file object is seekable even before reading the magic number (to avoid https://bugs.python.org/issue26440)
+ try:
+ f.seek(0)
+ except (AttributeError, io.UnsupportedOperation):
+ return None
+ magic_number = f.read(MAGIC_NUMBER_MAX_LENGTH)
+ f.seek(0)
+ for i in range(MAGIC_NUMBER_MAX_LENGTH):
+ compression = MAGIC_NUMBER_TO_COMPRESSION_PROTOCOL.get(magic_number[: MAGIC_NUMBER_MAX_LENGTH - i])
+ if compression is not None:
+ return compression
+ compression = MAGIC_NUMBER_TO_UNSUPPORTED_COMPRESSION_PROTOCOL.get(magic_number[: MAGIC_NUMBER_MAX_LENGTH - i])
+ if compression is not None:
+ raise NotImplementedError(f"Compression protocol '{compression}' not implemented.")
+
+
+def _get_extraction_protocol(urlpath: str, download_config: Optional[DownloadConfig] = None) -> Optional[str]:
+ # get inner file: zip://train-00000.json.gz::https://foo.bar/data.zip -> zip://train-00000.json.gz
+ urlpath = str(urlpath)
+ path = urlpath.split("::")[0]
+ extension = _get_path_extension(path)
+ if (
+ extension in BASE_KNOWN_EXTENSIONS
+ or extension in ["tgz", "tar"]
+ or path.endswith((".tar.gz", ".tar.bz2", ".tar.xz"))
+ ):
+ return None
+ elif extension in COMPRESSION_EXTENSION_TO_PROTOCOL:
+ return COMPRESSION_EXTENSION_TO_PROTOCOL[extension]
+ urlpath, storage_options = _prepare_path_and_storage_options(urlpath, download_config=download_config)
+ try:
+ with fsspec.open(urlpath, **(storage_options or {})) as f:
+ return _get_extraction_protocol_with_magic_number(f)
+ except FileNotFoundError:
+ if urlpath.startswith(config.HF_ENDPOINT):
+ raise FileNotFoundError(
+ urlpath + "\nIf the repo is private or gated, make sure to log in with `huggingface-cli login`."
+ ) from None
+ else:
+ raise
+
+
+def xjoin(a, *p):
+ """
+ This function extends os.path.join to support the "::" hop separator. It supports both paths and urls.
+
+ A shorthand, particularly useful where you have multiple hops, is to “chain” the URLs with the special separator "::".
+ This is used to access files inside a zip file over http for example.
+
+ Let's say you have a zip file at https://host.com/archive.zip, and you want to access the file inside the zip file at /folder1/file.txt.
+ Then you can just chain the url this way:
+
+ zip://folder1/file.txt::https://host.com/archive.zip
+
+ The xjoin function allows you to apply the join on the first path of the chain.
+
+ Example::
+
+ >>> xjoin("zip://folder1::https://host.com/archive.zip", "file.txt")
+ zip://folder1/file.txt::https://host.com/archive.zip
+ """
+ a, *b = str(a).split("::")
+ if is_local_path(a):
+ return os.path.join(a, *p)
+ else:
+ a = posixpath.join(a, *p)
+ return "::".join([a] + b)
+
+
+def xdirname(a):
+ """
+ This function extends os.path.dirname to support the "::" hop separator. It supports both paths and urls.
+
+ A shorthand, particularly useful where you have multiple hops, is to “chain” the URLs with the special separator "::".
+ This is used to access files inside a zip file over http for example.
+
+ Let's say you have a zip file at https://host.com/archive.zip, and you want to access the file inside the zip file at /folder1/file.txt.
+ Then you can just chain the url this way:
+
+ zip://folder1/file.txt::https://host.com/archive.zip
+
+ The xdirname function allows you to apply the dirname on the first path of the chain.
+
+ Example::
+
+ >>> xdirname("zip://folder1/file.txt::https://host.com/archive.zip")
+ zip://folder1::https://host.com/archive.zip
+ """
+ a, *b = str(a).split("::")
+ if is_local_path(a):
+ a = os.path.dirname(Path(a).as_posix())
+ else:
+ a = posixpath.dirname(a)
+ # if we end up at the root of the protocol, we get for example a = 'http:'
+ # so we have to fix it by adding the '//' that was removed:
+ if a.endswith(":"):
+ a += "//"
+ return "::".join([a] + b)
+
+
+def xexists(urlpath: str, download_config: Optional[DownloadConfig] = None):
+ """Extend `os.path.exists` function to support both local and remote files.
+
+ Args:
+ urlpath (`str`): URL path.
+ download_config : mainly use token or storage_options to support different platforms and auth types.
+
+ Returns:
+ `bool`
+ """
+
+ main_hop, *rest_hops = _as_str(urlpath).split("::")
+ if is_local_path(main_hop):
+ return os.path.exists(main_hop)
+ else:
+ urlpath, storage_options = _prepare_path_and_storage_options(urlpath, download_config=download_config)
+ main_hop, *rest_hops = urlpath.split("::")
+ fs, *_ = url_to_fs(urlpath, **storage_options)
+ return fs.exists(main_hop)
+
+
+def xbasename(a):
+ """
+ This function extends os.path.basename to support the "::" hop separator. It supports both paths and urls.
+
+ A shorthand, particularly useful where you have multiple hops, is to “chain” the URLs with the special separator "::".
+ This is used to access files inside a zip file over http for example.
+
+ Let's say you have a zip file at https://host.com/archive.zip, and you want to access the file inside the zip file at /folder1/file.txt.
+ Then you can just chain the url this way:
+
+ zip://folder1/file.txt::https://host.com/archive.zip
+
+ The xbasename function allows you to apply the basename on the first path of the chain.
+
+ Example::
+
+ >>> xbasename("zip://folder1/file.txt::https://host.com/archive.zip")
+ file.txt
+ """
+ a, *b = str(a).split("::")
+ if is_local_path(a):
+ return os.path.basename(Path(a).as_posix())
+ else:
+ return posixpath.basename(a)
+
+
+def xsplit(a):
+ """
+ This function extends os.path.split to support the "::" hop separator. It supports both paths and urls.
+
+ A shorthand, particularly useful where you have multiple hops, is to “chain” the URLs with the special separator "::".
+ This is used to access files inside a zip file over http for example.
+
+ Let's say you have a zip file at https://host.com/archive.zip, and you want to access the file inside the zip file at /folder1/file.txt.
+ Then you can just chain the url this way:
+
+ zip://folder1/file.txt::https://host.com/archive.zip
+
+ The xsplit function allows you to apply the xsplit on the first path of the chain.
+
+ Example::
+
+ >>> xsplit("zip://folder1/file.txt::https://host.com/archive.zip")
+ ('zip://folder1::https://host.com/archive.zip', 'file.txt')
+ """
+ a, *b = str(a).split("::")
+ if is_local_path(a):
+ return os.path.split(Path(a).as_posix())
+ else:
+ a, tail = posixpath.split(a)
+ return "::".join([a + "//" if a.endswith(":") else a] + b), tail
+
+
+def xsplitext(a):
+ """
+ This function extends os.path.splitext to support the "::" hop separator. It supports both paths and urls.
+
+ A shorthand, particularly useful where you have multiple hops, is to “chain” the URLs with the special separator "::".
+ This is used to access files inside a zip file over http for example.
+
+ Let's say you have a zip file at https://host.com/archive.zip, and you want to access the file inside the zip file at /folder1/file.txt.
+ Then you can just chain the url this way:
+
+ zip://folder1/file.txt::https://host.com/archive.zip
+
+ The xsplitext function allows you to apply the splitext on the first path of the chain.
+
+ Example::
+
+ >>> xsplitext("zip://folder1/file.txt::https://host.com/archive.zip")
+ ('zip://folder1/file::https://host.com/archive.zip', '.txt')
+ """
+ a, *b = str(a).split("::")
+ if is_local_path(a):
+ return os.path.splitext(Path(a).as_posix())
+ else:
+ a, ext = posixpath.splitext(a)
+ return "::".join([a] + b), ext
+
+
+def xisfile(path, download_config: Optional[DownloadConfig] = None) -> bool:
+ """Extend `os.path.isfile` function to support remote files.
+
+ Args:
+ path (`str`): URL path.
+ download_config : mainly use token or storage_options to support different platforms and auth types.
+
+ Returns:
+ `bool`
+ """
+ main_hop, *rest_hops = str(path).split("::")
+ if is_local_path(main_hop):
+ return os.path.isfile(path)
+ else:
+ path, storage_options = _prepare_path_and_storage_options(path, download_config=download_config)
+ main_hop, *rest_hops = path.split("::")
+ fs, *_ = url_to_fs(path, **storage_options)
+ return fs.isfile(main_hop)
+
+
+def xgetsize(path, download_config: Optional[DownloadConfig] = None) -> int:
+ """Extend `os.path.getsize` function to support remote files.
+
+ Args:
+ path (`str`): URL path.
+ download_config : mainly use token or storage_options to support different platforms and auth types.
+
+ Returns:
+ `int`: optional
+ """
+ main_hop, *rest_hops = str(path).split("::")
+ if is_local_path(main_hop):
+ return os.path.getsize(path)
+ else:
+ path, storage_options = _prepare_path_and_storage_options(path, download_config=download_config)
+ main_hop, *rest_hops = path.split("::")
+ fs, *_ = fs, *_ = url_to_fs(path, **storage_options)
+ try:
+ size = fs.size(main_hop)
+ except huggingface_hub.utils.EntryNotFoundError:
+ raise FileNotFoundError(f"No such file: {path}")
+ if size is None:
+ # use xopen instead of fs.open to make data fetching more robust
+ with xopen(path, download_config=download_config) as f:
+ size = len(f.read())
+ return size
+
+
+def xisdir(path, download_config: Optional[DownloadConfig] = None) -> bool:
+ """Extend `os.path.isdir` function to support remote files.
+
+ Args:
+ path (`str`): URL path.
+ download_config : mainly use token or storage_options to support different platforms and auth types.
+
+ Returns:
+ `bool`
+ """
+ main_hop, *rest_hops = str(path).split("::")
+ if is_local_path(main_hop):
+ return os.path.isdir(path)
+ else:
+ path, storage_options = _prepare_path_and_storage_options(path, download_config=download_config)
+ main_hop, *rest_hops = path.split("::")
+ fs, *_ = fs, *_ = url_to_fs(path, **storage_options)
+ inner_path = main_hop.split("://")[-1]
+ if not inner_path.strip("/"):
+ return True
+ return fs.isdir(inner_path)
+
+
+def xrelpath(path, start=None):
+ """Extend `os.path.relpath` function to support remote files.
+
+ Args:
+ path (`str`): URL path.
+ start (`str`): Start URL directory path.
+
+ Returns:
+ `str`
+ """
+ main_hop, *rest_hops = str(path).split("::")
+ if is_local_path(main_hop):
+ return os.path.relpath(main_hop, start=start) if start else os.path.relpath(main_hop)
+ else:
+ return posixpath.relpath(main_hop, start=str(start).split("::")[0]) if start else os.path.relpath(main_hop)
+
+
+def _add_retries_to_file_obj_read_method(file_obj):
+ read = file_obj.read
+ max_retries = config.STREAMING_READ_MAX_RETRIES
+
+ def read_with_retries(*args, **kwargs):
+ disconnect_err = None
+ for retry in range(1, max_retries + 1):
+ try:
+ out = read(*args, **kwargs)
+ break
+ except CONNECTION_ERRORS_TO_RETRY as err:
+ disconnect_err = err
+ logger.warning(
+ f"Got disconnected from remote data host. Retrying in {config.STREAMING_READ_RETRY_INTERVAL}sec [{retry}/{max_retries}]"
+ )
+ time.sleep(config.STREAMING_READ_RETRY_INTERVAL)
+ except huggingface_hub.errors.HfHubHTTPError as err:
+ if err.response is not None and err.response.status_code == SERVER_UNAVAILABLE_CODE:
+ disconnect_err = err
+ logger.warning(
+ f"Got disconnected from remote data host. Retrying in {config.STREAMING_READ_SERVER_UNAVAILABLE_RETRY_INTERVAL}sec [{retry}/{max_retries}]"
+ )
+ time.sleep(config.STREAMING_READ_SERVER_UNAVAILABLE_RETRY_INTERVAL)
+ elif err.response is not None and err.response.status_code == RATE_LIMIT_CODE:
+ disconnect_err = err
+ logger.warning(str(err))
+ logger.warning(
+ f"Got disconnected from remote data host. Retrying in {config.STREAMING_READ_RATE_LIMIT_RETRY_INTERVAL}sec [{retry}/{max_retries}]"
+ )
+ time.sleep(config.STREAMING_READ_RATE_LIMIT_RETRY_INTERVAL)
+ else:
+ raise
+ else:
+ raise ConnectionError("Server Disconnected") from disconnect_err
+ return out
+
+ try:
+ file_obj.read = read_with_retries
+ except AttributeError: # read-only attribute
+ orig_file_obj = file_obj
+ file_obj = io.RawIOBase()
+ file_obj.read = read_with_retries
+ file_obj.__getattr__ = lambda _, attr: getattr(orig_file_obj, attr)
+ return file_obj
+
+
+def _prepare_path_and_storage_options(
+ urlpath: str, download_config: Optional[DownloadConfig] = None
+) -> tuple[str, dict[str, dict[str, Any]]]:
+ prepared_urlpath = []
+ prepared_storage_options = {}
+ for hop in urlpath.split("::"):
+ hop, storage_options = _prepare_single_hop_path_and_storage_options(hop, download_config=download_config)
+ prepared_urlpath.append(hop)
+ prepared_storage_options.update(storage_options)
+ return "::".join(prepared_urlpath), storage_options
+
+
+def _prepare_single_hop_path_and_storage_options(
+ urlpath: str, download_config: Optional[DownloadConfig] = None
+) -> tuple[str, dict[str, dict[str, Any]]]:
+ """
+ Prepare the URL and the kwargs that must be passed to the HttpFileSystem or HfFileSystem
+
+ In particular it resolves google drive URLs
+ It also adds the authentication headers for the Hugging Face Hub, for both https:// and hf:// paths.
+
+ Storage options are formatted in the form {protocol: storage_options_for_protocol}
+ """
+ token = None if download_config is None else download_config.token
+ if urlpath.startswith(config.HF_ENDPOINT) and "/resolve/" in urlpath:
+ urlpath = "hf://" + urlpath[len(config.HF_ENDPOINT) + 1 :].replace("/resolve/", "@", 1)
+ protocol = urlpath.split("://")[0] if "://" in urlpath else "file"
+ if download_config is not None and protocol in download_config.storage_options:
+ storage_options = download_config.storage_options[protocol].copy()
+ elif download_config is not None and protocol not in download_config.storage_options:
+ storage_options = {
+ option_name: option_value
+ for option_name, option_value in download_config.storage_options.items()
+ if option_name not in fsspec.available_protocols()
+ }
+ else:
+ storage_options = {}
+ if protocol in {"http", "https"}:
+ client_kwargs = storage_options.pop("client_kwargs", {})
+ storage_options["client_kwargs"] = {"trust_env": True, **client_kwargs} # Enable reading proxy env variables
+ if "drive.google.com" in urlpath:
+ response = get_session().head(urlpath, timeout=10)
+ for k, v in response.cookies.items():
+ if k.startswith("download_warning"):
+ urlpath += "&confirm=" + v
+ cookies = response.cookies
+ storage_options = {"cookies": cookies, **storage_options}
+ # Fix Google Drive URL to avoid Virus scan warning
+ if "confirm=" not in urlpath:
+ urlpath += "&confirm=t"
+ if urlpath.startswith("https://raw.githubusercontent.com/"):
+ # Workaround for served data with gzip content-encoding: https://github.com/fsspec/filesystem_spec/issues/389
+ headers = storage_options.pop("headers", {})
+ storage_options["headers"] = {"Accept-Encoding": "identity", **headers}
+ elif protocol == "hf":
+ storage_options = {
+ "endpoint": config.HF_ENDPOINT,
+ "token": token,
+ **storage_options,
+ }
+ if storage_options:
+ storage_options = {protocol: storage_options}
+ return urlpath, storage_options
+
+
+def xopen(file: str, mode="r", *args, download_config: Optional[DownloadConfig] = None, **kwargs):
+ """Extend `open` function to support remote files using `fsspec`.
+
+ It also has a retry mechanism in case connection fails.
+ The `args` and `kwargs` are passed to `fsspec.open`, except `token` which is used for queries to private repos on huggingface.co
+
+ Args:
+ file (`str`): Path name of the file to be opened.
+ mode (`str`, *optional*, default "r"): Mode in which the file is opened.
+ *args: Arguments to be passed to `fsspec.open`.
+ download_config : mainly use token or storage_options to support different platforms and auth types.
+ **kwargs: Keyword arguments to be passed to `fsspec.open`.
+
+ Returns:
+ file object
+ """
+ # This works as well for `xopen(str(Path(...)))`
+ file_str = _as_str(file)
+ main_hop, *rest_hops = file_str.split("::")
+ if is_local_path(main_hop):
+ # ignore fsspec-specific kwargs
+ kwargs.pop("block_size", None)
+ return open(main_hop, mode, *args, **kwargs)
+ # add headers and cookies for authentication on the HF Hub and for Google Drive
+ file, storage_options = _prepare_path_and_storage_options(file_str, download_config=download_config)
+ kwargs = {**kwargs, **(storage_options or {})}
+
+ max_retries = config.STREAMING_OPEN_MAX_RETRIES
+
+ disconnect_err = None
+ for retry in range(1, max_retries + 1):
+ try:
+ file_obj = fsspec.open(file, mode=mode, *args, **kwargs).open()
+ break
+ except CONNECTION_ERRORS_TO_RETRY as err:
+ disconnect_err = err
+ logger.warning(
+ f"Failed to connect to remote data host. Retrying in {config.STREAMING_OPEN_RETRY_INTERVAL}sec [{retry}/{max_retries}]"
+ )
+ time.sleep(config.STREAMING_OPEN_RETRY_INTERVAL)
+ except ValueError as e:
+ if str(e) == "Cannot seek streaming HTTP file":
+ raise NonStreamableDatasetError(
+ "Streaming is not possible for this dataset because data host server doesn't support HTTP range "
+ "requests. You can still load this dataset in non-streaming mode by passing `streaming=False` (default)"
+ ) from e
+ else:
+ raise
+ except FileNotFoundError:
+ if file.startswith(config.HF_ENDPOINT):
+ raise FileNotFoundError(
+ file + "\nIf the repo is private or gated, make sure to log in with `huggingface-cli login`."
+ ) from None
+ else:
+ raise
+ else:
+ raise ConnectionError("Server Disconnected") from disconnect_err
+ file_obj = _add_retries_to_file_obj_read_method(file_obj)
+ return file_obj
+
+
+def xlistdir(path: str, download_config: Optional[DownloadConfig] = None) -> list[str]:
+ """Extend `os.listdir` function to support remote files.
+
+ Args:
+ path (`str`): URL path.
+ download_config : mainly use token or storage_options to support different platforms and auth types.
+
+ Returns:
+ `list` of `str`
+ """
+ main_hop, *rest_hops = _as_str(path).split("::")
+ if is_local_path(main_hop):
+ return os.listdir(path)
+ else:
+ # globbing inside a zip in a private repo requires authentication
+ path, storage_options = _prepare_path_and_storage_options(path, download_config=download_config)
+ main_hop, *rest_hops = path.split("::")
+ fs, *_ = url_to_fs(path, **storage_options)
+ inner_path = main_hop.split("://")[-1]
+ if inner_path.strip("/") and not fs.isdir(inner_path):
+ raise FileNotFoundError(f"Directory doesn't exist: {path}")
+ paths = fs.listdir(inner_path, detail=False)
+ return [os.path.basename(path.rstrip("/")) for path in paths]
+
+
+def xglob(urlpath, *, recursive=False, download_config: Optional[DownloadConfig] = None):
+ """Extend `glob.glob` function to support remote files.
+
+ Args:
+ urlpath (`str`): URL path with shell-style wildcard patterns.
+ recursive (`bool`, default `False`): Whether to match the "**" pattern recursively to zero or more
+ directories or subdirectories.
+ download_config : mainly use token or storage_options to support different platforms and auth types.
+
+ Returns:
+ `list` of `str`
+ """
+ main_hop, *rest_hops = _as_str(urlpath).split("::")
+ if is_local_path(main_hop):
+ return glob.glob(main_hop, recursive=recursive)
+ else:
+ # globbing inside a zip in a private repo requires authentication
+ urlpath, storage_options = _prepare_path_and_storage_options(urlpath, download_config=download_config)
+ main_hop, *rest_hops = urlpath.split("::")
+ fs, *_ = url_to_fs(urlpath, **storage_options)
+ inner_path = main_hop.split("://")[1]
+ globbed_paths = fs.glob(inner_path)
+ protocol = fs.protocol if isinstance(fs.protocol, str) else fs.protocol[-1]
+ return ["::".join([f"{protocol}://{globbed_path}"] + rest_hops) for globbed_path in globbed_paths]
+
+
+def xwalk(urlpath, download_config: Optional[DownloadConfig] = None, **kwargs):
+ """Extend `os.walk` function to support remote files.
+
+ Args:
+ urlpath (`str`): URL root path.
+ download_config : mainly use token or storage_options to support different platforms and auth types.
+ **kwargs: Additional keyword arguments forwarded to the underlying filesystem.
+
+
+ Yields:
+ `tuple`: 3-tuple (dirpath, dirnames, filenames).
+ """
+ main_hop, *rest_hops = _as_str(urlpath).split("::")
+ if is_local_path(main_hop):
+ yield from os.walk(main_hop, **kwargs)
+ else:
+ # walking inside a zip in a private repo requires authentication
+ urlpath, storage_options = _prepare_path_and_storage_options(urlpath, download_config=download_config)
+ main_hop, *rest_hops = urlpath.split("::")
+ fs, *_ = url_to_fs(urlpath, **storage_options)
+ inner_path = main_hop.split("://")[-1]
+ if inner_path.strip("/") and not fs.isdir(inner_path):
+ return []
+ protocol = fs.protocol if isinstance(fs.protocol, str) else fs.protocol[-1]
+ for dirpath, dirnames, filenames in fs.walk(inner_path, **kwargs):
+ yield "::".join([f"{protocol}://{dirpath}"] + rest_hops), dirnames, filenames
+
+
+class xPath(type(Path())):
+ """Extension of `pathlib.Path` to support both local paths and remote URLs."""
+
+ def __str__(self):
+ path_str = super().__str__()
+ main_hop, *rest_hops = path_str.split("::")
+ if is_local_path(main_hop):
+ return main_hop
+ path_as_posix = path_str.replace("\\", "/")
+ path_as_posix = SINGLE_SLASH_AFTER_PROTOCOL_PATTERN.sub("://", path_as_posix)
+ path_as_posix += "//" if path_as_posix.endswith(":") else "" # Add slashes to root of the protocol
+ return path_as_posix
+
+ def exists(self, download_config: Optional[DownloadConfig] = None):
+ """Extend `pathlib.Path.exists` method to support both local and remote files.
+
+ Args:
+ download_config : mainly use token or storage_options to support different platforms and auth types.
+
+ Returns:
+ `bool`
+ """
+ return xexists(str(self), download_config=download_config)
+
+ def glob(self, pattern, download_config: Optional[DownloadConfig] = None):
+ """Glob function for argument of type :obj:`~pathlib.Path` that supports both local paths end remote URLs.
+
+ Args:
+ pattern (`str`): Pattern that resulting paths must match.
+ download_config : mainly use token or storage_options to support different platforms and auth types.
+
+ Yields:
+ [`xPath`]
+ """
+ posix_path = self.as_posix()
+ main_hop, *rest_hops = posix_path.split("::")
+ if is_local_path(main_hop):
+ yield from Path(main_hop).glob(pattern)
+ else:
+ # globbing inside a zip in a private repo requires authentication
+ if rest_hops:
+ urlpath = rest_hops[0]
+ urlpath, storage_options = _prepare_path_and_storage_options(urlpath, download_config=download_config)
+ storage_options = {urlpath.split("://")[0]: storage_options}
+ posix_path = "::".join([main_hop, urlpath, *rest_hops[1:]])
+ else:
+ storage_options = None
+ fs, *_ = url_to_fs(xjoin(posix_path, pattern), **(storage_options or {}))
+ globbed_paths = fs.glob(xjoin(main_hop, pattern))
+ for globbed_path in globbed_paths:
+ yield type(self)("::".join([f"{fs.protocol}://{globbed_path}"] + rest_hops))
+
+ def rglob(self, pattern, **kwargs):
+ """Rglob function for argument of type :obj:`~pathlib.Path` that supports both local paths end remote URLs.
+
+ Args:
+ pattern (`str`): Pattern that resulting paths must match.
+
+ Yields:
+ [`xPath`]
+ """
+ return self.glob("**/" + pattern, **kwargs)
+
+ @property
+ def parent(self) -> "xPath":
+ """Name function for argument of type :obj:`~pathlib.Path` that supports both local paths end remote URLs.
+
+ Returns:
+ [`xPath`]
+ """
+ return type(self)(xdirname(self.as_posix()))
+
+ @property
+ def name(self) -> str:
+ """Name function for argument of type :obj:`~pathlib.Path` that supports both local paths end remote URLs.
+
+ Returns:
+ `str`
+ """
+ return PurePosixPath(self.as_posix().split("::")[0]).name
+
+ @property
+ def stem(self) -> str:
+ """Stem function for argument of type :obj:`~pathlib.Path` that supports both local paths end remote URLs.
+
+ Returns:
+ `str`
+ """
+ return PurePosixPath(self.as_posix().split("::")[0]).stem
+
+ @property
+ def suffix(self) -> str:
+ """Suffix function for argument of type :obj:`~pathlib.Path` that supports both local paths end remote URLs.
+
+ Returns:
+ `str`
+ """
+ return PurePosixPath(self.as_posix().split("::")[0]).suffix
+
+ def open(self, *args, **kwargs):
+ """Extend :func:`xopen` to support argument of type :obj:`~pathlib.Path`.
+
+ Args:
+ **args: Arguments passed to :func:`fsspec.open`.
+ **kwargs: Keyword arguments passed to :func:`fsspec.open`.
+
+ Returns:
+ `io.FileIO`: File-like object.
+ """
+ return xopen(str(self), *args, **kwargs)
+
+ def joinpath(self, *p: tuple[str, ...]) -> "xPath":
+ """Extend :func:`xjoin` to support argument of type :obj:`~pathlib.Path`.
+
+ Args:
+ *p (`tuple` of `str`): Other path components.
+
+ Returns:
+ [`xPath`]
+ """
+ return type(self)(xjoin(self.as_posix(), *p))
+
+ def __truediv__(self, p: str) -> "xPath":
+ return self.joinpath(p)
+
+ def with_suffix(self, suffix):
+ main_hop, *rest_hops = str(self).split("::")
+ if is_local_path(main_hop):
+ return type(self)(str(super().with_suffix(suffix)))
+ return type(self)("::".join([type(self)(PurePosixPath(main_hop).with_suffix(suffix)).as_posix()] + rest_hops))
+
+
+def _as_str(path: Union[str, Path, xPath]):
+ return str(path) if isinstance(path, xPath) else str(xPath(str(path)))
+
+
+def xgzip_open(filepath_or_buffer, *args, download_config: Optional[DownloadConfig] = None, **kwargs):
+ import gzip
+
+ if hasattr(filepath_or_buffer, "read"):
+ return gzip.open(filepath_or_buffer, *args, **kwargs)
+ else:
+ filepath_or_buffer = str(filepath_or_buffer)
+ return gzip.open(xopen(filepath_or_buffer, "rb", download_config=download_config), *args, **kwargs)
+
+
+def xnumpy_load(filepath_or_buffer, *args, download_config: Optional[DownloadConfig] = None, **kwargs):
+ import numpy as np
+
+ if hasattr(filepath_or_buffer, "read"):
+ return np.load(filepath_or_buffer, *args, **kwargs)
+ else:
+ filepath_or_buffer = str(filepath_or_buffer)
+ return np.load(xopen(filepath_or_buffer, "rb", download_config=download_config), *args, **kwargs)
+
+
+def xpandas_read_csv(filepath_or_buffer, download_config: Optional[DownloadConfig] = None, **kwargs):
+ import pandas as pd
+
+ if hasattr(filepath_or_buffer, "read"):
+ return pd.read_csv(filepath_or_buffer, **kwargs)
+ else:
+ filepath_or_buffer = str(filepath_or_buffer)
+ if kwargs.get("compression", "infer") == "infer":
+ kwargs["compression"] = _get_extraction_protocol(filepath_or_buffer, download_config=download_config)
+ return pd.read_csv(xopen(filepath_or_buffer, "rb", download_config=download_config), **kwargs)
+
+
+def xpandas_read_excel(filepath_or_buffer, download_config: Optional[DownloadConfig] = None, **kwargs):
+ import pandas as pd
+
+ if hasattr(filepath_or_buffer, "read"):
+ try:
+ return pd.read_excel(filepath_or_buffer, **kwargs)
+ except ValueError: # Cannot seek streaming HTTP file
+ return pd.read_excel(BytesIO(filepath_or_buffer.read()), **kwargs)
+ else:
+ filepath_or_buffer = str(filepath_or_buffer)
+ try:
+ return pd.read_excel(xopen(filepath_or_buffer, "rb", download_config=download_config), **kwargs)
+ except ValueError: # Cannot seek streaming HTTP file
+ return pd.read_excel(
+ BytesIO(xopen(filepath_or_buffer, "rb", download_config=download_config).read()), **kwargs
+ )
+
+
+def xpyarrow_parquet_read_table(filepath_or_buffer, download_config: Optional[DownloadConfig] = None, **kwargs):
+ import pyarrow.parquet as pq
+
+ if hasattr(filepath_or_buffer, "read"):
+ return pq.read_table(filepath_or_buffer, **kwargs)
+ else:
+ filepath_or_buffer = str(filepath_or_buffer)
+ return pq.read_table(xopen(filepath_or_buffer, mode="rb", download_config=download_config), **kwargs)
+
+
+def xsio_loadmat(filepath_or_buffer, download_config: Optional[DownloadConfig] = None, **kwargs):
+ import scipy.io as sio
+
+ if hasattr(filepath_or_buffer, "read"):
+ return sio.loadmat(filepath_or_buffer, **kwargs)
+ else:
+ return sio.loadmat(xopen(filepath_or_buffer, "rb", download_config=download_config), **kwargs)
+
+
+def xet_parse(source, parser=None, download_config: Optional[DownloadConfig] = None):
+ """Extend `xml.etree.ElementTree.parse` function to support remote files.
+
+ Args:
+ source: File path or file object.
+ parser (`XMLParser`, *optional*, default `XMLParser`): Parser instance.
+ download_config : mainly use token or storage_options to support different platforms and auth types.
+
+ Returns:
+ `xml.etree.ElementTree.Element`: Root element of the given source document.
+ """
+ if hasattr(source, "read"):
+ return ET.parse(source, parser=parser)
+ else:
+ with xopen(source, "rb", download_config=download_config) as f:
+ return ET.parse(f, parser=parser)
+
+
+def xxml_dom_minidom_parse(filename_or_file, download_config: Optional[DownloadConfig] = None, **kwargs):
+ """Extend `xml.dom.minidom.parse` function to support remote files.
+
+ Args:
+ filename_or_file (`str` or file): File path or file object.
+ download_config : mainly use token or storage_options to support different platforms and auth types.
+ **kwargs (optional): Additional keyword arguments passed to `xml.dom.minidom.parse`.
+
+ Returns:
+ :obj:`xml.dom.minidom.Document`: Parsed document.
+ """
+ if hasattr(filename_or_file, "read"):
+ return xml.dom.minidom.parse(filename_or_file, **kwargs)
+ else:
+ with xopen(filename_or_file, "rb", download_config=download_config) as f:
+ return xml.dom.minidom.parse(f, **kwargs)
+
+
+class ArchiveIterable(TrackedIterableFromGenerator):
+ """An iterable of (path, fileobj) from a TAR archive, used by `iter_archive`"""
+
+ @staticmethod
+ def _iter_tar(f):
+ stream = tarfile.open(fileobj=f, mode="r|*")
+ for tarinfo in stream:
+ file_path = tarinfo.name
+ if not tarinfo.isreg():
+ continue
+ if file_path is None:
+ continue
+ if os.path.basename(file_path).startswith((".", "__")):
+ # skipping hidden files
+ continue
+ file_obj = stream.extractfile(tarinfo)
+ yield file_path, file_obj
+ stream.members = []
+ del stream
+
+ @staticmethod
+ def _iter_zip(f):
+ zipf = zipfile.ZipFile(f)
+ for member in zipf.infolist():
+ file_path = member.filename
+ if member.is_dir():
+ continue
+ if file_path is None:
+ continue
+ if os.path.basename(file_path).startswith((".", "__")):
+ # skipping hidden files
+ continue
+ file_obj = zipf.open(member)
+ yield file_path, file_obj
+
+ @classmethod
+ def _iter_from_fileobj(cls, f) -> Generator[tuple, None, None]:
+ compression = _get_extraction_protocol_with_magic_number(f)
+ if compression == "zip":
+ yield from cls._iter_zip(f)
+ else:
+ yield from cls._iter_tar(f)
+
+ @classmethod
+ def _iter_from_urlpath(
+ cls, urlpath: str, download_config: Optional[DownloadConfig] = None
+ ) -> Generator[tuple, None, None]:
+ compression = _get_extraction_protocol(urlpath, download_config=download_config)
+ # Set block_size=0 to get faster streaming
+ # (e.g. for hf:// and https:// it uses streaming Requests file-like instances)
+ with xopen(urlpath, "rb", download_config=download_config, block_size=0) as f:
+ if compression == "zip":
+ yield from cls._iter_zip(f)
+ else:
+ yield from cls._iter_tar(f)
+
+ @classmethod
+ def from_buf(cls, fileobj) -> "ArchiveIterable":
+ return cls(cls._iter_from_fileobj, fileobj)
+
+ @classmethod
+ def from_urlpath(cls, urlpath_or_buf, download_config: Optional[DownloadConfig] = None) -> "ArchiveIterable":
+ return cls(cls._iter_from_urlpath, urlpath_or_buf, download_config)
+
+
+class FilesIterable(TrackedIterableFromGenerator):
+ """An iterable of paths from a list of directories or files"""
+
+ @classmethod
+ def _iter_from_urlpaths(
+ cls, urlpaths: Union[str, list[str]], download_config: Optional[DownloadConfig] = None
+ ) -> Generator[str, None, None]:
+ if not isinstance(urlpaths, list):
+ urlpaths = [urlpaths]
+ for urlpath in urlpaths:
+ if xisfile(urlpath, download_config=download_config):
+ yield urlpath
+ elif xisdir(urlpath, download_config=download_config):
+ for dirpath, dirnames, filenames in xwalk(urlpath, download_config=download_config):
+ # in-place modification to prune the search
+ dirnames[:] = sorted([dirname for dirname in dirnames if not dirname.startswith((".", "__"))])
+ if xbasename(dirpath).startswith((".", "__")):
+ # skipping hidden directories
+ continue
+ for filename in sorted(filenames):
+ if filename.startswith((".", "__")):
+ # skipping hidden files
+ continue
+ yield xjoin(dirpath, filename)
+ else:
+ raise FileNotFoundError(urlpath)
+
+ @classmethod
+ def from_urlpaths(cls, urlpaths, download_config: Optional[DownloadConfig] = None) -> "FilesIterable":
+ return cls(cls._iter_from_urlpaths, urlpaths, download_config)
diff --git a/venv/lib/python3.10/site-packages/datasets/utils/filelock.py b/venv/lib/python3.10/site-packages/datasets/utils/filelock.py
new file mode 100644
index 0000000000000000000000000000000000000000..df0728efe644d8eb32f0e578a85e39ba366e9743
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/utils/filelock.py
@@ -0,0 +1,11 @@
+# deprecated, please use the `filelock` package instead
+
+from filelock import ( # noqa: F401 # imported for backward compatibility TODO: remove in 3.0.0
+ BaseFileLock,
+ SoftFileLock,
+ Timeout,
+ UnixFileLock,
+ WindowsFileLock,
+)
+
+from ._filelock import FileLock # noqa: F401 # imported for backward compatibility. TODO: remove in 3.0.0
diff --git a/venv/lib/python3.10/site-packages/datasets/utils/hub.py b/venv/lib/python3.10/site-packages/datasets/utils/hub.py
new file mode 100644
index 0000000000000000000000000000000000000000..6d784333b23328c113ec7be444d0a77410f1b857
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/utils/hub.py
@@ -0,0 +1,6 @@
+from functools import partial
+
+from huggingface_hub import hf_hub_url
+
+
+hf_dataset_url = partial(hf_hub_url, repo_type="dataset")
diff --git a/venv/lib/python3.10/site-packages/datasets/utils/info_utils.py b/venv/lib/python3.10/site-packages/datasets/utils/info_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..d93f5b4509f47c5c12ad98c11fb34acfdab9b714
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/utils/info_utils.py
@@ -0,0 +1,106 @@
+import enum
+import os
+from typing import Optional
+
+from huggingface_hub.utils import insecure_hashlib
+
+from .. import config
+from ..exceptions import (
+ ExpectedMoreDownloadedFilesError,
+ ExpectedMoreSplitsError,
+ NonMatchingChecksumError,
+ NonMatchingSplitsSizesError,
+ UnexpectedDownloadedFileError,
+ UnexpectedSplitsError,
+)
+from .logging import get_logger
+
+
+logger = get_logger(__name__)
+
+
+class VerificationMode(enum.Enum):
+ """`Enum` that specifies which verification checks to run.
+
+ The default mode is `BASIC_CHECKS`, which will perform only rudimentary checks to avoid slowdowns
+ when generating/downloading a dataset for the first time.
+
+ The verification modes:
+
+ | | Verification checks |
+ |---------------------------|------------------------------------------------------------------------------ |
+ | `ALL_CHECKS` | Split checks, uniqueness of the keys yielded in case of the GeneratorBuilder |
+ | | and the validity (number of files, checksums, etc.) of downloaded files |
+ | `BASIC_CHECKS` (default) | Same as `ALL_CHECKS` but without checking downloaded files |
+ | `NO_CHECKS` | None |
+
+ """
+
+ ALL_CHECKS = "all_checks"
+ BASIC_CHECKS = "basic_checks"
+ NO_CHECKS = "no_checks"
+
+
+def verify_checksums(expected_checksums: Optional[dict], recorded_checksums: dict, verification_name=None):
+ if expected_checksums is None:
+ logger.info("Unable to verify checksums.")
+ return
+ if len(set(expected_checksums) - set(recorded_checksums)) > 0:
+ raise ExpectedMoreDownloadedFilesError(str(set(expected_checksums) - set(recorded_checksums)))
+ if len(set(recorded_checksums) - set(expected_checksums)) > 0:
+ raise UnexpectedDownloadedFileError(str(set(recorded_checksums) - set(expected_checksums)))
+ bad_urls = [url for url in expected_checksums if expected_checksums[url] != recorded_checksums[url]]
+ for_verification_name = " for " + verification_name if verification_name is not None else ""
+ if len(bad_urls) > 0:
+ raise NonMatchingChecksumError(
+ f"Checksums didn't match{for_verification_name}:\n"
+ f"{bad_urls}\n"
+ "Set `verification_mode='no_checks'` to skip checksums verification and ignore this error"
+ )
+ logger.info("All the checksums matched successfully" + for_verification_name)
+
+
+def verify_splits(expected_splits: Optional[dict], recorded_splits: dict):
+ if expected_splits is None:
+ logger.info("Unable to verify splits sizes.")
+ return
+ if len(set(expected_splits) - set(recorded_splits)) > 0:
+ raise ExpectedMoreSplitsError(str(set(expected_splits) - set(recorded_splits)))
+ if len(set(recorded_splits) - set(expected_splits)) > 0:
+ raise UnexpectedSplitsError(str(set(recorded_splits) - set(expected_splits)))
+ bad_splits = [
+ {"expected": expected_splits[name], "recorded": recorded_splits[name]}
+ for name in expected_splits
+ if expected_splits[name].num_examples != recorded_splits[name].num_examples
+ ]
+ if len(bad_splits) > 0:
+ raise NonMatchingSplitsSizesError(str(bad_splits))
+ logger.info("All the splits matched successfully.")
+
+
+def get_size_checksum_dict(path: str, record_checksum: bool = True) -> dict:
+ """Compute the file size and the sha256 checksum of a file"""
+ if record_checksum:
+ m = insecure_hashlib.sha256()
+ with open(path, "rb") as f:
+ for chunk in iter(lambda: f.read(1 << 20), b""):
+ m.update(chunk)
+ checksum = m.hexdigest()
+ else:
+ checksum = None
+ return {"num_bytes": os.path.getsize(path), "checksum": checksum}
+
+
+def is_small_dataset(dataset_size):
+ """Check if `dataset_size` is smaller than `config.IN_MEMORY_MAX_SIZE`.
+
+ Args:
+ dataset_size (int): Dataset size in bytes.
+
+ Returns:
+ bool: Whether `dataset_size` is smaller than `config.IN_MEMORY_MAX_SIZE`.
+ """
+ if dataset_size and config.IN_MEMORY_MAX_SIZE:
+ return dataset_size < config.IN_MEMORY_MAX_SIZE
+ else:
+ return False
diff --git a/venv/lib/python3.10/site-packages/datasets/utils/logging.py b/venv/lib/python3.10/site-packages/datasets/utils/logging.py
new file mode 100644
index 0000000000000000000000000000000000000000..851c1a2b73d2f411dbd180e16e37daf91181e13c
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/utils/logging.py
@@ -0,0 +1,175 @@
+# Copyright 2020 Optuna, Hugging Face
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Logging utilities."""
+
+import logging
+import os
+from logging import (
+ CRITICAL, # NOQA
+ DEBUG, # NOQA
+ ERROR, # NOQA
+ FATAL, # NOQA
+ INFO, # NOQA
+ NOTSET, # NOQA
+ WARN, # NOQA
+ WARNING, # NOQA
+)
+from typing import Optional
+
+from .tqdm import ( # noqa: F401 # imported for backward compatibility
+ disable_progress_bar,
+ enable_progress_bar,
+ is_progress_bar_enabled,
+ tqdm,
+)
+
+
+log_levels = {
+ "debug": logging.DEBUG,
+ "info": logging.INFO,
+ "warning": logging.WARNING,
+ "error": logging.ERROR,
+ "critical": logging.CRITICAL,
+}
+
+_default_log_level = logging.WARNING
+
+
+def _get_default_logging_level():
+ """
+ If DATASETS_VERBOSITY env var is set to one of the valid choices return that as the new default level.
+ If it is not - fall back to ``_default_log_level``
+ """
+ env_level_str = os.getenv("DATASETS_VERBOSITY", None)
+ if env_level_str:
+ if env_level_str in log_levels:
+ return log_levels[env_level_str]
+ else:
+ logging.getLogger().warning(
+ f"Unknown option DATASETS_VERBOSITY={env_level_str}, has to be one of: {', '.join(log_levels.keys())}"
+ )
+ return _default_log_level
+
+
+def _get_library_name() -> str:
+ return __name__.split(".")[0]
+
+
+def _get_library_root_logger() -> logging.Logger:
+ return logging.getLogger(_get_library_name())
+
+
+def _configure_library_root_logger() -> None:
+ # Apply our default configuration to the library root logger.
+ library_root_logger = _get_library_root_logger()
+ library_root_logger.addHandler(logging.StreamHandler())
+ library_root_logger.setLevel(_get_default_logging_level())
+
+
+def _reset_library_root_logger() -> None:
+ library_root_logger = _get_library_root_logger()
+ library_root_logger.setLevel(logging.NOTSET)
+
+
+def get_logger(name: Optional[str] = None) -> logging.Logger:
+ """Return a logger with the specified name.
+ This function can be used in dataset builders.
+ """
+ if name is None:
+ name = _get_library_name()
+ return logging.getLogger(name)
+
+
+def get_verbosity() -> int:
+ """Return the current level for the HuggingFace datasets library's root logger.
+ Returns:
+ Logging level, e.g., `datasets.logging.DEBUG` and `datasets.logging.INFO`.
+
+ > [!TIP]
+ > HuggingFace datasets library has following logging levels:
+ > - `datasets.logging.CRITICAL`, `datasets.logging.FATAL`
+ > - `datasets.logging.ERROR`
+ > - `datasets.logging.WARNING`, `datasets.logging.WARN`
+ > - `datasets.logging.INFO`
+ > - `datasets.logging.DEBUG`
+ """
+ return _get_library_root_logger().getEffectiveLevel()
+
+
+def set_verbosity(verbosity: int) -> None:
+ """Set the level for the Hugging Face Datasets library's root logger.
+ Args:
+ verbosity:
+ Logging level, e.g., `datasets.logging.DEBUG` and `datasets.logging.INFO`.
+ """
+ _get_library_root_logger().setLevel(verbosity)
+
+
+def set_verbosity_info():
+ """Set the level for the Hugging Face datasets library's root logger to `INFO`.
+
+ This will display most of the logging information and tqdm bars.
+
+ Shortcut to `datasets.logging.set_verbosity(datasets.logging.INFO)`.
+ """
+ return set_verbosity(INFO)
+
+
+def set_verbosity_warning():
+ """Set the level for the Hugging Face datasets library's root logger to `WARNING`.
+
+ This will display only the warning and errors logging information and tqdm bars.
+
+ Shortcut to `datasets.logging.set_verbosity(datasets.logging.WARNING)`.
+ """
+ return set_verbosity(WARNING)
+
+
+def set_verbosity_debug():
+ """Set the level for the Hugging Face datasets library's root logger to `DEBUG`.
+
+ This will display all the logging information and tqdm bars.
+
+ Shortcut to `datasets.logging.set_verbosity(datasets.logging.DEBUG)`.
+ """
+ return set_verbosity(DEBUG)
+
+
+def set_verbosity_error():
+ """Set the level for the Hugging Face datasets library's root logger to `ERROR`.
+
+ This will display only the errors logging information and tqdm bars.
+
+ Shortcut to `datasets.logging.set_verbosity(datasets.logging.ERROR)`.
+ """
+ return set_verbosity(ERROR)
+
+
+def disable_propagation() -> None:
+ """Disable propagation of the library log outputs.
+ Note that log propagation is disabled by default.
+ """
+ _get_library_root_logger().propagate = False
+
+
+def enable_propagation() -> None:
+ """Enable propagation of the library log outputs.
+ Please disable the Hugging Face datasets library's default handler to prevent double logging if the root logger has
+ been configured.
+ """
+ _get_library_root_logger().propagate = True
+
+
+# Configure the library root logger at the module level (singleton-like)
+_configure_library_root_logger()
diff --git a/venv/lib/python3.10/site-packages/datasets/utils/metadata.py b/venv/lib/python3.10/site-packages/datasets/utils/metadata.py
new file mode 100644
index 0000000000000000000000000000000000000000..cceecf35d63f518fcb7a62c804b9679e5d0c144e
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/utils/metadata.py
@@ -0,0 +1,233 @@
+import re
+import textwrap
+from collections import Counter
+from itertools import groupby
+from operator import itemgetter
+from typing import Any, ClassVar, Optional
+
+import yaml
+from huggingface_hub import DatasetCardData
+
+from ..config import METADATA_CONFIGS_FIELD
+from ..features import Features
+from ..info import DatasetInfo, DatasetInfosDict
+from ..naming import _split_re
+from ..utils.logging import get_logger
+
+
+logger = get_logger(__name__)
+
+
+class _NoDuplicateSafeLoader(yaml.SafeLoader):
+ def _check_no_duplicates_on_constructed_node(self, node):
+ keys = [self.constructed_objects[key_node] for key_node, _ in node.value]
+ keys = [tuple(key) if isinstance(key, list) else key for key in keys]
+ counter = Counter(keys)
+ duplicate_keys = [key for key in counter if counter[key] > 1]
+ if duplicate_keys:
+ raise TypeError(f"Got duplicate yaml keys: {duplicate_keys}")
+
+ def construct_mapping(self, node, deep=False):
+ mapping = super().construct_mapping(node, deep=deep)
+ self._check_no_duplicates_on_constructed_node(node)
+ return mapping
+
+
+def _split_yaml_from_readme(readme_content: str) -> tuple[Optional[str], str]:
+ full_content = list(readme_content.splitlines())
+ if full_content and full_content[0] == "---" and "---" in full_content[1:]:
+ sep_idx = full_content[1:].index("---") + 1
+ yamlblock = "\n".join(full_content[1:sep_idx])
+ return yamlblock, "\n".join(full_content[sep_idx + 1 :])
+
+ return None, "\n".join(full_content)
+
+
+class MetadataConfigs(dict[str, dict[str, Any]]):
+ """Should be in format {config_name: {**config_params}}."""
+
+ FIELD_NAME: ClassVar[str] = METADATA_CONFIGS_FIELD
+
+ @staticmethod
+ def _raise_if_data_files_field_not_valid(metadata_config: dict):
+ yaml_data_files = metadata_config.get("data_files")
+ if yaml_data_files is not None:
+ yaml_error_message = textwrap.dedent(
+ f"""
+ Expected data_files in YAML to be either a string or a list of strings
+ or a list of dicts with two keys: 'split' and 'path', but got {yaml_data_files}
+ Examples of data_files in YAML:
+
+ data_files: data.csv
+
+ data_files: data/*.png
+
+ data_files:
+ - part0/*
+ - part1/*
+
+ data_files:
+ - split: train
+ path: train/*
+ - split: test
+ path: test/*
+
+ data_files:
+ - split: train
+ path:
+ - train/part1/*
+ - train/part2/*
+ - split: test
+ path: test/*
+
+ PS: some symbols like dashes '-' are not allowed in split names
+ """
+ )
+ if not isinstance(yaml_data_files, (list, str)):
+ raise ValueError(yaml_error_message)
+ if isinstance(yaml_data_files, list):
+ for yaml_data_files_item in yaml_data_files:
+ if (
+ not isinstance(yaml_data_files_item, (str, dict))
+ or isinstance(yaml_data_files_item, dict)
+ and not (
+ len(yaml_data_files_item) == 2
+ and "split" in yaml_data_files_item
+ and re.match(_split_re, yaml_data_files_item["split"])
+ and isinstance(yaml_data_files_item.get("path"), (str, list))
+ )
+ ):
+ raise ValueError(yaml_error_message)
+
+ @classmethod
+ def _from_exported_parquet_files_and_dataset_infos(
+ cls,
+ parquet_commit_hash: str,
+ exported_parquet_files: list[dict[str, Any]],
+ dataset_infos: DatasetInfosDict,
+ ) -> "MetadataConfigs":
+ metadata_configs = {
+ config_name: {
+ "data_files": [
+ {
+ "split": split_name,
+ "path": [
+ parquet_file["url"].replace("refs%2Fconvert%2Fparquet", parquet_commit_hash)
+ for parquet_file in parquet_files_for_split
+ ],
+ }
+ for split_name, parquet_files_for_split in groupby(parquet_files_for_config, itemgetter("split"))
+ ],
+ "version": str(dataset_infos.get(config_name, DatasetInfo()).version or "0.0.0"),
+ }
+ for config_name, parquet_files_for_config in groupby(exported_parquet_files, itemgetter("config"))
+ }
+ if dataset_infos:
+ # Preserve order of configs and splits
+ metadata_configs = {
+ config_name: {
+ "data_files": [
+ data_file
+ for split_name in dataset_info.splits
+ for data_file in metadata_configs[config_name]["data_files"]
+ if data_file["split"] == split_name
+ ],
+ "version": metadata_configs[config_name]["version"],
+ }
+ for config_name, dataset_info in dataset_infos.items()
+ }
+ return cls(metadata_configs)
+
+ @classmethod
+ def from_dataset_card_data(cls, dataset_card_data: DatasetCardData) -> "MetadataConfigs":
+ if dataset_card_data.get(cls.FIELD_NAME):
+ metadata_configs = dataset_card_data[cls.FIELD_NAME]
+ if not isinstance(metadata_configs, list):
+ raise ValueError(f"Expected {cls.FIELD_NAME} to be a list, but got '{metadata_configs}'")
+ for metadata_config in metadata_configs:
+ if "config_name" not in metadata_config:
+ raise ValueError(
+ f"Each config must include `config_name` field with a string name of a config, "
+ f"but got {metadata_config}. "
+ )
+ cls._raise_if_data_files_field_not_valid(metadata_config)
+ return cls(
+ {
+ config.pop("config_name"): {
+ param: value if param != "features" else Features._from_yaml_list(value)
+ for param, value in config.items()
+ }
+ for metadata_config in metadata_configs
+ if (config := metadata_config.copy())
+ }
+ )
+ return cls()
+
+ def to_dataset_card_data(self, dataset_card_data: DatasetCardData) -> None:
+ if self:
+ for metadata_config in self.values():
+ self._raise_if_data_files_field_not_valid(metadata_config)
+ current_metadata_configs = self.from_dataset_card_data(dataset_card_data)
+ total_metadata_configs = dict(sorted({**current_metadata_configs, **self}.items()))
+ for config_name, config_metadata in total_metadata_configs.items():
+ config_metadata.pop("config_name", None)
+ dataset_card_data[self.FIELD_NAME] = [
+ {"config_name": config_name, **config_metadata}
+ for config_name, config_metadata in total_metadata_configs.items()
+ ]
+
+ def get_default_config_name(self) -> Optional[str]:
+ default_config_name = None
+ for config_name, metadata_config in self.items():
+ if len(self) == 1 or config_name == "default" or metadata_config.get("default"):
+ if default_config_name is None:
+ default_config_name = config_name
+ else:
+ raise ValueError(
+ f"Dataset has several default configs: '{default_config_name}' and '{config_name}'."
+ )
+ return default_config_name
+
+
+# DEPRECATED - just here to support old versions of evaluate like 0.2.2
+# To support new tasks on the Hugging Face Hub, please open a PR for this file:
+# https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/src/pipelines.ts
+known_task_ids = {
+ "image-classification": [],
+ "translation": [],
+ "image-segmentation": [],
+ "fill-mask": [],
+ "automatic-speech-recognition": [],
+ "token-classification": [],
+ "sentence-similarity": [],
+ "audio-classification": [],
+ "question-answering": [],
+ "summarization": [],
+ "zero-shot-classification": [],
+ "table-to-text": [],
+ "feature-extraction": [],
+ "other": [],
+ "multiple-choice": [],
+ "text-classification": [],
+ "text-to-image": [],
+ "text2text-generation": [],
+ "zero-shot-image-classification": [],
+ "tabular-classification": [],
+ "tabular-regression": [],
+ "image-to-image": [],
+ "tabular-to-text": [],
+ "unconditional-image-generation": [],
+ "text-retrieval": [],
+ "text-to-speech": [],
+ "object-detection": [],
+ "audio-to-audio": [],
+ "text-generation": [],
+ "conversational": [],
+ "table-question-answering": [],
+ "visual-question-answering": [],
+ "image-to-text": [],
+ "reinforcement-learning": [],
+ "voice-activity-detection": [],
+ "time-series-forecasting": [],
+ "document-question-answering": [],
+}
diff --git a/venv/lib/python3.10/site-packages/datasets/utils/patching.py b/venv/lib/python3.10/site-packages/datasets/utils/patching.py
new file mode 100644
index 0000000000000000000000000000000000000000..f245cabd97065d9e82a1320d02999f9ec03bda36
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/utils/patching.py
@@ -0,0 +1,119 @@
+from importlib import import_module
+
+from .logging import get_logger
+
+
+logger = get_logger(__name__)
+
+
+class _PatchedModuleObj:
+ """Set all the modules components as attributes of the _PatchedModuleObj object."""
+
+ def __init__(self, module, attrs=None):
+ attrs = attrs or []
+ if module is not None:
+ for key in module.__dict__:
+ if key in attrs or not key.startswith("__"):
+ setattr(self, key, getattr(module, key))
+ self._original_module = module._original_module if isinstance(module, _PatchedModuleObj) else module
+
+
+class patch_submodule:
+ """
+ Patch a submodule attribute of an object, by keeping all other submodules intact at all levels.
+
+ Example::
+
+ >>> import importlib
+ >>> from datasets.load import dataset_module_factory
+ >>> from datasets.streaming import patch_submodule, xjoin
+ >>>
+ >>> dataset_module = dataset_module_factory("snli")
+ >>> snli_module = importlib.import_module(dataset_module.module_path)
+ >>> patcher = patch_submodule(snli_module, "os.path.join", xjoin)
+ >>> patcher.start()
+ >>> assert snli_module.os.path.join is xjoin
+ """
+
+ _active_patches = []
+
+ def __init__(self, obj, target: str, new, attrs=None):
+ self.obj = obj
+ self.target = target
+ self.new = new
+ self.key = target.split(".")[0]
+ self.original = {}
+ self.attrs = attrs or []
+
+ def __enter__(self):
+ *submodules, target_attr = self.target.split(".")
+
+ # Patch modules:
+ # it's used to patch attributes of submodules like "os.path.join";
+ # in this case we need to patch "os" and "os.path"
+
+ for i in range(len(submodules)):
+ try:
+ submodule = import_module(".".join(submodules[: i + 1]))
+ except ModuleNotFoundError:
+ continue
+ # We iterate over all the globals in self.obj in case we find "os" or "os.path"
+ for attr in self.obj.__dir__():
+ obj_attr = getattr(self.obj, attr)
+ # We don't check for the name of the global, but rather if its value *is* "os" or "os.path".
+ # This allows to patch renamed modules like "from os import path as ospath".
+ if obj_attr is submodule or (
+ isinstance(obj_attr, _PatchedModuleObj) and obj_attr._original_module is submodule
+ ):
+ self.original[attr] = obj_attr
+ # patch at top level
+ setattr(self.obj, attr, _PatchedModuleObj(obj_attr, attrs=self.attrs))
+ patched = getattr(self.obj, attr)
+ # construct lower levels patches
+ for key in submodules[i + 1 :]:
+ setattr(patched, key, _PatchedModuleObj(getattr(patched, key, None), attrs=self.attrs))
+ patched = getattr(patched, key)
+ # finally set the target attribute
+ setattr(patched, target_attr, self.new)
+
+ # Patch attribute itself:
+ # it's used for builtins like "open",
+ # and also to patch "os.path.join" we may also need to patch "join"
+ # itself if it was imported as "from os.path import join".
+
+ if submodules: # if it's an attribute of a submodule like "os.path.join"
+ try:
+ attr_value = getattr(import_module(".".join(submodules)), target_attr)
+ except (AttributeError, ModuleNotFoundError):
+ return
+ # We iterate over all the globals in self.obj in case we find "os.path.join"
+ for attr in self.obj.__dir__():
+ # We don't check for the name of the global, but rather if its value *is* "os.path.join".
+ # This allows to patch renamed attributes like "from os.path import join as pjoin".
+ if getattr(self.obj, attr) is attr_value:
+ self.original[attr] = getattr(self.obj, attr)
+ setattr(self.obj, attr, self.new)
+ elif target_attr in globals()["__builtins__"]: # if it'a s builtin like "open"
+ self.original[target_attr] = globals()["__builtins__"][target_attr]
+ setattr(self.obj, target_attr, self.new)
+ else:
+ raise RuntimeError(f"Tried to patch attribute {target_attr} instead of a submodule.")
+
+ def __exit__(self, *exc_info):
+ for attr in list(self.original):
+ setattr(self.obj, attr, self.original.pop(attr))
+
+ def start(self):
+ """Activate a patch."""
+ self.__enter__()
+ self._active_patches.append(self)
+
+ def stop(self):
+ """Stop an active patch."""
+ try:
+ self._active_patches.remove(self)
+ except ValueError:
+ # If the patch hasn't been started this will fail
+ return None
+
+ return self.__exit__()
diff --git a/venv/lib/python3.10/site-packages/datasets/utils/py_utils.py b/venv/lib/python3.10/site-packages/datasets/utils/py_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..f7c38bc1fa37b7c767df98ad8b272e92d56cb7a0
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/utils/py_utils.py
@@ -0,0 +1,642 @@
+# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""Some python utils function and classes."""
+
+import copy
+import functools
+import itertools
+import multiprocessing.pool
+import os
+import queue
+import re
+import types
+import warnings
+from collections.abc import Iterable
+from contextlib import contextmanager
+from dataclasses import fields, is_dataclass
+from multiprocessing import Manager
+from queue import Empty
+from shutil import disk_usage
+from typing import Any, Callable, Optional, TypeVar, Union
+
+import multiprocess
+import multiprocess.pool
+import numpy as np
+from tqdm.auto import tqdm
+
+from .. import config
+from ..parallel import parallel_map
+from . import logging
+from . import tqdm as hf_tqdm
+from ._dill import ( # noqa: F401 # imported for backward compatibility. TODO: remove in 3.0.0
+ Pickler,
+ dump,
+ dumps,
+ pklregister,
+)
+
+
+try: # pragma: no branch
+ from typing import Final
+
+ import typing_extensions as _typing_extensions
+ from typing_extensions import Literal
+except ImportError:
+ _typing_extensions = Literal = Final = None
+
+
+logger = logging.get_logger(__name__)
+
+
+# NOTE: When used on an instance method, the cache is shared across all
+# instances and IS NOT per-instance.
+# See
+# https://stackoverflow.com/questions/14946264/python-lru-cache-decorator-per-instance
+# For @property methods, use @memoized_property below.
+memoize = functools.lru_cache
+
+
+def size_str(size_in_bytes):
+ """Returns a human readable size string.
+
+ If size_in_bytes is None, then returns "Unknown size".
+
+ For example `size_str(1.5 * datasets.units.GiB) == "1.50 GiB"`.
+
+ Args:
+ size_in_bytes: `int` or `None`, the size, in bytes, that we want to
+ format as a human-readable size string.
+ """
+ if not size_in_bytes:
+ return "Unknown size"
+
+ _NAME_LIST = [("PiB", 2**50), ("TiB", 2**40), ("GiB", 2**30), ("MiB", 2**20), ("KiB", 2**10)]
+
+ size_in_bytes = float(size_in_bytes)
+ for name, size_bytes in _NAME_LIST:
+ value = size_in_bytes / size_bytes
+ if value >= 1.0:
+ return f"{value:.2f} {name}"
+ return f"{int(size_in_bytes)} bytes"
+
+
+def convert_file_size_to_int(size: Union[int, str]) -> int:
+ """
+ Converts a size expressed as a string with digits an unit (like `"50MB"`) to an integer (in bytes).
+
+ Args:
+ size (`int` or `str`): The size to convert. Will be directly returned if an `int`.
+
+ Example:
+
+ ```py
+ >>> convert_file_size_to_int("1MiB")
+ 1048576
+ ```
+ """
+ if isinstance(size, int):
+ return size
+ if size.upper().endswith("PIB"):
+ return int(size[:-3]) * (2**50)
+ if size.upper().endswith("TIB"):
+ return int(size[:-3]) * (2**40)
+ if size.upper().endswith("GIB"):
+ return int(size[:-3]) * (2**30)
+ if size.upper().endswith("MIB"):
+ return int(size[:-3]) * (2**20)
+ if size.upper().endswith("KIB"):
+ return int(size[:-3]) * (2**10)
+ if size.upper().endswith("PB"):
+ int_size = int(size[:-2]) * (10**15)
+ return int_size // 8 if size.endswith("b") else int_size
+ if size.upper().endswith("TB"):
+ int_size = int(size[:-2]) * (10**12)
+ return int_size // 8 if size.endswith("b") else int_size
+ if size.upper().endswith("GB"):
+ int_size = int(size[:-2]) * (10**9)
+ return int_size // 8 if size.endswith("b") else int_size
+ if size.upper().endswith("MB"):
+ int_size = int(size[:-2]) * (10**6)
+ return int_size // 8 if size.endswith("b") else int_size
+ if size.upper().endswith("KB"):
+ int_size = int(size[:-2]) * (10**3)
+ return int_size // 8 if size.endswith("b") else int_size
+ raise ValueError(f"`size={size}` is not in a valid format. Use an integer followed by the unit, e.g., '5GB'.")
+
+
+def glob_pattern_to_regex(pattern):
+ # partially taken from fsspec:
+ # https://github.com/fsspec/filesystem_spec/blob/697d0f8133d8a5fbc3926e4761d7ecd51337ce50/fsspec/asyn.py#L735
+ return (
+ pattern.replace("\\", r"\\")
+ .replace(".", r"\.")
+ .replace("*", ".*")
+ .replace("+", r"\+")
+ .replace("//", "/")
+ .replace("(", r"\(")
+ .replace(")", r"\)")
+ .replace("|", r"\|")
+ .replace("^", r"\^")
+ .replace("$", r"\$")
+ .rstrip("/")
+ .replace("?", ".")
+ )
+
+
+def string_to_dict(string: str, pattern: str) -> Optional[dict[str, str]]:
+ """Un-format a string using a python f-string pattern.
+ From https://stackoverflow.com/a/36838374
+
+ Example::
+
+ >>> p = 'hello, my name is {name} and I am a {age} year old {what}'
+ >>> s = p.format(name='cody', age=18, what='quarterback')
+ >>> s
+ 'hello, my name is cody and I am a 18 year old quarterback'
+ >>> string_to_dict(s, p)
+ {'age': '18', 'name': 'cody', 'what': 'quarterback'}
+
+ Args:
+ string (str): input string
+ pattern (str): pattern formatted like a python f-string
+ This can be a regex - so in case of un-formatting paths you should use posix paths.
+ Otherwise backslashes for windows paths can cause issues.
+
+ Returns:
+ Optional[dict[str, str]]: dictionary of variable -> value, retrieved from the input using the pattern, or
+ `None` if the string does not match the pattern.
+ """
+ pattern = re.sub(r"{([^:}]+)(?::[^}]+)?}", r"{\1}", pattern) # remove format specifiers, e.g. {rank:05d} -> {rank}
+ regex = re.sub(r"{(.+?)}", r"(?P<_\1>.+)", pattern)
+ result = re.search(regex, string)
+ if result is None:
+ return None
+ values = list(result.groups())
+ keys = re.findall(r"{(.+?)}", pattern)
+ _dict = dict(zip(keys, values))
+ return _dict
+
+
+def asdict(obj):
+ """Convert an object to its dictionary representation recursively.
+
+
+ """
+
+ # Implementation based on https://docs.python.org/3/library/dataclasses.html#dataclasses.asdict
+
+ def _is_dataclass_instance(obj):
+ # https://docs.python.org/3/library/dataclasses.html#dataclasses.is_dataclass
+ return is_dataclass(obj) and not isinstance(obj, type)
+
+ def _asdict_inner(obj):
+ if _is_dataclass_instance(obj):
+ result = {}
+ for f in fields(obj):
+ value = _asdict_inner(getattr(obj, f.name))
+ if not f.init or value != f.default or f.metadata.get("include_in_asdict_even_if_is_default", False):
+ result[f.name] = value
+ return result
+ elif isinstance(obj, tuple) and hasattr(obj, "_fields"):
+ # obj is a namedtuple
+ return type(obj)(*[_asdict_inner(v) for v in obj])
+ elif isinstance(obj, (list, tuple)):
+ # Assume we can create an object of this type by passing in a
+ # generator (which is not true for namedtuples, handled
+ # above).
+ return type(obj)(_asdict_inner(v) for v in obj)
+ elif isinstance(obj, dict):
+ return {_asdict_inner(k): _asdict_inner(v) for k, v in obj.items()}
+ else:
+ return copy.deepcopy(obj)
+
+ if not isinstance(obj, dict) and not _is_dataclass_instance(obj):
+ raise TypeError(f"{obj} is not a dict or a dataclass")
+
+ return _asdict_inner(obj)
+
+
+@contextmanager
+def temporary_assignment(obj, attr, value):
+ """Temporarily assign obj.attr to value."""
+ original = getattr(obj, attr, None)
+ setattr(obj, attr, value)
+ try:
+ yield
+ finally:
+ setattr(obj, attr, original)
+
+
+@contextmanager
+def temp_seed(seed: int, set_pytorch=False, set_tensorflow=False):
+ """Temporarily set the random seed. This works for python numpy, pytorch and tensorflow."""
+ np_state = np.random.get_state()
+ np.random.seed(seed)
+
+ if set_pytorch and config.TORCH_AVAILABLE:
+ import torch
+
+ torch_state = torch.random.get_rng_state()
+ torch.random.manual_seed(seed)
+
+ if torch.cuda.is_available():
+ torch_cuda_states = torch.cuda.get_rng_state_all()
+ torch.cuda.manual_seed_all(seed)
+
+ if set_tensorflow and config.TF_AVAILABLE:
+ import tensorflow as tf
+ from tensorflow.python.eager import context as tfpycontext
+
+ tf_state = tf.random.get_global_generator()
+ temp_gen = tf.random.Generator.from_seed(seed)
+ tf.random.set_global_generator(temp_gen)
+
+ if not tf.executing_eagerly():
+ raise ValueError("Setting random seed for TensorFlow is only available in eager mode")
+
+ tf_context = tfpycontext.context() # eager mode context
+ tf_seed = tf_context._seed
+ tf_rng_initialized = hasattr(tf_context, "_rng")
+ if tf_rng_initialized:
+ tf_rng = tf_context._rng
+ tf_context._set_global_seed(seed)
+
+ try:
+ yield
+ finally:
+ np.random.set_state(np_state)
+
+ if set_pytorch and config.TORCH_AVAILABLE:
+ torch.random.set_rng_state(torch_state)
+ if torch.cuda.is_available():
+ torch.cuda.set_rng_state_all(torch_cuda_states)
+
+ if set_tensorflow and config.TF_AVAILABLE:
+ tf.random.set_global_generator(tf_state)
+
+ tf_context._seed = tf_seed
+ if tf_rng_initialized:
+ tf_context._rng = tf_rng
+ else:
+ delattr(tf_context, "_rng")
+
+
+def unique_values(values):
+ """Iterate over iterable and return only unique values in order."""
+ seen = set()
+ for value in values:
+ if value not in seen:
+ seen.add(value)
+ yield value
+
+
+def no_op_if_value_is_null(func):
+ """If the value is None, return None, else call `func`."""
+
+ def wrapper(value):
+ return func(value) if value is not None else None
+
+ return wrapper
+
+
+def first_non_null_value(iterable):
+ """Return the index and the value of the first non-null value in the iterable. If all values are None, return -1 as index."""
+ for i, value in enumerate(iterable):
+ if value is not None:
+ return i, value
+ return -1, None
+
+
+def first_non_null_non_empty_value(iterable):
+ """Return the index and the value of the first non-null non-empty value in the iterable. If all values are None or empty, return -1 as index."""
+ for i, value in enumerate(iterable):
+ if value is not None and not (isinstance(value, (dict, list)) and len(value) == 0):
+ return i, value
+ return -1, None
+
+
+def zip_dict(*dicts):
+ """Iterate over items of dictionaries grouped by their keys."""
+ for key in unique_values(itertools.chain(*dicts)): # set merge all keys
+ # Will raise KeyError if the dict don't have the same keys
+ yield key, tuple(d[key] for d in dicts)
+
+
+class NonMutableDict(dict):
+ """Dict where keys can only be added but not modified.
+
+ Will raise an error if the user try to overwrite one key. The error message
+ can be customized during construction. It will be formatted using {key} for
+ the overwritten key.
+ """
+
+ def __init__(self, *args, **kwargs):
+ self._error_msg = kwargs.pop(
+ "error_msg",
+ "Try to overwrite existing key: {key}",
+ )
+ if kwargs:
+ raise ValueError("NonMutableDict cannot be initialized with kwargs.")
+ super().__init__(*args, **kwargs)
+
+ def __setitem__(self, key, value):
+ if key in self:
+ raise ValueError(self._error_msg.format(key=key))
+ return super().__setitem__(key, value)
+
+ def update(self, other):
+ if any(k in self for k in other):
+ raise ValueError(self._error_msg.format(key=set(self) & set(other)))
+ return super().update(other)
+
+
+class classproperty(property): # pylint: disable=invalid-name
+ """Descriptor to be used as decorator for @classmethods."""
+
+ def __get__(self, obj, objtype=None):
+ return self.fget.__get__(None, objtype)()
+
+
+def _single_map_nested(args):
+ """Apply a function recursively to each element of a nested data struct."""
+ function, data_struct, batched, batch_size, types, rank, disable_tqdm, desc = args
+
+ # Singleton first to spare some computation
+ if not isinstance(data_struct, dict) and not isinstance(data_struct, types):
+ if batched:
+ return function([data_struct])[0]
+ else:
+ return function(data_struct)
+ if (
+ batched
+ and not isinstance(data_struct, dict)
+ and isinstance(data_struct, types)
+ and all(not isinstance(v, (dict, types)) for v in data_struct)
+ ):
+ return [mapped_item for batch in iter_batched(data_struct, batch_size) for mapped_item in function(batch)]
+
+ # Reduce logging to keep things readable in multiprocessing with tqdm
+ if rank is not None and logging.get_verbosity() < logging.WARNING:
+ logging.set_verbosity_warning()
+ # Print at least one thing to fix tqdm in notebooks in multiprocessing
+ # see https://github.com/tqdm/tqdm/issues/485#issuecomment-473338308
+ if rank is not None and not disable_tqdm and any("notebook" in tqdm_cls.__name__ for tqdm_cls in tqdm.__mro__):
+ print(" ", end="", flush=True)
+
+ # Loop over single examples or batches and write to buffer/file if examples are to be updated
+ pbar_iterable = data_struct.items() if isinstance(data_struct, dict) else data_struct
+ pbar_desc = (desc + " " if desc is not None else "") + "#" + str(rank) if rank is not None else desc
+ with hf_tqdm(pbar_iterable, disable=disable_tqdm, position=rank, unit="obj", desc=pbar_desc) as pbar:
+ if isinstance(data_struct, dict):
+ return {
+ k: _single_map_nested((function, v, batched, batch_size, types, None, True, None)) for k, v in pbar
+ }
+ else:
+ mapped = [_single_map_nested((function, v, batched, batch_size, types, None, True, None)) for v in pbar]
+ if isinstance(data_struct, list):
+ return mapped
+ elif isinstance(data_struct, tuple):
+ return tuple(mapped)
+ else:
+ return np.array(mapped)
+
+
+def map_nested(
+ function: Callable[[Any], Any],
+ data_struct: Any,
+ dict_only: bool = False,
+ map_list: bool = True,
+ map_tuple: bool = False,
+ map_numpy: bool = False,
+ num_proc: Optional[int] = None,
+ parallel_min_length: int = 2,
+ batched: bool = False,
+ batch_size: Optional[int] = 1000,
+ types: Optional[tuple] = None,
+ disable_tqdm: bool = True,
+ desc: Optional[str] = None,
+) -> Any:
+ """Apply a function recursively to each element of a nested data struct.
+
+ Use multiprocessing if num_proc > 1 and the length of data_struct is greater than or equal to
+ `parallel_min_length`.
+
+
+
+ Before version 2.5.0, multiprocessing was not used if `num_proc` was greater than or equal to ``len(iterable)``.
+
+ Now, if `num_proc` is greater than or equal to ``len(iterable)``, `num_proc` is set to ``len(iterable)`` and
+ multiprocessing is used.
+
+
+
+ Args:
+ function (`Callable`): Function to be applied to `data_struct`.
+ data_struct (`Any`): Data structure to apply `function` to.
+ dict_only (`bool`, default `False`): Whether only apply `function` recursively to `dict` values in
+ `data_struct`.
+ map_list (`bool`, default `True`): Whether also apply `function` recursively to `list` elements (besides `dict`
+ values).
+ map_tuple (`bool`, default `False`): Whether also apply `function` recursively to `tuple` elements (besides
+ `dict` values).
+ map_numpy (`bool, default `False`): Whether also apply `function` recursively to `numpy.array` elements (besides
+ `dict` values).
+ num_proc (`int`, *optional*): Number of processes.
+ The level in the data struct used for multiprocessing is the first level that has smaller sub-structs,
+ starting from the root.
+ parallel_min_length (`int`, default `2`): Minimum length of `data_struct` required for parallel
+ processing.
+
+ batched (`bool`, defaults to `False`):
+ Provide batch of items to `function`.
+
+ batch_size (`int`, *optional*, defaults to `1000`):
+ Number of items per batch provided to `function` if `batched=True`.
+ If `batch_size <= 0` or `batch_size == None`, provide the full iterable as a single batch to `function`.
+
+ types (`tuple`, *optional*): Additional types (besides `dict` values) to apply `function` recursively to their
+ elements.
+ disable_tqdm (`bool`, default `True`): Whether to disable the tqdm progressbar.
+ desc (`str`, *optional*): Prefix for the tqdm progressbar.
+
+ Returns:
+ `Any`
+ """
+ if types is None:
+ types = []
+ if not dict_only:
+ if map_list:
+ types.append(list)
+ if map_tuple:
+ types.append(tuple)
+ if map_numpy:
+ types.append(np.ndarray)
+ types = tuple(types)
+
+ # Singleton
+ if not isinstance(data_struct, dict) and not isinstance(data_struct, types):
+ if batched:
+ data_struct = [data_struct]
+ mapped = function(data_struct)
+ if batched:
+ mapped = mapped[0]
+ return mapped
+
+ iterable = list(data_struct.values()) if isinstance(data_struct, dict) else data_struct
+
+ if num_proc is None:
+ num_proc = 1
+ if any(isinstance(v, types) and len(v) > len(iterable) for v in iterable):
+ mapped = [
+ map_nested(
+ function=function,
+ data_struct=obj,
+ num_proc=num_proc,
+ parallel_min_length=parallel_min_length,
+ batched=batched,
+ batch_size=batch_size,
+ types=types,
+ )
+ for obj in iterable
+ ]
+ elif num_proc != -1 and num_proc <= 1 or len(iterable) < parallel_min_length:
+ if batched:
+ if batch_size is None or batch_size <= 0:
+ batch_size = max(len(iterable) // num_proc + int(len(iterable) % num_proc > 0), 1)
+ iterable = list(iter_batched(iterable, batch_size))
+ mapped = [
+ _single_map_nested((function, obj, batched, batch_size, types, None, True, None))
+ for obj in hf_tqdm(iterable, disable=disable_tqdm, desc=desc)
+ ]
+ if batched:
+ mapped = [mapped_item for mapped_batch in mapped for mapped_item in mapped_batch]
+ else:
+ with warnings.catch_warnings():
+ warnings.filterwarnings(
+ "ignore",
+ message=".* is experimental and might be subject to breaking changes in the future\\.$",
+ category=UserWarning,
+ )
+ if batched:
+ if batch_size is None or batch_size <= 0:
+ batch_size = len(iterable) // num_proc + int(len(iterable) % num_proc > 0)
+ iterable = list(iter_batched(iterable, batch_size))
+ mapped = parallel_map(
+ function, iterable, num_proc, batched, batch_size, types, disable_tqdm, desc, _single_map_nested
+ )
+ if batched:
+ mapped = [mapped_item for mapped_batch in mapped for mapped_item in mapped_batch]
+
+ if isinstance(data_struct, dict):
+ return dict(zip(data_struct.keys(), mapped))
+ else:
+ if isinstance(data_struct, list):
+ return mapped
+ elif isinstance(data_struct, tuple):
+ return tuple(mapped)
+ else:
+ return np.array(mapped)
+
+
+class NestedDataStructure:
+ def __init__(self, data=None):
+ self.data = data if data is not None else []
+
+ def flatten(self, data=None):
+ data = data if data is not None else self.data
+ if isinstance(data, dict):
+ return self.flatten(list(data.values()))
+ elif isinstance(data, (list, tuple)):
+ return [flattened for item in data for flattened in self.flatten(item)]
+ else:
+ return [data]
+
+
+def has_sufficient_disk_space(needed_bytes, directory="."):
+ try:
+ free_bytes = disk_usage(os.path.abspath(directory)).free
+ except OSError:
+ return True
+ return needed_bytes < free_bytes
+
+
+def copyfunc(func):
+ result = types.FunctionType(func.__code__, func.__globals__, func.__name__, func.__defaults__, func.__closure__)
+ result.__kwdefaults__ = func.__kwdefaults__
+ return result
+
+
+Y = TypeVar("Y")
+
+
+def _write_generator_to_queue(queue: queue.Queue, func: Callable[..., Iterable[Y]], kwargs: dict) -> int:
+ for i, result in enumerate(func(**kwargs)):
+ queue.put(result)
+ return i
+
+
+def _get_pool_pid(pool: Union[multiprocessing.pool.Pool, multiprocess.pool.Pool]) -> set[int]:
+ return {f.pid for f in pool._pool}
+
+
+def iflatmap_unordered(
+ pool: Union[multiprocessing.pool.Pool, multiprocess.pool.Pool],
+ func: Callable[..., Iterable[Y]],
+ *,
+ kwargs_iterable: Iterable[dict],
+) -> Iterable[Y]:
+ initial_pool_pid = _get_pool_pid(pool)
+ pool_changed = False
+ manager_cls = Manager if isinstance(pool, multiprocessing.pool.Pool) else multiprocess.Manager
+ with manager_cls() as manager:
+ queue = manager.Queue()
+ async_results = [
+ pool.apply_async(_write_generator_to_queue, (queue, func, kwargs)) for kwargs in kwargs_iterable
+ ]
+ try:
+ while True:
+ try:
+ yield queue.get(timeout=0.05)
+ except Empty:
+ if all(async_result.ready() for async_result in async_results) and queue.empty():
+ break
+ if _get_pool_pid(pool) != initial_pool_pid:
+ pool_changed = True
+ # One of the subprocesses has died. We should not wait forever.
+ raise RuntimeError(
+ "One of the subprocesses has abruptly died during map operation."
+ "To debug the error, disable multiprocessing."
+ )
+ finally:
+ if not pool_changed:
+ # we get the result in case there's an error to raise
+ [async_result.get(timeout=0.05) for async_result in async_results]
+
+
+T = TypeVar("T")
+
+
+def iter_batched(iterable: Iterable[T], n: int) -> Iterable[list[T]]:
+ if n < 1:
+ raise ValueError(f"Invalid batch size {n}")
+ batch = []
+ for item in iterable:
+ batch.append(item)
+ if len(batch) == n:
+ yield batch
+ batch = []
+ if batch:
+ yield batch
diff --git a/venv/lib/python3.10/site-packages/datasets/utils/resources/__init__.py b/venv/lib/python3.10/site-packages/datasets/utils/resources/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/venv/lib/python3.10/site-packages/datasets/utils/resources/__pycache__/__init__.cpython-310.pyc b/venv/lib/python3.10/site-packages/datasets/utils/resources/__pycache__/__init__.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..8b5780f92fc4ccaa2596b458bc5e253c7a50464b
Binary files /dev/null and b/venv/lib/python3.10/site-packages/datasets/utils/resources/__pycache__/__init__.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/datasets/utils/resources/creators.json b/venv/lib/python3.10/site-packages/datasets/utils/resources/creators.json
new file mode 100644
index 0000000000000000000000000000000000000000..d9e15f0039cc27ed8abd9fdf394423a3fada2c95
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/utils/resources/creators.json
@@ -0,0 +1,17 @@
+{
+ "language": [
+ "found",
+ "crowdsourced",
+ "expert-generated",
+ "machine-generated",
+ "other"
+ ],
+ "annotations": [
+ "found",
+ "crowdsourced",
+ "expert-generated",
+ "machine-generated",
+ "no-annotation",
+ "other"
+ ]
+}
diff --git a/venv/lib/python3.10/site-packages/datasets/utils/resources/languages.json b/venv/lib/python3.10/site-packages/datasets/utils/resources/languages.json
new file mode 100644
index 0000000000000000000000000000000000000000..ea7686f956b898af3faf97b86be89b71d88855d4
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/utils/resources/languages.json
@@ -0,0 +1,8026 @@
+{
+ "code": "Programming language (C++, Java, Javascript, Python, etc.)",
+ "aa": "Afar",
+ "aaa": "Ghotuo",
+ "aab": "Alumu-Tesu",
+ "aac": "Ari",
+ "aad": "Amal",
+ "aae": "Arbëreshë Albanian",
+ "aaf": "Aranadan",
+ "aag": "Ambrak",
+ "aah": "Abu' Arapesh",
+ "aai": "Arifama-Miniafia",
+ "aak": "Ankave",
+ "aal": "Afade",
+ "aan": "Anambé",
+ "aao": "Algerian Saharan Arabic",
+ "aap": "Pará Arára",
+ "aaq": "Eastern Abnaki",
+ "aas": "Aasáx",
+ "aat": "Arvanitika Albanian",
+ "aau": "Abau",
+ "aav": "Austro-Asiatic languages",
+ "aaw": "Solong",
+ "aax": "Mandobo Atas",
+ "aaz": "Amarasi",
+ "ab": "Abkhazian",
+ "aba": "Abé",
+ "abb": "Bankon",
+ "abc": "Ambala Ayta",
+ "abd": "Manide",
+ "abe": "Western Abnaki",
+ "abf": "Abai Sungai",
+ "abg": "Abaga",
+ "abh": "Tajiki Arabic",
+ "abi": "Abidji",
+ "abj": "Aka-Bea",
+ "abl": "Lampung Nyo",
+ "abm": "Abanyom",
+ "abn": "Abua",
+ "abo": "Abon",
+ "abp": "Abellen Ayta",
+ "abq": "Abaza",
+ "abr": "Abron",
+ "abs": "Ambonese Malay",
+ "abt": "Ambulas",
+ "abu": "Abure",
+ "abv": "Baharna Arabic",
+ "abw": "Pal",
+ "abx": "Inabaknon",
+ "aby": "Aneme Wake",
+ "abz": "Abui",
+ "aca": "Achagua",
+ "acb": "Áncá",
+ "acd": "Gikyode",
+ "ace": "Achinese",
+ "acf": "Saint Lucian Creole French",
+ "ach": "Acoli",
+ "aci": "Aka-Cari",
+ "ack": "Aka-Kora",
+ "acl": "Akar-Bale",
+ "acm": "Mesopotamian Arabic",
+ "acn": "Achang",
+ "acp": "Eastern Acipa",
+ "acq": "Ta'izzi-Adeni Arabic",
+ "acr": "Achi",
+ "acs": "Acroá",
+ "act": "Achterhoeks",
+ "acu": "Achuar-Shiwiar",
+ "acv": "Achumawi",
+ "acw": "Hijazi Arabic",
+ "acx": "Omani Arabic",
+ "acy": "Cypriot Arabic",
+ "acz": "Acheron",
+ "ada": "Adangme",
+ "adb": "Atauran",
+ "add": "Lidzonka; Dzodinka",
+ "ade": "Adele",
+ "adf": "Dhofari Arabic",
+ "adg": "Andegerebinha",
+ "adh": "Adhola",
+ "adi": "Adi",
+ "adj": "Adioukrou",
+ "adl": "Galo",
+ "adn": "Adang",
+ "ado": "Abu",
+ "adq": "Adangbe",
+ "adr": "Adonara",
+ "ads": "Adamorobe Sign Language",
+ "adt": "Adnyamathanha",
+ "adu": "Aduge",
+ "adw": "Amundava",
+ "adx": "Amdo Tibetan",
+ "ady": "Adyghe; Adygei",
+ "adz": "Adzera",
+ "ae": "Avestan",
+ "aea": "Areba",
+ "aeb": "Tunisian Arabic",
+ "aec": "Saidi Arabic",
+ "aed": "Argentine Sign Language",
+ "aee": "Northeast Pashai; Northeast Pashayi",
+ "aek": "Haeke",
+ "ael": "Ambele",
+ "aem": "Arem",
+ "aen": "Armenian Sign Language",
+ "aeq": "Aer",
+ "aer": "Eastern Arrernte",
+ "aes": "Alsea",
+ "aeu": "Akeu",
+ "aew": "Ambakich",
+ "aey": "Amele",
+ "aez": "Aeka",
+ "af": "Afrikaans",
+ "afa": "Afro-Asiatic languages",
+ "afb": "Gulf Arabic",
+ "afd": "Andai",
+ "afe": "Putukwam",
+ "afg": "Afghan Sign Language",
+ "afh": "Afrihili",
+ "afi": "Akrukay; Chini",
+ "afk": "Nanubae",
+ "afn": "Defaka",
+ "afo": "Eloyi",
+ "afp": "Tapei",
+ "afs": "Afro-Seminole Creole",
+ "aft": "Afitti",
+ "afu": "Awutu",
+ "afz": "Obokuitai",
+ "aga": "Aguano",
+ "agb": "Legbo",
+ "agc": "Agatu",
+ "agd": "Agarabi",
+ "age": "Angal",
+ "agf": "Arguni",
+ "agg": "Angor",
+ "agh": "Ngelima",
+ "agi": "Agariya",
+ "agj": "Argobba",
+ "agk": "Isarog Agta",
+ "agl": "Fembe",
+ "agm": "Angaataha",
+ "agn": "Agutaynen",
+ "ago": "Tainae",
+ "agq": "Aghem",
+ "agr": "Aguaruna",
+ "ags": "Esimbi",
+ "agt": "Central Cagayan Agta",
+ "agu": "Aguacateco",
+ "agv": "Remontado Dumagat",
+ "agw": "Kahua",
+ "agx": "Aghul",
+ "agy": "Southern Alta",
+ "agz": "Mt. Iriga Agta",
+ "aha": "Ahanta",
+ "ahb": "Axamb",
+ "ahg": "Qimant",
+ "ahh": "Aghu",
+ "ahi": "Tiagbamrin Aizi",
+ "ahk": "Akha",
+ "ahl": "Igo",
+ "ahm": "Mobumrin Aizi",
+ "ahn": "Àhàn",
+ "aho": "Ahom",
+ "ahp": "Aproumu Aizi",
+ "ahr": "Ahirani",
+ "ahs": "Ashe",
+ "aht": "Ahtena",
+ "aia": "Arosi",
+ "aib": "Ainu (China)",
+ "aic": "Ainbai",
+ "aid": "Alngith",
+ "aie": "Amara",
+ "aif": "Agi",
+ "aig": "Antigua and Barbuda Creole English",
+ "aih": "Ai-Cham",
+ "aii": "Assyrian Neo-Aramaic",
+ "aij": "Lishanid Noshan",
+ "aik": "Ake",
+ "ail": "Aimele",
+ "aim": "Aimol",
+ "ain": "Ainu (Japan)",
+ "aio": "Aiton",
+ "aip": "Burumakok",
+ "aiq": "Aimaq",
+ "air": "Airoran",
+ "ait": "Arikem",
+ "aiw": "Aari",
+ "aix": "Aighon",
+ "aiy": "Ali",
+ "aja": "Aja (South Sudan)",
+ "ajg": "Aja (Benin)",
+ "aji": "Ajië",
+ "ajn": "Andajin",
+ "ajp": "South Levantine Arabic",
+ "ajs": "Algerian Jewish Sign Language",
+ "aju": "Judeo-Moroccan Arabic",
+ "ajw": "Ajawa",
+ "ajz": "Amri Karbi",
+ "ak": "Akan",
+ "akb": "Batak Angkola",
+ "akc": "Mpur",
+ "akd": "Ukpet-Ehom",
+ "ake": "Akawaio",
+ "akf": "Akpa",
+ "akg": "Anakalangu",
+ "akh": "Angal Heneng",
+ "aki": "Aiome",
+ "akj": "Aka-Jeru",
+ "akk": "Akkadian",
+ "akl": "Aklanon",
+ "akm": "Aka-Bo",
+ "ako": "Akurio",
+ "akp": "Siwu",
+ "akq": "Ak",
+ "akr": "Araki",
+ "aks": "Akaselem",
+ "akt": "Akolet",
+ "aku": "Akum",
+ "akv": "Akhvakh",
+ "akw": "Akwa",
+ "akx": "Aka-Kede",
+ "aky": "Aka-Kol",
+ "akz": "Alabama",
+ "ala": "Alago",
+ "alc": "Qawasqar",
+ "ald": "Alladian",
+ "ale": "Aleut",
+ "alf": "Alege",
+ "alg": "Algonquian languages",
+ "alh": "Alawa",
+ "ali": "Amaimon",
+ "alj": "Alangan",
+ "alk": "Alak",
+ "all": "Allar",
+ "alm": "Amblong",
+ "aln": "Gheg Albanian",
+ "alo": "Larike-Wakasihu",
+ "alp": "Alune",
+ "alq": "Algonquin",
+ "alr": "Alutor",
+ "als": "Tosk Albanian",
+ "alt": "Southern Altai",
+ "alu": "'Are'are",
+ "alv": "Atlantic-Congo languages",
+ "alw": "Alaba-K’abeena; Wanbasana",
+ "alx": "Amol",
+ "aly": "Alyawarr",
+ "alz": "Alur",
+ "am": "Amharic",
+ "ama": "Amanayé",
+ "amb": "Ambo",
+ "amc": "Amahuaca",
+ "ame": "Yanesha'",
+ "amf": "Hamer-Banna",
+ "amg": "Amurdak",
+ "ami": "Amis",
+ "amj": "Amdang",
+ "amk": "Ambai",
+ "aml": "War-Jaintia",
+ "amm": "Ama (Papua New Guinea)",
+ "amn": "Amanab",
+ "amo": "Amo",
+ "amp": "Alamblak",
+ "amq": "Amahai",
+ "amr": "Amarakaeri",
+ "ams": "Southern Amami-Oshima",
+ "amt": "Amto",
+ "amu": "Guerrero Amuzgo",
+ "amv": "Ambelau",
+ "amw": "Western Neo-Aramaic",
+ "amx": "Anmatyerre",
+ "amy": "Ami",
+ "amz": "Atampaya",
+ "an": "Aragonese",
+ "ana": "Andaqui",
+ "anb": "Andoa",
+ "anc": "Ngas",
+ "and": "Ansus",
+ "ane": "Xârâcùù",
+ "anf": "Animere",
+ "ang": "Old English (ca. 450-1100)",
+ "anh": "Nend",
+ "ani": "Andi",
+ "anj": "Anor",
+ "ank": "Goemai",
+ "anl": "Anu-Hkongso Chin",
+ "anm": "Anal",
+ "ann": "Obolo",
+ "ano": "Andoque",
+ "anp": "Angika",
+ "anq": "Jarawa (India)",
+ "anr": "Andh",
+ "ans": "Anserma",
+ "ant": "Antakarinya; Antikarinya",
+ "anu": "Anuak",
+ "anv": "Denya",
+ "anw": "Anaang",
+ "anx": "Andra-Hus",
+ "any": "Anyin",
+ "anz": "Anem",
+ "aoa": "Angolar",
+ "aob": "Abom",
+ "aoc": "Pemon",
+ "aod": "Andarum",
+ "aoe": "Angal Enen",
+ "aof": "Bragat",
+ "aog": "Angoram",
+ "aoi": "Anindilyakwa",
+ "aoj": "Mufian",
+ "aok": "Arhö",
+ "aol": "Alor",
+ "aom": "Ömie",
+ "aon": "Bumbita Arapesh",
+ "aor": "Aore",
+ "aos": "Taikat",
+ "aot": "Atong (India); A'tong",
+ "aou": "A'ou",
+ "aox": "Atorada",
+ "aoz": "Uab Meto",
+ "apa": "Apache languages",
+ "apb": "Sa'a",
+ "apc": "North Levantine Arabic",
+ "apd": "Sudanese Arabic",
+ "ape": "Bukiyip",
+ "apf": "Pahanan Agta",
+ "apg": "Ampanang",
+ "aph": "Athpariya",
+ "api": "Apiaká",
+ "apj": "Jicarilla Apache",
+ "apk": "Kiowa Apache",
+ "apl": "Lipan Apache",
+ "apm": "Mescalero-Chiricahua Apache",
+ "apn": "Apinayé",
+ "apo": "Ambul",
+ "app": "Apma",
+ "apq": "A-Pucikwar",
+ "apr": "Arop-Lokep",
+ "aps": "Arop-Sissano",
+ "apt": "Apatani",
+ "apu": "Apurinã",
+ "apv": "Alapmunte",
+ "apw": "Western Apache",
+ "apx": "Aputai",
+ "apy": "Apalaí",
+ "apz": "Safeyoka",
+ "aqa": "Alacalufan languages",
+ "aqc": "Archi",
+ "aqd": "Ampari Dogon",
+ "aqg": "Arigidi",
+ "aqk": "Aninka",
+ "aql": "Algic languages",
+ "aqm": "Atohwaim",
+ "aqn": "Northern Alta",
+ "aqp": "Atakapa",
+ "aqr": "Arhâ",
+ "aqt": "Angaité",
+ "aqz": "Akuntsu",
+ "ar": "Arabic",
+ "arb": "Standard Arabic",
+ "arc": "Official Aramaic (700-300 BCE); Imperial Aramaic (700-300 BCE)",
+ "ard": "Arabana",
+ "are": "Western Arrarnta",
+ "arh": "Arhuaco",
+ "ari": "Arikara",
+ "arj": "Arapaso",
+ "ark": "Arikapú",
+ "arl": "Arabela",
+ "arn": "Mapudungun; Mapuche",
+ "aro": "Araona",
+ "arp": "Arapaho",
+ "arq": "Algerian Arabic",
+ "arr": "Karo (Brazil)",
+ "ars": "Najdi Arabic",
+ "art": "Artificial languages",
+ "aru": "Aruá (Amazonas State); Arawá",
+ "arv": "Arbore",
+ "arw": "Arawak",
+ "arx": "Aruá (Rodonia State)",
+ "ary": "Moroccan Arabic",
+ "arz": "Egyptian Arabic",
+ "as": "Assamese",
+ "asa": "Asu (Tanzania)",
+ "asb": "Assiniboine",
+ "asc": "Casuarina Coast Asmat",
+ "ase": "American Sign Language",
+ "asf": "Auslan; Australian Sign Language",
+ "asg": "Cishingini",
+ "ash": "Abishira",
+ "asi": "Buruwai",
+ "asj": "Sari",
+ "ask": "Ashkun",
+ "asl": "Asilulu",
+ "asn": "Xingú Asuriní",
+ "aso": "Dano",
+ "asp": "Algerian Sign Language",
+ "asq": "Austrian Sign Language",
+ "asr": "Asuri",
+ "ass": "Ipulo",
+ "ast": "Asturian; Asturleonese; Bable; Leonese",
+ "asu": "Tocantins Asurini",
+ "asv": "Asoa",
+ "asw": "Australian Aborigines Sign Language",
+ "asx": "Muratayak",
+ "asy": "Yaosakor Asmat",
+ "asz": "As",
+ "ata": "Pele-Ata",
+ "atb": "Zaiwa",
+ "atc": "Atsahuaca",
+ "atd": "Ata Manobo",
+ "ate": "Atemble",
+ "atg": "Ivbie North-Okpela-Arhe",
+ "ath": "Athapascan languages",
+ "ati": "Attié",
+ "atj": "Atikamekw",
+ "atk": "Ati",
+ "atl": "Mt. Iraya Agta",
+ "atm": "Ata",
+ "atn": "Ashtiani",
+ "ato": "Atong (Cameroon)",
+ "atp": "Pudtol Atta",
+ "atq": "Aralle-Tabulahan",
+ "atr": "Waimiri-Atroari",
+ "ats": "Gros Ventre",
+ "att": "Pamplona Atta",
+ "atu": "Reel",
+ "atv": "Northern Altai",
+ "atw": "Atsugewi",
+ "atx": "Arutani",
+ "aty": "Aneityum",
+ "atz": "Arta",
+ "aua": "Asumboa",
+ "aub": "Alugu",
+ "auc": "Waorani",
+ "aud": "Anuta",
+ "auf": "Arauan languages",
+ "aug": "Aguna",
+ "auh": "Aushi",
+ "aui": "Anuki",
+ "auj": "Awjilah",
+ "auk": "Heyo",
+ "aul": "Aulua",
+ "aum": "Asu (Nigeria)",
+ "aun": "Molmo One",
+ "auo": "Auyokawa",
+ "aup": "Makayam",
+ "auq": "Anus; Korur",
+ "aur": "Aruek",
+ "aus": "Australian languages",
+ "aut": "Austral",
+ "auu": "Auye",
+ "auw": "Awyi",
+ "aux": "Aurá",
+ "auy": "Awiyaana",
+ "auz": "Uzbeki Arabic",
+ "av": "Avaric",
+ "avb": "Avau",
+ "avd": "Alviri-Vidari",
+ "avi": "Avikam",
+ "avk": "Kotava",
+ "avl": "Eastern Egyptian Bedawi Arabic",
+ "avm": "Angkamuthi",
+ "avn": "Avatime",
+ "avo": "Agavotaguerra",
+ "avs": "Aushiri",
+ "avt": "Au",
+ "avu": "Avokaya",
+ "avv": "Avá-Canoeiro",
+ "awa": "Awadhi",
+ "awb": "Awa (Papua New Guinea)",
+ "awc": "Cicipu",
+ "awd": "Arawakan languages",
+ "awe": "Awetí",
+ "awg": "Anguthimri",
+ "awh": "Awbono",
+ "awi": "Aekyom",
+ "awk": "Awabakal",
+ "awm": "Arawum",
+ "awn": "Awngi",
+ "awo": "Awak",
+ "awr": "Awera",
+ "aws": "South Awyu",
+ "awt": "Araweté",
+ "awu": "Central Awyu",
+ "awv": "Jair Awyu",
+ "aww": "Awun",
+ "awx": "Awara",
+ "awy": "Edera Awyu",
+ "axb": "Abipon",
+ "axe": "Ayerrerenge",
+ "axg": "Mato Grosso Arára",
+ "axk": "Yaka (Central African Republic)",
+ "axl": "Lower Southern Aranda",
+ "axm": "Middle Armenian",
+ "axx": "Xârâgurè",
+ "ay": "Aymara",
+ "aya": "Awar",
+ "ayb": "Ayizo Gbe",
+ "ayc": "Southern Aymara",
+ "ayd": "Ayabadhu",
+ "aye": "Ayere",
+ "ayg": "Ginyanga",
+ "ayh": "Hadrami Arabic",
+ "ayi": "Leyigha",
+ "ayk": "Akuku",
+ "ayl": "Libyan Arabic",
+ "ayn": "Sanaani Arabic",
+ "ayo": "Ayoreo",
+ "ayp": "North Mesopotamian Arabic",
+ "ayq": "Ayi (Papua New Guinea)",
+ "ayr": "Central Aymara",
+ "ays": "Sorsogon Ayta",
+ "ayt": "Magbukun Ayta",
+ "ayu": "Ayu",
+ "ayz": "Mai Brat",
+ "az": "Azerbaijani",
+ "aza": "Azha",
+ "azb": "South Azerbaijani",
+ "azc": "Uto-Aztecan languages",
+ "azd": "Eastern Durango Nahuatl",
+ "azg": "San Pedro Amuzgos Amuzgo",
+ "azj": "North Azerbaijani",
+ "azm": "Ipalapa Amuzgo",
+ "azn": "Western Durango Nahuatl",
+ "azo": "Awing",
+ "azt": "Faire Atta",
+ "azz": "Highland Puebla Nahuatl",
+ "ba": "Bashkir",
+ "baa": "Babatana",
+ "bab": "Bainouk-Gunyuño",
+ "bac": "Badui",
+ "bad": "Banda languages",
+ "bae": "Baré",
+ "baf": "Nubaca",
+ "bag": "Tuki",
+ "bah": "Bahamas Creole English",
+ "bai": "Bamileke languages",
+ "baj": "Barakai",
+ "bal": "Baluchi",
+ "ban": "Balinese",
+ "bao": "Waimaha",
+ "bap": "Bantawa",
+ "bar": "Bavarian",
+ "bas": "Basa (Cameroon)",
+ "bat": "Baltic languages",
+ "bau": "Bada (Nigeria)",
+ "bav": "Vengo",
+ "baw": "Bambili-Bambui",
+ "bax": "Bamun",
+ "bay": "Batuley",
+ "bba": "Baatonum",
+ "bbb": "Barai",
+ "bbc": "Batak Toba",
+ "bbd": "Bau",
+ "bbe": "Bangba",
+ "bbf": "Baibai",
+ "bbg": "Barama",
+ "bbh": "Bugan",
+ "bbi": "Barombi",
+ "bbj": "Ghomálá'",
+ "bbk": "Babanki",
+ "bbl": "Bats",
+ "bbm": "Babango",
+ "bbn": "Uneapa",
+ "bbo": "Northern Bobo Madaré; Konabéré",
+ "bbp": "West Central Banda",
+ "bbq": "Bamali",
+ "bbr": "Girawa",
+ "bbs": "Bakpinka",
+ "bbt": "Mburku",
+ "bbu": "Kulung (Nigeria)",
+ "bbv": "Karnai",
+ "bbw": "Baba",
+ "bbx": "Bubia",
+ "bby": "Befang",
+ "bca": "Central Bai",
+ "bcb": "Bainouk-Samik",
+ "bcc": "Southern Balochi",
+ "bcd": "North Babar",
+ "bce": "Bamenyam",
+ "bcf": "Bamu",
+ "bcg": "Baga Pokur",
+ "bch": "Bariai",
+ "bci": "Baoulé",
+ "bcj": "Bardi",
+ "bck": "Bunuba",
+ "bcl": "Central Bikol",
+ "bcm": "Bannoni",
+ "bcn": "Bali (Nigeria)",
+ "bco": "Kaluli",
+ "bcp": "Bali (Democratic Republic of Congo)",
+ "bcq": "Bench",
+ "bcr": "Babine",
+ "bcs": "Kohumono",
+ "bct": "Bendi",
+ "bcu": "Awad Bing",
+ "bcv": "Shoo-Minda-Nye",
+ "bcw": "Bana",
+ "bcy": "Bacama",
+ "bcz": "Bainouk-Gunyaamolo",
+ "bda": "Bayot",
+ "bdb": "Basap",
+ "bdc": "Emberá-Baudó",
+ "bdd": "Bunama",
+ "bde": "Bade",
+ "bdf": "Biage",
+ "bdg": "Bonggi",
+ "bdh": "Baka (South Sudan)",
+ "bdi": "Burun",
+ "bdj": "Bai (South Sudan); Bai",
+ "bdk": "Budukh",
+ "bdl": "Indonesian Bajau",
+ "bdm": "Buduma",
+ "bdn": "Baldemu",
+ "bdo": "Morom",
+ "bdp": "Bende",
+ "bdq": "Bahnar",
+ "bdr": "West Coast Bajau",
+ "bds": "Burunge",
+ "bdt": "Bokoto",
+ "bdu": "Oroko",
+ "bdv": "Bodo Parja",
+ "bdw": "Baham",
+ "bdx": "Budong-Budong",
+ "bdy": "Bandjalang",
+ "bdz": "Badeshi",
+ "be": "Belarusian",
+ "bea": "Beaver",
+ "beb": "Bebele",
+ "bec": "Iceve-Maci",
+ "bed": "Bedoanas",
+ "bee": "Byangsi",
+ "bef": "Benabena",
+ "beg": "Belait",
+ "beh": "Biali",
+ "bei": "Bekati'",
+ "bej": "Beja; Bedawiyet",
+ "bek": "Bebeli",
+ "bem": "Bemba (Zambia)",
+ "beo": "Beami",
+ "bep": "Besoa",
+ "beq": "Beembe",
+ "ber": "Berber languages",
+ "bes": "Besme",
+ "bet": "Guiberoua Béte",
+ "beu": "Blagar",
+ "bev": "Daloa Bété",
+ "bew": "Betawi",
+ "bex": "Jur Modo",
+ "bey": "Beli (Papua New Guinea)",
+ "bez": "Bena (Tanzania)",
+ "bfa": "Bari",
+ "bfb": "Pauri Bareli",
+ "bfc": "Panyi Bai; Northern Bai",
+ "bfd": "Bafut",
+ "bfe": "Betaf; Tena",
+ "bff": "Bofi",
+ "bfg": "Busang Kayan",
+ "bfh": "Blafe",
+ "bfi": "British Sign Language",
+ "bfj": "Bafanji",
+ "bfk": "Ban Khor Sign Language",
+ "bfl": "Banda-Ndélé",
+ "bfm": "Mmen",
+ "bfn": "Bunak",
+ "bfo": "Malba Birifor",
+ "bfp": "Beba",
+ "bfq": "Badaga",
+ "bfr": "Bazigar",
+ "bfs": "Southern Bai",
+ "bft": "Balti",
+ "bfu": "Gahri",
+ "bfw": "Bondo",
+ "bfx": "Bantayanon",
+ "bfy": "Bagheli",
+ "bfz": "Mahasu Pahari",
+ "bg": "Bulgarian",
+ "bga": "Gwamhi-Wuri",
+ "bgb": "Bobongko",
+ "bgc": "Haryanvi",
+ "bgd": "Rathwi Bareli",
+ "bge": "Bauria",
+ "bgf": "Bangandu",
+ "bgg": "Bugun",
+ "bgi": "Giangan",
+ "bgj": "Bangolan",
+ "bgk": "Bit; Buxinhua",
+ "bgl": "Bo (Laos)",
+ "bgn": "Western Balochi",
+ "bgo": "Baga Koga",
+ "bgp": "Eastern Balochi",
+ "bgq": "Bagri",
+ "bgr": "Bawm Chin",
+ "bgs": "Tagabawa",
+ "bgt": "Bughotu",
+ "bgu": "Mbongno",
+ "bgv": "Warkay-Bipim",
+ "bgw": "Bhatri",
+ "bgx": "Balkan Gagauz Turkish",
+ "bgy": "Benggoi",
+ "bgz": "Banggai",
+ "bh": "Bihari languages",
+ "bha": "Bharia",
+ "bhb": "Bhili",
+ "bhc": "Biga",
+ "bhd": "Bhadrawahi",
+ "bhe": "Bhaya",
+ "bhf": "Odiai",
+ "bhg": "Binandere",
+ "bhh": "Bukharic",
+ "bhi": "Bhilali",
+ "bhj": "Bahing",
+ "bhl": "Bimin",
+ "bhm": "Bathari",
+ "bhn": "Bohtan Neo-Aramaic",
+ "bho": "Bhojpuri",
+ "bhp": "Bima",
+ "bhq": "Tukang Besi South",
+ "bhr": "Bara Malagasy",
+ "bhs": "Buwal",
+ "bht": "Bhattiyali",
+ "bhu": "Bhunjia",
+ "bhv": "Bahau",
+ "bhw": "Biak",
+ "bhx": "Bhalay",
+ "bhy": "Bhele",
+ "bhz": "Bada (Indonesia)",
+ "bi": "Bislama",
+ "bia": "Badimaya",
+ "bib": "Bissa; Bisa",
+ "bid": "Bidiyo",
+ "bie": "Bepour",
+ "bif": "Biafada",
+ "big": "Biangai",
+ "bik": "Bikol",
+ "bil": "Bile",
+ "bim": "Bimoba",
+ "bin": "Bini; Edo",
+ "bio": "Nai",
+ "bip": "Bila",
+ "biq": "Bipi",
+ "bir": "Bisorio",
+ "bit": "Berinomo",
+ "biu": "Biete",
+ "biv": "Southern Birifor",
+ "biw": "Kol (Cameroon)",
+ "bix": "Bijori",
+ "biy": "Birhor",
+ "biz": "Baloi",
+ "bja": "Budza",
+ "bjb": "Banggarla",
+ "bjc": "Bariji",
+ "bje": "Biao-Jiao Mien",
+ "bjf": "Barzani Jewish Neo-Aramaic",
+ "bjg": "Bidyogo",
+ "bjh": "Bahinemo",
+ "bji": "Burji",
+ "bjj": "Kanauji",
+ "bjk": "Barok",
+ "bjl": "Bulu (Papua New Guinea)",
+ "bjm": "Bajelani",
+ "bjn": "Banjar",
+ "bjo": "Mid-Southern Banda",
+ "bjp": "Fanamaket",
+ "bjr": "Binumarien",
+ "bjs": "Bajan",
+ "bjt": "Balanta-Ganja",
+ "bju": "Busuu",
+ "bjv": "Bedjond",
+ "bjw": "Bakwé",
+ "bjx": "Banao Itneg",
+ "bjy": "Bayali",
+ "bjz": "Baruga",
+ "bka": "Kyak",
+ "bkc": "Baka (Cameroon)",
+ "bkd": "Binukid; Talaandig",
+ "bkf": "Beeke",
+ "bkg": "Buraka",
+ "bkh": "Bakoko",
+ "bki": "Baki",
+ "bkj": "Pande",
+ "bkk": "Brokskat",
+ "bkl": "Berik",
+ "bkm": "Kom (Cameroon)",
+ "bkn": "Bukitan",
+ "bko": "Kwa'",
+ "bkp": "Boko (Democratic Republic of Congo)",
+ "bkq": "Bakairí",
+ "bkr": "Bakumpai",
+ "bks": "Northern Sorsoganon",
+ "bkt": "Boloki",
+ "bku": "Buhid",
+ "bkv": "Bekwarra",
+ "bkw": "Bekwel",
+ "bkx": "Baikeno",
+ "bky": "Bokyi",
+ "bkz": "Bungku",
+ "bla": "Siksika",
+ "blb": "Bilua",
+ "blc": "Bella Coola",
+ "bld": "Bolango",
+ "ble": "Balanta-Kentohe",
+ "blf": "Buol",
+ "blh": "Kuwaa",
+ "bli": "Bolia",
+ "blj": "Bolongan",
+ "blk": "Pa'o Karen; Pa'O",
+ "bll": "Biloxi",
+ "blm": "Beli (South Sudan)",
+ "bln": "Southern Catanduanes Bikol",
+ "blo": "Anii",
+ "blp": "Blablanga",
+ "blq": "Baluan-Pam",
+ "blr": "Blang",
+ "bls": "Balaesang",
+ "blt": "Tai Dam",
+ "blv": "Kibala; Bolo",
+ "blw": "Balangao",
+ "blx": "Mag-Indi Ayta",
+ "bly": "Notre",
+ "blz": "Balantak",
+ "bm": "Bambara",
+ "bma": "Lame",
+ "bmb": "Bembe",
+ "bmc": "Biem",
+ "bmd": "Baga Manduri",
+ "bme": "Limassa",
+ "bmf": "Bom-Kim",
+ "bmg": "Bamwe",
+ "bmh": "Kein",
+ "bmi": "Bagirmi",
+ "bmj": "Bote-Majhi",
+ "bmk": "Ghayavi",
+ "bml": "Bomboli",
+ "bmm": "Northern Betsimisaraka Malagasy",
+ "bmn": "Bina (Papua New Guinea)",
+ "bmo": "Bambalang",
+ "bmp": "Bulgebi",
+ "bmq": "Bomu",
+ "bmr": "Muinane",
+ "bms": "Bilma Kanuri",
+ "bmt": "Biao Mon",
+ "bmu": "Somba-Siawari",
+ "bmv": "Bum",
+ "bmw": "Bomwali",
+ "bmx": "Baimak",
+ "bmz": "Baramu",
+ "bn": "Bengali; Bangla",
+ "bna": "Bonerate",
+ "bnb": "Bookan",
+ "bnc": "Bontok",
+ "bnd": "Banda (Indonesia)",
+ "bne": "Bintauna",
+ "bnf": "Masiwang",
+ "bng": "Benga",
+ "bni": "Bangi",
+ "bnj": "Eastern Tawbuid",
+ "bnk": "Bierebo",
+ "bnl": "Boon",
+ "bnm": "Batanga",
+ "bnn": "Bunun",
+ "bno": "Bantoanon",
+ "bnp": "Bola",
+ "bnq": "Bantik",
+ "bnr": "Butmas-Tur",
+ "bns": "Bundeli",
+ "bnt": "Bantu languages",
+ "bnu": "Bentong",
+ "bnv": "Bonerif; Beneraf; Edwas",
+ "bnw": "Bisis",
+ "bnx": "Bangubangu",
+ "bny": "Bintulu",
+ "bnz": "Beezen",
+ "bo": "Tibetan",
+ "boa": "Bora",
+ "bob": "Aweer",
+ "boe": "Mundabli",
+ "bof": "Bolon",
+ "bog": "Bamako Sign Language",
+ "boh": "Boma",
+ "boi": "Barbareño",
+ "boj": "Anjam",
+ "bok": "Bonjo",
+ "bol": "Bole",
+ "bom": "Berom",
+ "bon": "Bine",
+ "boo": "Tiemacèwè Bozo",
+ "bop": "Bonkiman",
+ "boq": "Bogaya",
+ "bor": "Borôro",
+ "bot": "Bongo",
+ "bou": "Bondei",
+ "bov": "Tuwuli",
+ "bow": "Rema",
+ "box": "Buamu",
+ "boy": "Bodo (Central African Republic)",
+ "boz": "Tiéyaxo Bozo",
+ "bpa": "Daakaka",
+ "bpc": "Mbuk",
+ "bpd": "Banda-Banda",
+ "bpe": "Bauni",
+ "bpg": "Bonggo",
+ "bph": "Botlikh",
+ "bpi": "Bagupi",
+ "bpj": "Binji",
+ "bpk": "Orowe; 'Ôrôê",
+ "bpl": "Broome Pearling Lugger Pidgin",
+ "bpm": "Biyom",
+ "bpn": "Dzao Min",
+ "bpo": "Anasi",
+ "bpp": "Kaure",
+ "bpq": "Banda Malay",
+ "bpr": "Koronadal Blaan",
+ "bps": "Sarangani Blaan",
+ "bpt": "Barrow Point",
+ "bpu": "Bongu",
+ "bpv": "Bian Marind",
+ "bpw": "Bo (Papua New Guinea)",
+ "bpx": "Palya Bareli",
+ "bpy": "Bishnupriya",
+ "bpz": "Bilba",
+ "bqa": "Tchumbuli",
+ "bqb": "Bagusa",
+ "bqc": "Boko (Benin); Boo",
+ "bqd": "Bung",
+ "bqf": "Baga Kaloum",
+ "bqg": "Bago-Kusuntu",
+ "bqh": "Baima",
+ "bqi": "Bakhtiari",
+ "bqj": "Bandial",
+ "bqk": "Banda-Mbrès",
+ "bql": "Bilakura",
+ "bqm": "Wumboko",
+ "bqn": "Bulgarian Sign Language",
+ "bqo": "Balo",
+ "bqp": "Busa",
+ "bqq": "Biritai",
+ "bqr": "Burusu",
+ "bqs": "Bosngun",
+ "bqt": "Bamukumbit",
+ "bqu": "Boguru",
+ "bqv": "Koro Wachi; Begbere-Ejar",
+ "bqw": "Buru (Nigeria)",
+ "bqx": "Baangi",
+ "bqy": "Bengkala Sign Language",
+ "bqz": "Bakaka",
+ "br": "Breton",
+ "bra": "Braj",
+ "brb": "Brao; Lave",
+ "brc": "Berbice Creole Dutch",
+ "brd": "Baraamu",
+ "brf": "Bira",
+ "brg": "Baure",
+ "brh": "Brahui",
+ "bri": "Mokpwe",
+ "brj": "Bieria",
+ "brk": "Birked",
+ "brl": "Birwa",
+ "brm": "Barambu",
+ "brn": "Boruca",
+ "bro": "Brokkat",
+ "brp": "Barapasi",
+ "brq": "Breri",
+ "brr": "Birao",
+ "brs": "Baras",
+ "brt": "Bitare",
+ "bru": "Eastern Bru",
+ "brv": "Western Bru",
+ "brw": "Bellari",
+ "brx": "Bodo (India)",
+ "bry": "Burui",
+ "brz": "Bilbil",
+ "bs": "Bosnian",
+ "bsa": "Abinomn",
+ "bsb": "Brunei Bisaya",
+ "bsc": "Bassari; Oniyan",
+ "bse": "Wushi",
+ "bsf": "Bauchi",
+ "bsg": "Bashkardi",
+ "bsh": "Kati",
+ "bsi": "Bassossi",
+ "bsj": "Bangwinji",
+ "bsk": "Burushaski",
+ "bsl": "Basa-Gumna",
+ "bsm": "Busami",
+ "bsn": "Barasana-Eduria",
+ "bso": "Buso",
+ "bsp": "Baga Sitemu",
+ "bsq": "Bassa",
+ "bsr": "Bassa-Kontagora",
+ "bss": "Akoose",
+ "bst": "Basketo",
+ "bsu": "Bahonsuai",
+ "bsv": "Baga Sobané",
+ "bsw": "Baiso",
+ "bsx": "Yangkam",
+ "bsy": "Sabah Bisaya",
+ "bta": "Bata",
+ "btc": "Bati (Cameroon)",
+ "btd": "Batak Dairi",
+ "bte": "Gamo-Ningi",
+ "btf": "Birgit",
+ "btg": "Gagnoa Bété",
+ "bth": "Biatah Bidayuh",
+ "bti": "Burate",
+ "btj": "Bacanese Malay",
+ "btk": "Batak languages",
+ "btm": "Batak Mandailing",
+ "btn": "Ratagnon",
+ "bto": "Rinconada Bikol",
+ "btp": "Budibud",
+ "btq": "Batek",
+ "btr": "Baetora",
+ "bts": "Batak Simalungun",
+ "btt": "Bete-Bendi",
+ "btu": "Batu",
+ "btv": "Bateri",
+ "btw": "Butuanon",
+ "btx": "Batak Karo",
+ "bty": "Bobot",
+ "btz": "Batak Alas-Kluet",
+ "bua": "Buriat",
+ "bub": "Bua",
+ "buc": "Bushi",
+ "bud": "Ntcham",
+ "bue": "Beothuk",
+ "buf": "Bushoong",
+ "bug": "Buginese",
+ "buh": "Younuo Bunu",
+ "bui": "Bongili",
+ "buj": "Basa-Gurmana",
+ "buk": "Bugawac",
+ "bum": "Bulu (Cameroon)",
+ "bun": "Sherbro",
+ "buo": "Terei",
+ "bup": "Busoa",
+ "buq": "Brem",
+ "bus": "Bokobaru",
+ "but": "Bungain",
+ "buu": "Budu",
+ "buv": "Bun",
+ "buw": "Bubi",
+ "bux": "Boghom",
+ "buy": "Bullom So",
+ "buz": "Bukwen",
+ "bva": "Barein",
+ "bvb": "Bube",
+ "bvc": "Baelelea",
+ "bvd": "Baeggu",
+ "bve": "Berau Malay",
+ "bvf": "Boor",
+ "bvg": "Bonkeng",
+ "bvh": "Bure",
+ "bvi": "Belanda Viri",
+ "bvj": "Baan",
+ "bvk": "Bukat",
+ "bvl": "Bolivian Sign Language",
+ "bvm": "Bamunka",
+ "bvn": "Buna",
+ "bvo": "Bolgo",
+ "bvp": "Bumang",
+ "bvq": "Birri",
+ "bvr": "Burarra",
+ "bvt": "Bati (Indonesia)",
+ "bvu": "Bukit Malay",
+ "bvv": "Baniva",
+ "bvw": "Boga",
+ "bvx": "Dibole",
+ "bvy": "Baybayanon",
+ "bvz": "Bauzi",
+ "bwa": "Bwatoo",
+ "bwb": "Namosi-Naitasiri-Serua",
+ "bwc": "Bwile",
+ "bwd": "Bwaidoka",
+ "bwe": "Bwe Karen",
+ "bwf": "Boselewa",
+ "bwg": "Barwe",
+ "bwh": "Bishuo",
+ "bwi": "Baniwa",
+ "bwj": "Láá Láá Bwamu",
+ "bwk": "Bauwaki",
+ "bwl": "Bwela",
+ "bwm": "Biwat",
+ "bwn": "Wunai Bunu",
+ "bwo": "Boro (Ethiopia); Borna (Ethiopia)",
+ "bwp": "Mandobo Bawah",
+ "bwq": "Southern Bobo Madaré",
+ "bwr": "Bura-Pabir",
+ "bws": "Bomboma",
+ "bwt": "Bafaw-Balong",
+ "bwu": "Buli (Ghana)",
+ "bww": "Bwa",
+ "bwx": "Bu-Nao Bunu",
+ "bwy": "Cwi Bwamu",
+ "bwz": "Bwisi",
+ "bxa": "Tairaha",
+ "bxb": "Belanda Bor",
+ "bxc": "Molengue",
+ "bxd": "Pela",
+ "bxe": "Birale",
+ "bxf": "Bilur; Minigir",
+ "bxg": "Bangala",
+ "bxh": "Buhutu",
+ "bxi": "Pirlatapa",
+ "bxj": "Bayungu",
+ "bxk": "Bukusu; Lubukusu",
+ "bxl": "Jalkunan",
+ "bxm": "Mongolia Buriat",
+ "bxn": "Burduna",
+ "bxo": "Barikanchi",
+ "bxp": "Bebil",
+ "bxq": "Beele",
+ "bxr": "Russia Buriat",
+ "bxs": "Busam",
+ "bxu": "China Buriat",
+ "bxv": "Berakou",
+ "bxw": "Bankagooma",
+ "bxz": "Binahari",
+ "bya": "Batak",
+ "byb": "Bikya",
+ "byc": "Ubaghara",
+ "byd": "Benyadu'",
+ "bye": "Pouye",
+ "byf": "Bete",
+ "byg": "Baygo",
+ "byh": "Bhujel",
+ "byi": "Buyu",
+ "byj": "Bina (Nigeria)",
+ "byk": "Biao",
+ "byl": "Bayono",
+ "bym": "Bidjara",
+ "byn": "Bilin; Blin",
+ "byo": "Biyo",
+ "byp": "Bumaji",
+ "byq": "Basay",
+ "byr": "Baruya; Yipma",
+ "bys": "Burak",
+ "byt": "Berti",
+ "byv": "Medumba",
+ "byw": "Belhariya",
+ "byx": "Qaqet",
+ "byz": "Banaro",
+ "bza": "Bandi",
+ "bzb": "Andio",
+ "bzc": "Southern Betsimisaraka Malagasy",
+ "bzd": "Bribri",
+ "bze": "Jenaama Bozo",
+ "bzf": "Boikin",
+ "bzg": "Babuza",
+ "bzh": "Mapos Buang",
+ "bzi": "Bisu",
+ "bzj": "Belize Kriol English",
+ "bzk": "Nicaragua Creole English",
+ "bzl": "Boano (Sulawesi)",
+ "bzm": "Bolondo",
+ "bzn": "Boano (Maluku)",
+ "bzo": "Bozaba",
+ "bzp": "Kemberano",
+ "bzq": "Buli (Indonesia)",
+ "bzr": "Biri",
+ "bzs": "Brazilian Sign Language",
+ "bzt": "Brithenig",
+ "bzu": "Burmeso",
+ "bzv": "Naami",
+ "bzw": "Basa (Nigeria)",
+ "bzx": "Kɛlɛngaxo Bozo",
+ "bzy": "Obanliku",
+ "bzz": "Evant",
+ "ca": "Catalan; Valencian",
+ "caa": "Chortí",
+ "cab": "Garifuna",
+ "cac": "Chuj",
+ "cad": "Caddo",
+ "cae": "Lehar; Laalaa",
+ "caf": "Southern Carrier",
+ "cag": "Nivaclé",
+ "cah": "Cahuarano",
+ "cai": "Central American Indian languages",
+ "caj": "Chané",
+ "cak": "Kaqchikel; Cakchiquel",
+ "cal": "Carolinian",
+ "cam": "Cemuhî",
+ "can": "Chambri",
+ "cao": "Chácobo",
+ "cap": "Chipaya",
+ "caq": "Car Nicobarese",
+ "car": "Galibi Carib",
+ "cas": "Tsimané",
+ "cau": "Caucasian languages",
+ "cav": "Cavineña",
+ "caw": "Callawalla",
+ "cax": "Chiquitano",
+ "cay": "Cayuga",
+ "caz": "Canichana",
+ "cba": "Chibchan languages",
+ "cbb": "Cabiyarí",
+ "cbc": "Carapana",
+ "cbd": "Carijona",
+ "cbg": "Chimila",
+ "cbi": "Chachi",
+ "cbj": "Ede Cabe",
+ "cbk": "Chavacano",
+ "cbl": "Bualkhaw Chin",
+ "cbn": "Nyahkur",
+ "cbo": "Izora",
+ "cbq": "Tsucuba; Cuba",
+ "cbr": "Cashibo-Cacataibo",
+ "cbs": "Cashinahua",
+ "cbt": "Chayahuita",
+ "cbu": "Candoshi-Shapra",
+ "cbv": "Cacua",
+ "cbw": "Kinabalian",
+ "cby": "Carabayo",
+ "ccc": "Chamicuro",
+ "ccd": "Cafundo Creole",
+ "cce": "Chopi",
+ "ccg": "Samba Daka",
+ "cch": "Atsam",
+ "ccj": "Kasanga",
+ "ccl": "Cutchi-Swahili",
+ "ccm": "Malaccan Creole Malay",
+ "ccn": "North Caucasian languages",
+ "cco": "Comaltepec Chinantec",
+ "ccp": "Chakma",
+ "ccr": "Cacaopera",
+ "ccs": "South Caucasian languages",
+ "cda": "Choni",
+ "cdc": "Chadic languages",
+ "cdd": "Caddoan languages",
+ "cde": "Chenchu",
+ "cdf": "Chiru",
+ "cdh": "Chambeali",
+ "cdi": "Chodri",
+ "cdj": "Churahi",
+ "cdm": "Chepang",
+ "cdn": "Chaudangsi",
+ "cdo": "Min Dong Chinese",
+ "cdr": "Cinda-Regi-Tiyal",
+ "cds": "Chadian Sign Language",
+ "cdy": "Chadong",
+ "cdz": "Koda",
+ "ce": "Chechen",
+ "cea": "Lower Chehalis",
+ "ceb": "Cebuano",
+ "ceg": "Chamacoco",
+ "cek": "Eastern Khumi Chin",
+ "cel": "Celtic languages",
+ "cen": "Cen",
+ "cet": "Centúúm",
+ "cey": "Ekai Chin",
+ "cfa": "Dijim-Bwilim",
+ "cfd": "Cara",
+ "cfg": "Como Karim",
+ "cfm": "Falam Chin",
+ "cga": "Changriwa",
+ "cgc": "Kagayanen",
+ "cgg": "Chiga",
+ "cgk": "Chocangacakha",
+ "ch": "Chamorro",
+ "chb": "Chibcha",
+ "chc": "Catawba",
+ "chd": "Highland Oaxaca Chontal",
+ "chf": "Tabasco Chontal",
+ "chg": "Chagatai",
+ "chh": "Chinook",
+ "chj": "Ojitlán Chinantec",
+ "chk": "Chuukese",
+ "chl": "Cahuilla",
+ "chm": "Mari (Russia)",
+ "chn": "Chinook jargon",
+ "cho": "Choctaw",
+ "chp": "Chipewyan; Dene Suline",
+ "chq": "Quiotepec Chinantec",
+ "chr": "Cherokee",
+ "cht": "Cholón",
+ "chw": "Chuwabu",
+ "chx": "Chantyal",
+ "chy": "Cheyenne",
+ "chz": "Ozumacín Chinantec",
+ "cia": "Cia-Cia",
+ "cib": "Ci Gbe",
+ "cic": "Chickasaw",
+ "cid": "Chimariko",
+ "cie": "Cineni",
+ "cih": "Chinali",
+ "cik": "Chitkuli Kinnauri",
+ "cim": "Cimbrian",
+ "cin": "Cinta Larga",
+ "cip": "Chiapanec",
+ "cir": "Tiri; Haméa; Méa",
+ "ciw": "Chippewa",
+ "ciy": "Chaima",
+ "cja": "Western Cham",
+ "cje": "Chru",
+ "cjh": "Upper Chehalis",
+ "cji": "Chamalal",
+ "cjk": "Chokwe",
+ "cjm": "Eastern Cham",
+ "cjn": "Chenapian",
+ "cjo": "Ashéninka Pajonal",
+ "cjp": "Cabécar",
+ "cjs": "Shor",
+ "cjv": "Chuave",
+ "cjy": "Jinyu Chinese",
+ "ckb": "Central Kurdish",
+ "ckh": "Chak",
+ "ckl": "Cibak",
+ "ckm": "Chakavian",
+ "ckn": "Kaang Chin",
+ "cko": "Anufo",
+ "ckq": "Kajakse",
+ "ckr": "Kairak",
+ "cks": "Tayo",
+ "ckt": "Chukot",
+ "cku": "Koasati",
+ "ckv": "Kavalan",
+ "ckx": "Caka",
+ "cky": "Cakfem-Mushere",
+ "ckz": "Cakchiquel-Quiché Mixed Language",
+ "cla": "Ron",
+ "clc": "Chilcotin",
+ "cld": "Chaldean Neo-Aramaic",
+ "cle": "Lealao Chinantec",
+ "clh": "Chilisso",
+ "cli": "Chakali",
+ "clj": "Laitu Chin",
+ "clk": "Idu-Mishmi",
+ "cll": "Chala",
+ "clm": "Clallam",
+ "clo": "Lowland Oaxaca Chontal",
+ "clt": "Lautu Chin",
+ "clu": "Caluyanun",
+ "clw": "Chulym",
+ "cly": "Eastern Highland Chatino",
+ "cma": "Maa",
+ "cmc": "Chamic languages",
+ "cme": "Cerma",
+ "cmg": "Classical Mongolian",
+ "cmi": "Emberá-Chamí",
+ "cml": "Campalagian",
+ "cmm": "Michigamea",
+ "cmn": "Mandarin Chinese",
+ "cmo": "Central Mnong",
+ "cmr": "Mro-Khimi Chin",
+ "cms": "Messapic",
+ "cmt": "Camtho",
+ "cna": "Changthang",
+ "cnb": "Chinbon Chin",
+ "cnc": "Côông",
+ "cng": "Northern Qiang",
+ "cnh": "Hakha Chin; Haka Chin",
+ "cni": "Asháninka",
+ "cnk": "Khumi Chin",
+ "cnl": "Lalana Chinantec",
+ "cno": "Con",
+ "cnp": "Northern Ping Chinese; Northern Pinghua",
+ "cnq": "Chung",
+ "cnr": "Montenegrin",
+ "cns": "Central Asmat",
+ "cnt": "Tepetotutla Chinantec",
+ "cnu": "Chenoua",
+ "cnw": "Ngawn Chin",
+ "cnx": "Middle Cornish",
+ "co": "Corsican",
+ "coa": "Cocos Islands Malay",
+ "cob": "Chicomuceltec",
+ "coc": "Cocopa",
+ "cod": "Cocama-Cocamilla",
+ "coe": "Koreguaje",
+ "cof": "Colorado",
+ "cog": "Chong",
+ "coh": "Chonyi-Dzihana-Kauma; Chichonyi-Chidzihana-Chikauma",
+ "coj": "Cochimi",
+ "cok": "Santa Teresa Cora",
+ "col": "Columbia-Wenatchi",
+ "com": "Comanche",
+ "con": "Cofán",
+ "coo": "Comox",
+ "cop": "Coptic",
+ "coq": "Coquille",
+ "cot": "Caquinte",
+ "cou": "Wamey",
+ "cov": "Cao Miao",
+ "cow": "Cowlitz",
+ "cox": "Nanti",
+ "coz": "Chochotec",
+ "cpa": "Palantla Chinantec",
+ "cpb": "Ucayali-Yurúa Ashéninka",
+ "cpc": "Ajyíninka Apurucayali",
+ "cpe": "English-based creoles and pidgins",
+ "cpf": "French-based creoles and pidgins",
+ "cpg": "Cappadocian Greek",
+ "cpi": "Chinese Pidgin English",
+ "cpn": "Cherepon",
+ "cpo": "Kpeego",
+ "cpp": "Portuguese-based creoles and pidgins",
+ "cps": "Capiznon",
+ "cpu": "Pichis Ashéninka",
+ "cpx": "Pu-Xian Chinese",
+ "cpy": "South Ucayali Ashéninka",
+ "cqd": "Chuanqiandian Cluster Miao",
+ "cr": "Cree",
+ "cra": "Chara",
+ "crb": "Island Carib",
+ "crc": "Lonwolwol",
+ "crd": "Coeur d'Alene",
+ "crf": "Caramanta",
+ "crg": "Michif",
+ "crh": "Crimean Tatar; Crimean Turkish",
+ "cri": "Sãotomense",
+ "crj": "Southern East Cree",
+ "crk": "Plains Cree",
+ "crl": "Northern East Cree",
+ "crm": "Moose Cree",
+ "crn": "El Nayar Cora",
+ "cro": "Crow",
+ "crp": "Creoles and pidgins",
+ "crq": "Iyo'wujwa Chorote",
+ "crr": "Carolina Algonquian",
+ "crs": "Seselwa Creole French",
+ "crt": "Iyojwa'ja Chorote",
+ "crv": "Chaura",
+ "crw": "Chrau",
+ "crx": "Carrier",
+ "cry": "Cori",
+ "crz": "Cruzeño",
+ "cs": "Czech",
+ "csa": "Chiltepec Chinantec",
+ "csb": "Kashubian",
+ "csc": "Catalan Sign Language; Lengua de señas catalana; Llengua de Signes Catalana",
+ "csd": "Chiangmai Sign Language",
+ "cse": "Czech Sign Language",
+ "csf": "Cuba Sign Language",
+ "csg": "Chilean Sign Language",
+ "csh": "Asho Chin",
+ "csi": "Coast Miwok",
+ "csj": "Songlai Chin",
+ "csk": "Jola-Kasa",
+ "csl": "Chinese Sign Language",
+ "csm": "Central Sierra Miwok",
+ "csn": "Colombian Sign Language",
+ "cso": "Sochiapam Chinantec; Sochiapan Chinantec",
+ "csp": "Southern Ping Chinese; Southern Pinghua",
+ "csq": "Croatia Sign Language",
+ "csr": "Costa Rican Sign Language",
+ "css": "Southern Ohlone",
+ "cst": "Northern Ohlone",
+ "csu": "Central Sudanic languages",
+ "csv": "Sumtu Chin",
+ "csw": "Swampy Cree",
+ "csx": "Cambodian Sign Language",
+ "csy": "Siyin Chin",
+ "csz": "Coos",
+ "cta": "Tataltepec Chatino",
+ "ctc": "Chetco",
+ "ctd": "Tedim Chin",
+ "cte": "Tepinapa Chinantec",
+ "ctg": "Chittagonian",
+ "cth": "Thaiphum Chin",
+ "ctl": "Tlacoatzintepec Chinantec",
+ "ctm": "Chitimacha",
+ "ctn": "Chhintange",
+ "cto": "Emberá-Catío",
+ "ctp": "Western Highland Chatino",
+ "cts": "Northern Catanduanes Bikol",
+ "ctt": "Wayanad Chetti",
+ "ctu": "Chol",
+ "cty": "Moundadan Chetty",
+ "ctz": "Zacatepec Chatino",
+ "cu": "Church Slavic; Church Slavonic; Old Bulgarian; Old Church Slavonic; Old Slavonic",
+ "cua": "Cua",
+ "cub": "Cubeo",
+ "cuc": "Usila Chinantec",
+ "cuh": "Chuka; Gichuka",
+ "cui": "Cuiba",
+ "cuj": "Mashco Piro",
+ "cuk": "San Blas Kuna",
+ "cul": "Culina; Kulina",
+ "cuo": "Cumanagoto",
+ "cup": "Cupeño",
+ "cuq": "Cun",
+ "cur": "Chhulung",
+ "cus": "Cushitic languages",
+ "cut": "Teutila Cuicatec",
+ "cuu": "Tai Ya",
+ "cuv": "Cuvok",
+ "cuw": "Chukwa",
+ "cux": "Tepeuxila Cuicatec",
+ "cuy": "Cuitlatec",
+ "cv": "Chuvash",
+ "cvg": "Chug",
+ "cvn": "Valle Nacional Chinantec",
+ "cwa": "Kabwa",
+ "cwb": "Maindo",
+ "cwd": "Woods Cree",
+ "cwe": "Kwere",
+ "cwg": "Chewong; Cheq Wong",
+ "cwt": "Kuwaataay",
+ "cy": "Welsh",
+ "cya": "Nopala Chatino",
+ "cyb": "Cayubaba",
+ "cyo": "Cuyonon",
+ "czh": "Huizhou Chinese",
+ "czk": "Knaanic",
+ "czn": "Zenzontepec Chatino",
+ "czo": "Min Zhong Chinese",
+ "czt": "Zotung Chin",
+ "da": "Danish",
+ "daa": "Dangaléat",
+ "dac": "Dambi",
+ "dad": "Marik",
+ "dae": "Duupa",
+ "dag": "Dagbani",
+ "dah": "Gwahatike",
+ "dai": "Day",
+ "daj": "Dar Fur Daju",
+ "dak": "Dakota",
+ "dal": "Dahalo",
+ "dam": "Damakawa",
+ "dao": "Daai Chin",
+ "daq": "Dandami Maria",
+ "dar": "Dargwa",
+ "das": "Daho-Doo",
+ "dau": "Dar Sila Daju",
+ "dav": "Taita; Dawida",
+ "daw": "Davawenyo",
+ "dax": "Dayi",
+ "day": "Land Dayak languages",
+ "daz": "Dao",
+ "dba": "Bangime",
+ "dbb": "Deno",
+ "dbd": "Dadiya",
+ "dbe": "Dabe",
+ "dbf": "Edopi",
+ "dbg": "Dogul Dom Dogon",
+ "dbi": "Doka",
+ "dbj": "Ida'an",
+ "dbl": "Dyirbal",
+ "dbm": "Duguri",
+ "dbn": "Duriankere",
+ "dbo": "Dulbu",
+ "dbp": "Duwai",
+ "dbq": "Daba",
+ "dbr": "Dabarre",
+ "dbt": "Ben Tey Dogon",
+ "dbu": "Bondum Dom Dogon",
+ "dbv": "Dungu",
+ "dbw": "Bankan Tey Dogon",
+ "dby": "Dibiyaso",
+ "dcc": "Deccan",
+ "dcr": "Negerhollands",
+ "dda": "Dadi Dadi",
+ "ddd": "Dongotono",
+ "dde": "Doondo",
+ "ddg": "Fataluku",
+ "ddi": "West Goodenough",
+ "ddj": "Jaru",
+ "ddn": "Dendi (Benin)",
+ "ddo": "Dido",
+ "ddr": "Dhudhuroa",
+ "dds": "Donno So Dogon",
+ "ddw": "Dawera-Daweloor",
+ "de": "German",
+ "dec": "Dagik",
+ "ded": "Dedua",
+ "dee": "Dewoin",
+ "def": "Dezfuli",
+ "deg": "Degema",
+ "deh": "Dehwari",
+ "dei": "Demisa",
+ "dek": "Dek",
+ "del": "Delaware",
+ "dem": "Dem",
+ "den": "Slave (Athapascan)",
+ "dep": "Pidgin Delaware",
+ "deq": "Dendi (Central African Republic)",
+ "der": "Deori",
+ "des": "Desano",
+ "dev": "Domung",
+ "dez": "Dengese",
+ "dga": "Southern Dagaare",
+ "dgb": "Bunoge Dogon",
+ "dgc": "Casiguran Dumagat Agta",
+ "dgd": "Dagaari Dioula",
+ "dge": "Degenan",
+ "dgg": "Doga",
+ "dgh": "Dghwede",
+ "dgi": "Northern Dagara",
+ "dgk": "Dagba",
+ "dgl": "Andaandi; Dongolawi",
+ "dgn": "Dagoman",
+ "dgo": "Dogri (individual language)",
+ "dgr": "Dogrib; Tłı̨chǫ",
+ "dgs": "Dogoso",
+ "dgt": "Ndra'ngith",
+ "dgw": "Daungwurrung",
+ "dgx": "Doghoro",
+ "dgz": "Daga",
+ "dhd": "Dhundari",
+ "dhg": "Dhangu-Djangu; Dhangu; Djangu",
+ "dhi": "Dhimal",
+ "dhl": "Dhalandji",
+ "dhm": "Zemba",
+ "dhn": "Dhanki",
+ "dho": "Dhodia",
+ "dhr": "Dhargari",
+ "dhs": "Dhaiso",
+ "dhu": "Dhurga",
+ "dhv": "Dehu; Drehu",
+ "dhw": "Dhanwar (Nepal)",
+ "dhx": "Dhungaloo",
+ "dia": "Dia",
+ "dib": "South Central Dinka",
+ "dic": "Lakota Dida",
+ "did": "Didinga",
+ "dif": "Dieri; Diyari",
+ "dig": "Digo; Chidigo",
+ "dih": "Kumiai",
+ "dii": "Dimbong",
+ "dij": "Dai",
+ "dik": "Southwestern Dinka",
+ "dil": "Dilling",
+ "dim": "Dime",
+ "din": "Dinka",
+ "dio": "Dibo",
+ "dip": "Northeastern Dinka",
+ "diq": "Dimli (individual language)",
+ "dir": "Dirim",
+ "dis": "Dimasa",
+ "diu": "Diriku",
+ "diw": "Northwestern Dinka",
+ "dix": "Dixon Reef",
+ "diy": "Diuwe",
+ "diz": "Ding",
+ "dja": "Djadjawurrung",
+ "djb": "Djinba",
+ "djc": "Dar Daju Daju",
+ "djd": "Djamindjung; Ngaliwurru",
+ "dje": "Zarma",
+ "djf": "Djangun",
+ "dji": "Djinang",
+ "djj": "Djeebbana",
+ "djk": "Eastern Maroon Creole; Businenge Tongo; Nenge",
+ "djm": "Jamsay Dogon",
+ "djn": "Jawoyn; Djauan",
+ "djo": "Jangkang",
+ "djr": "Djambarrpuyngu",
+ "dju": "Kapriman",
+ "djw": "Djawi",
+ "dka": "Dakpakha",
+ "dkg": "Kadung",
+ "dkk": "Dakka",
+ "dkr": "Kuijau",
+ "dks": "Southeastern Dinka",
+ "dkx": "Mazagway",
+ "dlg": "Dolgan",
+ "dlk": "Dahalik",
+ "dlm": "Dalmatian",
+ "dln": "Darlong",
+ "dma": "Duma",
+ "dmb": "Mombo Dogon",
+ "dmc": "Gavak",
+ "dmd": "Madhi Madhi",
+ "dme": "Dugwor",
+ "dmf": "Medefaidrin",
+ "dmg": "Upper Kinabatangan",
+ "dmk": "Domaaki",
+ "dml": "Dameli",
+ "dmm": "Dama",
+ "dmn": "Mande languages",
+ "dmo": "Kemedzung",
+ "dmr": "East Damar",
+ "dms": "Dampelas",
+ "dmu": "Dubu; Tebi",
+ "dmv": "Dumpas",
+ "dmw": "Mudburra",
+ "dmx": "Dema",
+ "dmy": "Demta; Sowari",
+ "dna": "Upper Grand Valley Dani",
+ "dnd": "Daonda",
+ "dne": "Ndendeule",
+ "dng": "Dungan",
+ "dni": "Lower Grand Valley Dani",
+ "dnj": "Dan",
+ "dnk": "Dengka",
+ "dnn": "Dzùùngoo",
+ "dno": "Ndrulo; Northern Lendu",
+ "dnr": "Danaru",
+ "dnt": "Mid Grand Valley Dani",
+ "dnu": "Danau",
+ "dnv": "Danu",
+ "dnw": "Western Dani",
+ "dny": "Dení",
+ "doa": "Dom",
+ "dob": "Dobu",
+ "doc": "Northern Dong",
+ "doe": "Doe",
+ "dof": "Domu",
+ "doh": "Dong",
+ "doi": "Dogri (macrolanguage)",
+ "dok": "Dondo",
+ "dol": "Doso",
+ "don": "Toura (Papua New Guinea)",
+ "doo": "Dongo",
+ "dop": "Lukpa",
+ "doq": "Dominican Sign Language",
+ "dor": "Dori'o",
+ "dos": "Dogosé",
+ "dot": "Dass",
+ "dov": "Dombe",
+ "dow": "Doyayo",
+ "dox": "Bussa",
+ "doy": "Dompo",
+ "doz": "Dorze",
+ "dpp": "Papar",
+ "dra": "Dravidian languages",
+ "drb": "Dair",
+ "drc": "Minderico",
+ "drd": "Darmiya",
+ "dre": "Dolpo",
+ "drg": "Rungus",
+ "dri": "C'Lela",
+ "drl": "Paakantyi",
+ "drn": "West Damar",
+ "dro": "Daro-Matu Melanau",
+ "drq": "Dura",
+ "drs": "Gedeo",
+ "drt": "Drents",
+ "dru": "Rukai",
+ "dry": "Darai",
+ "dsb": "Lower Sorbian",
+ "dse": "Dutch Sign Language",
+ "dsh": "Daasanach",
+ "dsi": "Disa",
+ "dsl": "Danish Sign Language",
+ "dsn": "Dusner",
+ "dso": "Desiya",
+ "dsq": "Tadaksahak",
+ "dsz": "Mardin Sign Language",
+ "dta": "Daur",
+ "dtb": "Labuk-Kinabatangan Kadazan",
+ "dtd": "Ditidaht",
+ "dth": "Adithinngithigh",
+ "dti": "Ana Tinga Dogon",
+ "dtk": "Tene Kan Dogon",
+ "dtm": "Tomo Kan Dogon",
+ "dtn": "Daatsʼíin",
+ "dto": "Tommo So Dogon",
+ "dtp": "Kadazan Dusun; Central Dusun",
+ "dtr": "Lotud",
+ "dts": "Toro So Dogon",
+ "dtt": "Toro Tegu Dogon",
+ "dtu": "Tebul Ure Dogon",
+ "dty": "Dotyali",
+ "dua": "Duala",
+ "dub": "Dubli",
+ "duc": "Duna",
+ "due": "Umiray Dumaget Agta",
+ "duf": "Dumbea; Drubea",
+ "dug": "Duruma; Chiduruma",
+ "duh": "Dungra Bhil",
+ "dui": "Dumun",
+ "duk": "Uyajitaya",
+ "dul": "Alabat Island Agta",
+ "dum": "Middle Dutch (ca. 1050-1350)",
+ "dun": "Dusun Deyah",
+ "duo": "Dupaninan Agta",
+ "dup": "Duano",
+ "duq": "Dusun Malang",
+ "dur": "Dii",
+ "dus": "Dumi",
+ "duu": "Drung",
+ "duv": "Duvle",
+ "duw": "Dusun Witu",
+ "dux": "Duungooma",
+ "duy": "Dicamay Agta",
+ "duz": "Duli-Gey",
+ "dv": "Dhivehi; Divehi; Maldivian",
+ "dva": "Duau",
+ "dwa": "Diri",
+ "dwk": "Dawik Kui",
+ "dwr": "Dawro",
+ "dws": "Dutton World Speedwords",
+ "dwu": "Dhuwal",
+ "dww": "Dawawa",
+ "dwy": "Dhuwaya",
+ "dwz": "Dewas Rai",
+ "dya": "Dyan",
+ "dyb": "Dyaberdyaber",
+ "dyd": "Dyugun",
+ "dyg": "Villa Viciosa Agta",
+ "dyi": "Djimini Senoufo",
+ "dym": "Yanda Dom Dogon",
+ "dyn": "Dyangadi; Dhanggatti",
+ "dyo": "Jola-Fonyi",
+ "dyu": "Dyula",
+ "dyy": "Djabugay; Dyaabugay",
+ "dz": "Dzongkha",
+ "dza": "Tunzu",
+ "dze": "Djiwarli",
+ "dzg": "Dazaga",
+ "dzl": "Dzalakha",
+ "dzn": "Dzando",
+ "eaa": "Karenggapa",
+ "ebc": "Beginci",
+ "ebg": "Ebughu",
+ "ebk": "Eastern Bontok",
+ "ebo": "Teke-Ebo",
+ "ebr": "Ebrié",
+ "ebu": "Embu; Kiembu",
+ "ecr": "Eteocretan",
+ "ecs": "Ecuadorian Sign Language",
+ "ecy": "Eteocypriot",
+ "ee": "Ewe",
+ "eee": "E",
+ "efa": "Efai",
+ "efe": "Efe",
+ "efi": "Efik",
+ "ega": "Ega",
+ "egl": "Emilian",
+ "egm": "Benamanga",
+ "ego": "Eggon",
+ "egx": "Egyptian languages",
+ "egy": "Egyptian (Ancient)",
+ "ehs": "Miyakubo Sign Language",
+ "ehu": "Ehueun",
+ "eip": "Eipomek",
+ "eit": "Eitiep",
+ "eiv": "Askopan",
+ "eja": "Ejamat",
+ "eka": "Ekajuk",
+ "eke": "Ekit",
+ "ekg": "Ekari",
+ "eki": "Eki",
+ "ekk": "Standard Estonian",
+ "ekl": "Kol (Bangladesh); Kol",
+ "ekm": "Elip",
+ "eko": "Koti",
+ "ekp": "Ekpeye",
+ "ekr": "Yace",
+ "eky": "Eastern Kayah",
+ "el": "Modern Greek (1453-)",
+ "ele": "Elepi",
+ "elh": "El Hugeirat",
+ "eli": "Nding",
+ "elk": "Elkei",
+ "elm": "Eleme",
+ "elo": "El Molo",
+ "elu": "Elu",
+ "elx": "Elamite",
+ "ema": "Emai-Iuleha-Ora",
+ "emb": "Embaloh",
+ "eme": "Emerillon",
+ "emg": "Eastern Meohang",
+ "emi": "Mussau-Emira",
+ "emk": "Eastern Maninkakan",
+ "emm": "Mamulique",
+ "emn": "Eman",
+ "emp": "Northern Emberá",
+ "emq": "Eastern Minyag",
+ "ems": "Pacific Gulf Yupik",
+ "emu": "Eastern Muria",
+ "emw": "Emplawas",
+ "emx": "Erromintxela",
+ "emy": "Epigraphic Mayan",
+ "emz": "Mbessa",
+ "en": "English",
+ "ena": "Apali",
+ "enb": "Markweeta",
+ "enc": "En",
+ "end": "Ende",
+ "enf": "Forest Enets",
+ "enh": "Tundra Enets",
+ "enl": "Enlhet",
+ "enm": "Middle English (1100-1500)",
+ "enn": "Engenni",
+ "eno": "Enggano",
+ "enq": "Enga",
+ "enr": "Emumu; Emem",
+ "enu": "Enu",
+ "env": "Enwan (Edo State)",
+ "enw": "Enwan (Akwa Ibom State)",
+ "enx": "Enxet",
+ "eo": "Esperanto",
+ "eot": "Beti (Côte d'Ivoire)",
+ "epi": "Epie",
+ "era": "Eravallan",
+ "erg": "Sie",
+ "erh": "Eruwa",
+ "eri": "Ogea",
+ "erk": "South Efate",
+ "ero": "Horpa",
+ "err": "Erre",
+ "ers": "Ersu",
+ "ert": "Eritai",
+ "erw": "Erokwanas",
+ "es": "Spanish; Castilian",
+ "ese": "Ese Ejja",
+ "esg": "Aheri Gondi",
+ "esh": "Eshtehardi",
+ "esi": "North Alaskan Inupiatun",
+ "esk": "Northwest Alaska Inupiatun",
+ "esl": "Egypt Sign Language",
+ "esm": "Esuma",
+ "esn": "Salvadoran Sign Language",
+ "eso": "Estonian Sign Language",
+ "esq": "Esselen",
+ "ess": "Central Siberian Yupik",
+ "esu": "Central Yupik",
+ "esx": "Eskimo-Aleut languages",
+ "esy": "Eskayan",
+ "et": "Estonian",
+ "etb": "Etebi",
+ "etc": "Etchemin",
+ "eth": "Ethiopian Sign Language",
+ "etn": "Eton (Vanuatu)",
+ "eto": "Eton (Cameroon)",
+ "etr": "Edolo",
+ "ets": "Yekhee",
+ "ett": "Etruscan",
+ "etu": "Ejagham",
+ "etx": "Eten",
+ "etz": "Semimi",
+ "eu": "Basque",
+ "euq": "Basque (family)",
+ "eve": "Even",
+ "evh": "Uvbie",
+ "evn": "Evenki",
+ "ewo": "Ewondo",
+ "ext": "Extremaduran",
+ "eya": "Eyak",
+ "eyo": "Keiyo",
+ "eza": "Ezaa",
+ "eze": "Uzekwe",
+ "fa": "Persian",
+ "faa": "Fasu",
+ "fab": "Fa d'Ambu",
+ "fad": "Wagi",
+ "faf": "Fagani",
+ "fag": "Finongan",
+ "fah": "Baissa Fali",
+ "fai": "Faiwol",
+ "faj": "Faita",
+ "fak": "Fang (Cameroon)",
+ "fal": "South Fali",
+ "fam": "Fam",
+ "fan": "Fang (Equatorial Guinea)",
+ "fap": "Paloor",
+ "far": "Fataleka",
+ "fat": "Fanti",
+ "fau": "Fayu",
+ "fax": "Fala",
+ "fay": "Southwestern Fars",
+ "faz": "Northwestern Fars",
+ "fbl": "West Albay Bikol",
+ "fcs": "Quebec Sign Language",
+ "fer": "Feroge",
+ "ff": "Fulah",
+ "ffi": "Foia Foia",
+ "ffm": "Maasina Fulfulde",
+ "fgr": "Fongoro",
+ "fi": "Finnish",
+ "fia": "Nobiin",
+ "fie": "Fyer",
+ "fif": "Faifi",
+ "fil": "Filipino; Pilipino",
+ "fip": "Fipa",
+ "fir": "Firan",
+ "fit": "Tornedalen Finnish; Meänkieli",
+ "fiu": "Finno-Ugrian languages",
+ "fiw": "Fiwaga",
+ "fj": "Fijian",
+ "fkk": "Kirya-Konzəl",
+ "fkv": "Kven Finnish",
+ "fla": "Kalispel-Pend d'Oreille",
+ "flh": "Foau",
+ "fli": "Fali",
+ "fll": "North Fali",
+ "fln": "Flinders Island",
+ "flr": "Fuliiru",
+ "fly": "Flaaitaal; Tsotsitaal",
+ "fmp": "Fe'fe'",
+ "fmu": "Far Western Muria",
+ "fnb": "Fanbak",
+ "fng": "Fanagalo",
+ "fni": "Fania",
+ "fo": "Faroese",
+ "fod": "Foodo",
+ "foi": "Foi",
+ "fom": "Foma",
+ "fon": "Fon",
+ "for": "Fore",
+ "fos": "Siraya",
+ "fox": "Formosan languages",
+ "fpe": "Fernando Po Creole English",
+ "fqs": "Fas",
+ "fr": "French",
+ "frc": "Cajun French",
+ "frd": "Fordata",
+ "frk": "Frankish",
+ "frm": "Middle French (ca. 1400-1600)",
+ "fro": "Old French (842-ca. 1400)",
+ "frp": "Arpitan; Francoprovençal",
+ "frq": "Forak",
+ "frr": "Northern Frisian",
+ "frs": "Eastern Frisian",
+ "frt": "Fortsenal",
+ "fse": "Finnish Sign Language",
+ "fsl": "French Sign Language",
+ "fss": "Finland-Swedish Sign Language; finlandssvenskt teckenspråk; suomenruotsalainen viittomakieli",
+ "fub": "Adamawa Fulfulde",
+ "fuc": "Pulaar",
+ "fud": "East Futuna",
+ "fue": "Borgu Fulfulde",
+ "fuf": "Pular",
+ "fuh": "Western Niger Fulfulde",
+ "fui": "Bagirmi Fulfulde",
+ "fuj": "Ko",
+ "fum": "Fum",
+ "fun": "Fulniô",
+ "fuq": "Central-Eastern Niger Fulfulde",
+ "fur": "Friulian",
+ "fut": "Futuna-Aniwa",
+ "fuu": "Furu",
+ "fuv": "Nigerian Fulfulde",
+ "fuy": "Fuyug",
+ "fvr": "Fur",
+ "fwa": "Fwâi",
+ "fwe": "Fwe",
+ "fy": "Western Frisian",
+ "ga": "Irish",
+ "gaa": "Ga",
+ "gab": "Gabri",
+ "gac": "Mixed Great Andamanese",
+ "gad": "Gaddang",
+ "gae": "Guarequena",
+ "gaf": "Gende",
+ "gag": "Gagauz",
+ "gah": "Alekano",
+ "gai": "Borei",
+ "gaj": "Gadsup",
+ "gak": "Gamkonora",
+ "gal": "Galolen",
+ "gam": "Kandawo",
+ "gan": "Gan Chinese",
+ "gao": "Gants",
+ "gap": "Gal",
+ "gaq": "Gata'",
+ "gar": "Galeya",
+ "gas": "Adiwasi Garasia",
+ "gat": "Kenati",
+ "gau": "Mudhili Gadaba",
+ "gaw": "Nobonob",
+ "gax": "Borana-Arsi-Guji Oromo",
+ "gay": "Gayo",
+ "gaz": "West Central Oromo",
+ "gba": "Gbaya (Central African Republic)",
+ "gbb": "Kaytetye",
+ "gbd": "Karajarri",
+ "gbe": "Niksek",
+ "gbf": "Gaikundi",
+ "gbg": "Gbanziri",
+ "gbh": "Defi Gbe",
+ "gbi": "Galela",
+ "gbj": "Bodo Gadaba",
+ "gbk": "Gaddi",
+ "gbl": "Gamit",
+ "gbm": "Garhwali",
+ "gbn": "Mo'da",
+ "gbo": "Northern Grebo",
+ "gbp": "Gbaya-Bossangoa",
+ "gbq": "Gbaya-Bozoum",
+ "gbr": "Gbagyi",
+ "gbs": "Gbesi Gbe",
+ "gbu": "Gagadu",
+ "gbv": "Gbanu",
+ "gbw": "Gabi-Gabi",
+ "gbx": "Eastern Xwla Gbe",
+ "gby": "Gbari",
+ "gbz": "Zoroastrian Dari",
+ "gcc": "Mali",
+ "gcd": "Ganggalida",
+ "gce": "Galice",
+ "gcf": "Guadeloupean Creole French",
+ "gcl": "Grenadian Creole English",
+ "gcn": "Gaina",
+ "gcr": "Guianese Creole French",
+ "gct": "Colonia Tovar German",
+ "gd": "Scottish Gaelic; Gaelic",
+ "gda": "Gade Lohar",
+ "gdb": "Pottangi Ollar Gadaba",
+ "gdc": "Gugu Badhun",
+ "gdd": "Gedaged",
+ "gde": "Gude",
+ "gdf": "Guduf-Gava",
+ "gdg": "Ga'dang",
+ "gdh": "Gadjerawang; Gajirrabeng",
+ "gdi": "Gundi",
+ "gdj": "Gurdjar",
+ "gdk": "Gadang",
+ "gdl": "Dirasha",
+ "gdm": "Laal",
+ "gdn": "Umanakaina",
+ "gdo": "Ghodoberi",
+ "gdq": "Mehri",
+ "gdr": "Wipi",
+ "gds": "Ghandruk Sign Language",
+ "gdt": "Kungardutyi",
+ "gdu": "Gudu",
+ "gdx": "Godwari",
+ "gea": "Geruma",
+ "geb": "Kire",
+ "gec": "Gboloo Grebo",
+ "ged": "Gade",
+ "gef": "Gerai",
+ "geg": "Gengle",
+ "geh": "Hutterite German; Hutterisch",
+ "gei": "Gebe",
+ "gej": "Gen",
+ "gek": "Ywom",
+ "gel": "ut-Ma'in",
+ "gem": "Germanic languages",
+ "geq": "Geme",
+ "ges": "Geser-Gorom",
+ "gev": "Eviya",
+ "gew": "Gera",
+ "gex": "Garre",
+ "gey": "Enya",
+ "gez": "Geez",
+ "gfk": "Patpatar",
+ "gft": "Gafat",
+ "gga": "Gao",
+ "ggb": "Gbii",
+ "ggd": "Gugadj",
+ "gge": "Gurr-goni",
+ "ggg": "Gurgula",
+ "ggk": "Kungarakany",
+ "ggl": "Ganglau",
+ "ggt": "Gitua",
+ "ggu": "Gagu; Gban",
+ "ggw": "Gogodala",
+ "gha": "Ghadamès",
+ "ghc": "Hiberno-Scottish Gaelic",
+ "ghe": "Southern Ghale",
+ "ghh": "Northern Ghale",
+ "ghk": "Geko Karen",
+ "ghl": "Ghulfan",
+ "ghn": "Ghanongga",
+ "gho": "Ghomara",
+ "ghr": "Ghera",
+ "ghs": "Guhu-Samane",
+ "ght": "Kuke; Kutang Ghale",
+ "gia": "Kija",
+ "gib": "Gibanawa",
+ "gic": "Gail",
+ "gid": "Gidar",
+ "gie": "Gaɓogbo; Guébie",
+ "gig": "Goaria",
+ "gih": "Githabul",
+ "gii": "Girirra",
+ "gil": "Gilbertese",
+ "gim": "Gimi (Eastern Highlands)",
+ "gin": "Hinukh",
+ "gip": "Gimi (West New Britain)",
+ "giq": "Green Gelao",
+ "gir": "Red Gelao",
+ "gis": "North Giziga",
+ "git": "Gitxsan",
+ "giu": "Mulao",
+ "giw": "White Gelao",
+ "gix": "Gilima",
+ "giy": "Giyug",
+ "giz": "South Giziga",
+ "gjk": "Kachi Koli",
+ "gjm": "Gunditjmara",
+ "gjn": "Gonja",
+ "gjr": "Gurindji Kriol",
+ "gju": "Gujari",
+ "gka": "Guya",
+ "gkd": "Magɨ (Madang Province)",
+ "gke": "Ndai",
+ "gkn": "Gokana",
+ "gko": "Kok-Nar",
+ "gkp": "Guinea Kpelle",
+ "gku": "ǂUngkue",
+ "gl": "Galician",
+ "glb": "Belning",
+ "glc": "Bon Gula",
+ "gld": "Nanai",
+ "glh": "Northwest Pashai; Northwest Pashayi",
+ "glj": "Gula Iro",
+ "glk": "Gilaki",
+ "gll": "Garlali",
+ "glo": "Galambu",
+ "glr": "Glaro-Twabo",
+ "glu": "Gula (Chad)",
+ "glw": "Glavda",
+ "gly": "Gule",
+ "gma": "Gambera",
+ "gmb": "Gula'alaa",
+ "gmd": "Mághdì",
+ "gme": "East Germanic languages",
+ "gmg": "Magɨyi",
+ "gmh": "Middle High German (ca. 1050-1500)",
+ "gml": "Middle Low German",
+ "gmm": "Gbaya-Mbodomo",
+ "gmn": "Gimnime",
+ "gmq": "North Germanic languages",
+ "gmr": "Mirning; Mirniny",
+ "gmu": "Gumalu",
+ "gmv": "Gamo",
+ "gmw": "West Germanic languages",
+ "gmx": "Magoma",
+ "gmy": "Mycenaean Greek",
+ "gmz": "Mgbolizhia",
+ "gn": "Guarani",
+ "gna": "Kaansa",
+ "gnb": "Gangte",
+ "gnc": "Guanche",
+ "gnd": "Zulgo-Gemzek",
+ "gne": "Ganang",
+ "gng": "Ngangam",
+ "gnh": "Lere",
+ "gni": "Gooniyandi",
+ "gnj": "Ngen",
+ "gnk": "ǁGana",
+ "gnl": "Gangulu",
+ "gnm": "Ginuman",
+ "gnn": "Gumatj",
+ "gno": "Northern Gondi",
+ "gnq": "Gana",
+ "gnr": "Gureng Gureng",
+ "gnt": "Guntai",
+ "gnu": "Gnau",
+ "gnw": "Western Bolivian Guaraní",
+ "gnz": "Ganzi",
+ "goa": "Guro",
+ "gob": "Playero",
+ "goc": "Gorakor",
+ "god": "Godié",
+ "goe": "Gongduk",
+ "gof": "Gofa",
+ "gog": "Gogo",
+ "goh": "Old High German (ca. 750-1050)",
+ "goi": "Gobasi",
+ "goj": "Gowlan",
+ "gok": "Gowli",
+ "gol": "Gola",
+ "gom": "Goan Konkani",
+ "gon": "Gondi",
+ "goo": "Gone Dau",
+ "gop": "Yeretuar",
+ "goq": "Gorap",
+ "gor": "Gorontalo",
+ "gos": "Gronings",
+ "got": "Gothic",
+ "gou": "Gavar",
+ "gov": "Goo",
+ "gow": "Gorowa",
+ "gox": "Gobu",
+ "goy": "Goundo",
+ "goz": "Gozarkhani",
+ "gpa": "Gupa-Abawa",
+ "gpe": "Ghanaian Pidgin English",
+ "gpn": "Taiap",
+ "gqa": "Ga'anda",
+ "gqi": "Guiqiong",
+ "gqn": "Guana (Brazil)",
+ "gqr": "Gor",
+ "gqu": "Qau",
+ "gra": "Rajput Garasia",
+ "grb": "Grebo",
+ "grc": "Ancient Greek (to 1453)",
+ "grd": "Guruntum-Mbaaru",
+ "grg": "Madi",
+ "grh": "Gbiri-Niragu",
+ "gri": "Ghari",
+ "grj": "Southern Grebo",
+ "grk": "Greek languages",
+ "grm": "Kota Marudu Talantang",
+ "gro": "Groma",
+ "grq": "Gorovu",
+ "grr": "Taznatit",
+ "grs": "Gresi",
+ "grt": "Garo",
+ "gru": "Kistane",
+ "grv": "Central Grebo",
+ "grw": "Gweda",
+ "grx": "Guriaso",
+ "gry": "Barclayville Grebo",
+ "grz": "Guramalum",
+ "gse": "Ghanaian Sign Language",
+ "gsg": "German Sign Language",
+ "gsl": "Gusilay",
+ "gsm": "Guatemalan Sign Language",
+ "gsn": "Nema; Gusan",
+ "gso": "Southwest Gbaya",
+ "gsp": "Wasembo",
+ "gss": "Greek Sign Language",
+ "gsw": "Swiss German; Alemannic; Alsatian",
+ "gta": "Guató",
+ "gtu": "Aghu-Tharnggala",
+ "gu": "Gujarati",
+ "gua": "Shiki",
+ "gub": "Guajajára",
+ "guc": "Wayuu",
+ "gud": "Yocoboué Dida",
+ "gue": "Gurindji",
+ "guf": "Gupapuyngu",
+ "gug": "Paraguayan Guaraní",
+ "guh": "Guahibo",
+ "gui": "Eastern Bolivian Guaraní",
+ "guk": "Gumuz",
+ "gul": "Sea Island Creole English",
+ "gum": "Guambiano",
+ "gun": "Mbyá Guaraní",
+ "guo": "Guayabero",
+ "gup": "Gunwinggu",
+ "guq": "Aché",
+ "gur": "Farefare",
+ "gus": "Guinean Sign Language",
+ "gut": "Maléku Jaíka",
+ "guu": "Yanomamö",
+ "guw": "Gun",
+ "gux": "Gourmanchéma",
+ "guz": "Gusii; Ekegusii",
+ "gv": "Manx",
+ "gva": "Guana (Paraguay)",
+ "gvc": "Guanano",
+ "gve": "Duwet",
+ "gvf": "Golin",
+ "gvj": "Guajá",
+ "gvl": "Gulay",
+ "gvm": "Gurmana",
+ "gvn": "Kuku-Yalanji",
+ "gvo": "Gavião Do Jiparaná",
+ "gvp": "Pará Gavião",
+ "gvr": "Gurung",
+ "gvs": "Gumawana",
+ "gvy": "Guyani",
+ "gwa": "Mbato",
+ "gwb": "Gwa",
+ "gwc": "Gawri; Kalami",
+ "gwd": "Gawwada",
+ "gwe": "Gweno",
+ "gwf": "Gowro",
+ "gwg": "Moo",
+ "gwi": "Gwichʼin",
+ "gwj": "ǀGwi",
+ "gwm": "Awngthim",
+ "gwn": "Gwandara",
+ "gwr": "Gwere",
+ "gwt": "Gawar-Bati",
+ "gwu": "Guwamu",
+ "gww": "Kwini",
+ "gwx": "Gua",
+ "gxx": "Wè Southern",
+ "gya": "Northwest Gbaya",
+ "gyb": "Garus",
+ "gyd": "Kayardild",
+ "gye": "Gyem",
+ "gyf": "Gungabula",
+ "gyg": "Gbayi",
+ "gyi": "Gyele",
+ "gyl": "Gayil",
+ "gym": "Ngäbere",
+ "gyn": "Guyanese Creole English",
+ "gyo": "Gyalsumdo",
+ "gyr": "Guarayu",
+ "gyy": "Gunya",
+ "gyz": "Geji; Gyaazi",
+ "gza": "Ganza",
+ "gzi": "Gazi",
+ "gzn": "Gane",
+ "ha": "Hausa",
+ "haa": "Han",
+ "hab": "Hanoi Sign Language",
+ "hac": "Gurani",
+ "had": "Hatam",
+ "hae": "Eastern Oromo",
+ "haf": "Haiphong Sign Language",
+ "hag": "Hanga",
+ "hah": "Hahon",
+ "hai": "Haida",
+ "haj": "Hajong",
+ "hak": "Hakka Chinese",
+ "hal": "Halang",
+ "ham": "Hewa",
+ "han": "Hangaza",
+ "hao": "Hakö",
+ "hap": "Hupla",
+ "haq": "Ha",
+ "har": "Harari",
+ "has": "Haisla",
+ "hav": "Havu",
+ "haw": "Hawaiian",
+ "hax": "Southern Haida",
+ "hay": "Haya",
+ "haz": "Hazaragi",
+ "hba": "Hamba",
+ "hbb": "Huba",
+ "hbn": "Heiban",
+ "hbo": "Ancient Hebrew",
+ "hbu": "Habu",
+ "hca": "Andaman Creole Hindi",
+ "hch": "Huichol",
+ "hdn": "Northern Haida",
+ "hds": "Honduras Sign Language",
+ "hdy": "Hadiyya",
+ "he": "Hebrew",
+ "hea": "Northern Qiandong Miao",
+ "hed": "Herdé",
+ "heg": "Helong",
+ "heh": "Hehe",
+ "hei": "Heiltsuk",
+ "hem": "Hemba",
+ "hgm": "Haiǁom",
+ "hgw": "Haigwai",
+ "hhi": "Hoia Hoia",
+ "hhr": "Kerak",
+ "hhy": "Hoyahoya",
+ "hi": "Hindi",
+ "hia": "Lamang",
+ "hib": "Hibito",
+ "hid": "Hidatsa",
+ "hif": "Fiji Hindi",
+ "hig": "Kamwe",
+ "hih": "Pamosu",
+ "hii": "Hinduri",
+ "hij": "Hijuk",
+ "hik": "Seit-Kaitetu",
+ "hil": "Hiligaynon",
+ "him": "Himachali languages; Western Pahari languages",
+ "hio": "Tsoa",
+ "hir": "Himarimã",
+ "hit": "Hittite",
+ "hiw": "Hiw",
+ "hix": "Hixkaryána",
+ "hji": "Haji",
+ "hka": "Kahe",
+ "hke": "Hunde",
+ "hkh": "Khah; Poguli",
+ "hkk": "Hunjara-Kaina Ke",
+ "hkn": "Mel-Khaonh",
+ "hks": "Hong Kong Sign Language; Heung Kong Sau Yue",
+ "hla": "Halia",
+ "hlb": "Halbi",
+ "hld": "Halang Doan",
+ "hle": "Hlersu",
+ "hlt": "Matu Chin",
+ "hlu": "Hieroglyphic Luwian",
+ "hma": "Southern Mashan Hmong; Southern Mashan Miao",
+ "hmb": "Humburi Senni Songhay",
+ "hmc": "Central Huishui Hmong; Central Huishui Miao",
+ "hmd": "Large Flowery Miao; A-hmaos; Da-Hua Miao",
+ "hme": "Eastern Huishui Hmong; Eastern Huishui Miao",
+ "hmf": "Hmong Don",
+ "hmg": "Southwestern Guiyang Hmong",
+ "hmh": "Southwestern Huishui Hmong; Southwestern Huishui Miao",
+ "hmi": "Northern Huishui Hmong; Northern Huishui Miao",
+ "hmj": "Ge; Gejia",
+ "hmk": "Maek",
+ "hml": "Luopohe Hmong; Luopohe Miao",
+ "hmm": "Central Mashan Hmong; Central Mashan Miao",
+ "hmn": "Hmong; Mong",
+ "hmp": "Northern Mashan Hmong; Northern Mashan Miao",
+ "hmq": "Eastern Qiandong Miao",
+ "hmr": "Hmar",
+ "hms": "Southern Qiandong Miao",
+ "hmt": "Hamtai",
+ "hmu": "Hamap",
+ "hmv": "Hmong Dô",
+ "hmw": "Western Mashan Hmong; Western Mashan Miao",
+ "hmx": "Hmong-Mien languages",
+ "hmy": "Southern Guiyang Hmong; Southern Guiyang Miao",
+ "hmz": "Hmong Shua; Sinicized Miao",
+ "hna": "Mina (Cameroon)",
+ "hnd": "Southern Hindko",
+ "hne": "Chhattisgarhi",
+ "hng": "Hungu",
+ "hnh": "ǁAni",
+ "hni": "Hani",
+ "hnj": "Hmong Njua; Mong Leng; Mong Njua",
+ "hnn": "Hanunoo",
+ "hno": "Northern Hindko",
+ "hns": "Caribbean Hindustani",
+ "hnu": "Hung",
+ "ho": "Hiri Motu",
+ "hoa": "Hoava",
+ "hob": "Mari (Madang Province)",
+ "hoc": "Ho",
+ "hod": "Holma",
+ "hoe": "Horom",
+ "hoh": "Hobyót",
+ "hoi": "Holikachuk",
+ "hoj": "Hadothi; Haroti",
+ "hok": "Hokan languages",
+ "hol": "Holu",
+ "hom": "Homa",
+ "hoo": "Holoholo",
+ "hop": "Hopi",
+ "hor": "Horo",
+ "hos": "Ho Chi Minh City Sign Language",
+ "hot": "Hote; Malê",
+ "hov": "Hovongan",
+ "how": "Honi",
+ "hoy": "Holiya",
+ "hoz": "Hozo",
+ "hpo": "Hpon",
+ "hps": "Hawai'i Sign Language (HSL); Hawai'i Pidgin Sign Language",
+ "hr": "Croatian",
+ "hra": "Hrangkhol",
+ "hrc": "Niwer Mil",
+ "hre": "Hre",
+ "hrk": "Haruku",
+ "hrm": "Horned Miao",
+ "hro": "Haroi",
+ "hrp": "Nhirrpi",
+ "hrt": "Hértevin",
+ "hru": "Hruso",
+ "hrw": "Warwar Feni",
+ "hrx": "Hunsrik",
+ "hrz": "Harzani",
+ "hsb": "Upper Sorbian",
+ "hsh": "Hungarian Sign Language",
+ "hsl": "Hausa Sign Language",
+ "hsn": "Xiang Chinese",
+ "hss": "Harsusi",
+ "ht": "Haitian; Haitian Creole",
+ "hti": "Hoti",
+ "hto": "Minica Huitoto",
+ "hts": "Hadza",
+ "htu": "Hitu",
+ "htx": "Middle Hittite",
+ "hu": "Hungarian",
+ "hub": "Huambisa",
+ "huc": "ǂHua; ǂʼAmkhoe",
+ "hud": "Huaulu",
+ "hue": "San Francisco Del Mar Huave",
+ "huf": "Humene",
+ "hug": "Huachipaeri",
+ "huh": "Huilliche",
+ "hui": "Huli",
+ "huj": "Northern Guiyang Hmong; Northern Guiyang Miao",
+ "huk": "Hulung",
+ "hul": "Hula",
+ "hum": "Hungana",
+ "huo": "Hu",
+ "hup": "Hupa",
+ "huq": "Tsat",
+ "hur": "Halkomelem",
+ "hus": "Huastec",
+ "hut": "Humla",
+ "huu": "Murui Huitoto",
+ "huv": "San Mateo Del Mar Huave",
+ "huw": "Hukumina",
+ "hux": "Nüpode Huitoto",
+ "huy": "Hulaulá",
+ "huz": "Hunzib",
+ "hvc": "Haitian Vodoun Culture Language",
+ "hve": "San Dionisio Del Mar Huave",
+ "hvk": "Haveke",
+ "hvn": "Sabu",
+ "hvv": "Santa María Del Mar Huave",
+ "hwa": "Wané",
+ "hwc": "Hawai'i Creole English; Hawai'i Pidgin",
+ "hwo": "Hwana",
+ "hy": "Armenian",
+ "hya": "Hya",
+ "hyw": "Western Armenian",
+ "hyx": "Armenian (family)",
+ "hz": "Herero",
+ "ia": "Interlingua (International Auxiliary Language Association)",
+ "iai": "Iaai",
+ "ian": "Iatmul",
+ "iar": "Purari",
+ "iba": "Iban",
+ "ibb": "Ibibio",
+ "ibd": "Iwaidja",
+ "ibe": "Akpes",
+ "ibg": "Ibanag",
+ "ibh": "Bih",
+ "ibl": "Ibaloi",
+ "ibm": "Agoi",
+ "ibn": "Ibino",
+ "ibr": "Ibuoro",
+ "ibu": "Ibu",
+ "iby": "Ibani",
+ "ica": "Ede Ica",
+ "ich": "Etkywan",
+ "icl": "Icelandic Sign Language",
+ "icr": "Islander Creole English",
+ "id": "Indonesian",
+ "ida": "Idakho-Isukha-Tiriki; Luidakho-Luisukha-Lutirichi",
+ "idb": "Indo-Portuguese",
+ "idc": "Idon; Ajiya",
+ "idd": "Ede Idaca",
+ "ide": "Idere",
+ "idi": "Idi",
+ "idr": "Indri",
+ "ids": "Idesa",
+ "idt": "Idaté",
+ "idu": "Idoma",
+ "ie": "Interlingue; Occidental",
+ "ifa": "Amganad Ifugao",
+ "ifb": "Batad Ifugao; Ayangan Ifugao",
+ "ife": "Ifè",
+ "iff": "Ifo",
+ "ifk": "Tuwali Ifugao",
+ "ifm": "Teke-Fuumu",
+ "ifu": "Mayoyao Ifugao",
+ "ify": "Keley-I Kallahan",
+ "ig": "Igbo",
+ "igb": "Ebira",
+ "ige": "Igede",
+ "igg": "Igana",
+ "igl": "Igala",
+ "igm": "Kanggape",
+ "ign": "Ignaciano",
+ "igo": "Isebe",
+ "igs": "Interglossa",
+ "igw": "Igwe",
+ "ihb": "Iha Based Pidgin",
+ "ihi": "Ihievbe",
+ "ihp": "Iha",
+ "ihw": "Bidhawal",
+ "ii": "Sichuan Yi; Nuosu",
+ "iin": "Thiin",
+ "iir": "Indo-Iranian languages",
+ "ijc": "Izon",
+ "ije": "Biseni",
+ "ijj": "Ede Ije",
+ "ijn": "Kalabari",
+ "ijo": "Ijo languages",
+ "ijs": "Southeast Ijo",
+ "ik": "Inupiaq",
+ "ike": "Eastern Canadian Inuktitut",
+ "iki": "Iko",
+ "ikk": "Ika",
+ "ikl": "Ikulu",
+ "iko": "Olulumo-Ikom",
+ "ikp": "Ikpeshi",
+ "ikr": "Ikaranggal",
+ "iks": "Inuit Sign Language",
+ "ikt": "Inuinnaqtun; Western Canadian Inuktitut",
+ "ikv": "Iku-Gora-Ankwa",
+ "ikw": "Ikwere",
+ "ikx": "Ik",
+ "ikz": "Ikizu",
+ "ila": "Ile Ape",
+ "ilb": "Ila",
+ "ilg": "Garig-Ilgar",
+ "ili": "Ili Turki",
+ "ilk": "Ilongot",
+ "ilm": "Iranun (Malaysia)",
+ "ilo": "Iloko",
+ "ilp": "Iranun (Philippines)",
+ "ils": "International Sign",
+ "ilu": "Ili'uun",
+ "ilv": "Ilue",
+ "ima": "Mala Malasar",
+ "imi": "Anamgura",
+ "iml": "Miluk",
+ "imn": "Imonda",
+ "imo": "Imbongu",
+ "imr": "Imroing",
+ "ims": "Marsian",
+ "imt": "Imotong",
+ "imy": "Milyan",
+ "inb": "Inga",
+ "inc": "Indic languages",
+ "ine": "Indo-European languages",
+ "ing": "Degexit'an",
+ "inh": "Ingush",
+ "inj": "Jungle Inga",
+ "inl": "Indonesian Sign Language",
+ "inm": "Minaean",
+ "inn": "Isinai",
+ "ino": "Inoke-Yate",
+ "inp": "Iñapari",
+ "ins": "Indian Sign Language",
+ "int": "Intha",
+ "inz": "Ineseño",
+ "io": "Ido",
+ "ior": "Inor",
+ "iou": "Tuma-Irumu",
+ "iow": "Iowa-Oto",
+ "ipi": "Ipili",
+ "ipo": "Ipiko",
+ "iqu": "Iquito",
+ "iqw": "Ikwo",
+ "ira": "Iranian languages",
+ "ire": "Iresim",
+ "irh": "Irarutu",
+ "iri": "Rigwe; Irigwe",
+ "irk": "Iraqw",
+ "irn": "Irántxe",
+ "iro": "Iroquoian languages",
+ "irr": "Ir",
+ "iru": "Irula",
+ "irx": "Kamberau",
+ "iry": "Iraya",
+ "is": "Icelandic",
+ "isa": "Isabi",
+ "isc": "Isconahua",
+ "isd": "Isnag",
+ "ise": "Italian Sign Language",
+ "isg": "Irish Sign Language",
+ "ish": "Esan",
+ "isi": "Nkem-Nkum",
+ "isk": "Ishkashimi",
+ "ism": "Masimasi",
+ "isn": "Isanzu",
+ "iso": "Isoko",
+ "isr": "Israeli Sign Language",
+ "ist": "Istriot",
+ "isu": "Isu (Menchum Division)",
+ "it": "Italian",
+ "itb": "Binongan Itneg",
+ "itc": "Italic languages",
+ "itd": "Southern Tidung",
+ "ite": "Itene",
+ "iti": "Inlaod Itneg",
+ "itk": "Judeo-Italian",
+ "itl": "Itelmen",
+ "itm": "Itu Mbon Uzo",
+ "ito": "Itonama",
+ "itr": "Iteri",
+ "its": "Isekiri",
+ "itt": "Maeng Itneg",
+ "itv": "Itawit",
+ "itw": "Ito",
+ "itx": "Itik",
+ "ity": "Moyadan Itneg",
+ "itz": "Itzá",
+ "iu": "Inuktitut",
+ "ium": "Iu Mien",
+ "ivb": "Ibatan",
+ "ivv": "Ivatan",
+ "iwk": "I-Wak",
+ "iwm": "Iwam",
+ "iwo": "Iwur",
+ "iws": "Sepik Iwam",
+ "ixc": "Ixcatec",
+ "ixl": "Ixil",
+ "iya": "Iyayu",
+ "iyo": "Mesaka",
+ "iyx": "Yaka (Congo)",
+ "izh": "Ingrian",
+ "izr": "Izere",
+ "izz": "Izii",
+ "ja": "Japanese",
+ "jaa": "Jamamadí",
+ "jab": "Hyam",
+ "jac": "Popti'; Jakalteko",
+ "jad": "Jahanka",
+ "jae": "Yabem",
+ "jaf": "Jara",
+ "jah": "Jah Hut",
+ "jaj": "Zazao",
+ "jak": "Jakun",
+ "jal": "Yalahatan",
+ "jam": "Jamaican Creole English",
+ "jan": "Jandai",
+ "jao": "Yanyuwa",
+ "jaq": "Yaqay",
+ "jas": "New Caledonian Javanese",
+ "jat": "Jakati",
+ "jau": "Yaur",
+ "jax": "Jambi Malay",
+ "jay": "Yan-nhangu; Nhangu",
+ "jaz": "Jawe",
+ "jbe": "Judeo-Berber",
+ "jbi": "Badjiri",
+ "jbj": "Arandai",
+ "jbk": "Barikewa",
+ "jbm": "Bijim",
+ "jbn": "Nafusi",
+ "jbo": "Lojban",
+ "jbr": "Jofotek-Bromnya",
+ "jbt": "Jabutí",
+ "jbu": "Jukun Takum",
+ "jbw": "Yawijibaya",
+ "jcs": "Jamaican Country Sign Language",
+ "jct": "Krymchak",
+ "jda": "Jad",
+ "jdg": "Jadgali",
+ "jdt": "Judeo-Tat",
+ "jeb": "Jebero",
+ "jee": "Jerung",
+ "jeh": "Jeh",
+ "jei": "Yei",
+ "jek": "Jeri Kuo",
+ "jel": "Yelmek",
+ "jen": "Dza",
+ "jer": "Jere",
+ "jet": "Manem",
+ "jeu": "Jonkor Bourmataguil",
+ "jgb": "Ngbee",
+ "jge": "Judeo-Georgian",
+ "jgk": "Gwak",
+ "jgo": "Ngomba",
+ "jhi": "Jehai",
+ "jhs": "Jhankot Sign Language",
+ "jia": "Jina",
+ "jib": "Jibu",
+ "jic": "Tol",
+ "jid": "Bu (Kaduna State)",
+ "jie": "Jilbe",
+ "jig": "Jingulu; Djingili",
+ "jih": "sTodsde; Shangzhai",
+ "jii": "Jiiddu",
+ "jil": "Jilim",
+ "jim": "Jimi (Cameroon)",
+ "jio": "Jiamao",
+ "jiq": "Guanyinqiao; Lavrung",
+ "jit": "Jita",
+ "jiu": "Youle Jinuo",
+ "jiv": "Shuar",
+ "jiy": "Buyuan Jinuo",
+ "jje": "Jejueo",
+ "jjr": "Bankal",
+ "jka": "Kaera",
+ "jkm": "Mobwa Karen",
+ "jko": "Kubo",
+ "jkp": "Paku Karen",
+ "jkr": "Koro (India)",
+ "jks": "Amami Koniya Sign Language",
+ "jku": "Labir",
+ "jle": "Ngile",
+ "jls": "Jamaican Sign Language",
+ "jma": "Dima",
+ "jmb": "Zumbun",
+ "jmc": "Machame",
+ "jmd": "Yamdena",
+ "jmi": "Jimi (Nigeria)",
+ "jml": "Jumli",
+ "jmn": "Makuri Naga",
+ "jmr": "Kamara",
+ "jms": "Mashi (Nigeria)",
+ "jmw": "Mouwase",
+ "jmx": "Western Juxtlahuaca Mixtec",
+ "jna": "Jangshung",
+ "jnd": "Jandavra",
+ "jng": "Yangman",
+ "jni": "Janji",
+ "jnj": "Yemsa",
+ "jnl": "Rawat",
+ "jns": "Jaunsari",
+ "job": "Joba",
+ "jod": "Wojenaka",
+ "jog": "Jogi",
+ "jor": "Jorá",
+ "jos": "Jordanian Sign Language",
+ "jow": "Jowulu",
+ "jpa": "Jewish Palestinian Aramaic",
+ "jpr": "Judeo-Persian",
+ "jpx": "Japanese (family)",
+ "jqr": "Jaqaru",
+ "jra": "Jarai",
+ "jrb": "Judeo-Arabic",
+ "jrr": "Jiru",
+ "jrt": "Jakattoe",
+ "jru": "Japrería",
+ "jsl": "Japanese Sign Language",
+ "jua": "Júma",
+ "jub": "Wannu",
+ "juc": "Jurchen",
+ "jud": "Worodougou",
+ "juh": "Hõne",
+ "jui": "Ngadjuri",
+ "juk": "Wapan",
+ "jul": "Jirel",
+ "jum": "Jumjum",
+ "jun": "Juang",
+ "juo": "Jiba",
+ "jup": "Hupdë",
+ "jur": "Jurúna",
+ "jus": "Jumla Sign Language",
+ "jut": "Jutish",
+ "juu": "Ju",
+ "juw": "Wãpha",
+ "juy": "Juray",
+ "jv": "Javanese",
+ "jvd": "Javindo",
+ "jvn": "Caribbean Javanese",
+ "jwi": "Jwira-Pepesa",
+ "jya": "Jiarong",
+ "jye": "Judeo-Yemeni Arabic",
+ "jyy": "Jaya",
+ "ka": "Georgian",
+ "kaa": "Kara-Kalpak; Karakalpak",
+ "kab": "Kabyle",
+ "kac": "Kachin; Jingpho",
+ "kad": "Adara",
+ "kae": "Ketangalan",
+ "kaf": "Katso",
+ "kag": "Kajaman",
+ "kah": "Kara (Central African Republic)",
+ "kai": "Karekare",
+ "kaj": "Jju",
+ "kak": "Kalanguya; Kayapa Kallahan",
+ "kam": "Kamba (Kenya)",
+ "kao": "Xaasongaxango",
+ "kap": "Bezhta",
+ "kaq": "Capanahua",
+ "kar": "Karen languages",
+ "kav": "Katukína",
+ "kaw": "Kawi",
+ "kax": "Kao",
+ "kay": "Kamayurá",
+ "kba": "Kalarko",
+ "kbb": "Kaxuiâna",
+ "kbc": "Kadiwéu",
+ "kbd": "Kabardian",
+ "kbe": "Kanju",
+ "kbg": "Khamba",
+ "kbh": "Camsá",
+ "kbi": "Kaptiau",
+ "kbj": "Kari",
+ "kbk": "Grass Koiari",
+ "kbl": "Kanembu",
+ "kbm": "Iwal",
+ "kbn": "Kare (Central African Republic)",
+ "kbo": "Keliko",
+ "kbp": "Kabiyè",
+ "kbq": "Kamano",
+ "kbr": "Kafa",
+ "kbs": "Kande",
+ "kbt": "Abadi",
+ "kbu": "Kabutra",
+ "kbv": "Dera (Indonesia)",
+ "kbw": "Kaiep",
+ "kbx": "Ap Ma",
+ "kby": "Manga Kanuri",
+ "kbz": "Duhwa",
+ "kca": "Khanty",
+ "kcb": "Kawacha",
+ "kcc": "Lubila",
+ "kcd": "Ngkâlmpw Kanum",
+ "kce": "Kaivi",
+ "kcf": "Ukaan",
+ "kcg": "Tyap",
+ "kch": "Vono",
+ "kci": "Kamantan",
+ "kcj": "Kobiana",
+ "kck": "Kalanga",
+ "kcl": "Kela (Papua New Guinea); Kala",
+ "kcm": "Gula (Central African Republic)",
+ "kcn": "Nubi",
+ "kco": "Kinalakna",
+ "kcp": "Kanga",
+ "kcq": "Kamo",
+ "kcr": "Katla",
+ "kcs": "Koenoem",
+ "kct": "Kaian",
+ "kcu": "Kami (Tanzania)",
+ "kcv": "Kete",
+ "kcw": "Kabwari",
+ "kcx": "Kachama-Ganjule",
+ "kcy": "Korandje",
+ "kcz": "Konongo",
+ "kda": "Worimi",
+ "kdc": "Kutu",
+ "kdd": "Yankunytjatjara",
+ "kde": "Makonde",
+ "kdf": "Mamusi",
+ "kdg": "Seba",
+ "kdh": "Tem",
+ "kdi": "Kumam",
+ "kdj": "Karamojong",
+ "kdk": "Numèè; Kwényi",
+ "kdl": "Tsikimba",
+ "kdm": "Kagoma",
+ "kdn": "Kunda",
+ "kdo": "Kordofanian languages",
+ "kdp": "Kaningdon-Nindem",
+ "kdq": "Koch",
+ "kdr": "Karaim",
+ "kdt": "Kuy",
+ "kdu": "Kadaru",
+ "kdw": "Koneraw",
+ "kdx": "Kam",
+ "kdy": "Keder; Keijar",
+ "kdz": "Kwaja",
+ "kea": "Kabuverdianu",
+ "keb": "Kélé",
+ "kec": "Keiga",
+ "ked": "Kerewe",
+ "kee": "Eastern Keres",
+ "kef": "Kpessi",
+ "keg": "Tese",
+ "keh": "Keak",
+ "kei": "Kei",
+ "kej": "Kadar",
+ "kek": "Kekchí",
+ "kel": "Kela (Democratic Republic of Congo)",
+ "kem": "Kemak",
+ "ken": "Kenyang",
+ "keo": "Kakwa",
+ "kep": "Kaikadi",
+ "keq": "Kamar",
+ "ker": "Kera",
+ "kes": "Kugbo",
+ "ket": "Ket",
+ "keu": "Akebu",
+ "kev": "Kanikkaran",
+ "kew": "West Kewa",
+ "kex": "Kukna",
+ "key": "Kupia",
+ "kez": "Kukele",
+ "kfa": "Kodava",
+ "kfb": "Northwestern Kolami",
+ "kfc": "Konda-Dora",
+ "kfd": "Korra Koraga",
+ "kfe": "Kota (India)",
+ "kff": "Koya",
+ "kfg": "Kudiya",
+ "kfh": "Kurichiya",
+ "kfi": "Kannada Kurumba",
+ "kfj": "Kemiehua",
+ "kfk": "Kinnauri",
+ "kfl": "Kung",
+ "kfm": "Khunsari",
+ "kfn": "Kuk",
+ "kfo": "Koro (Côte d'Ivoire)",
+ "kfp": "Korwa",
+ "kfq": "Korku",
+ "kfr": "Kachhi; Kutchi",
+ "kfs": "Bilaspuri",
+ "kft": "Kanjari",
+ "kfu": "Katkari",
+ "kfv": "Kurmukar",
+ "kfw": "Kharam Naga",
+ "kfx": "Kullu Pahari",
+ "kfy": "Kumaoni",
+ "kfz": "Koromfé",
+ "kg": "Kongo",
+ "kga": "Koyaga",
+ "kgb": "Kawe",
+ "kge": "Komering",
+ "kgf": "Kube",
+ "kgg": "Kusunda",
+ "kgi": "Selangor Sign Language",
+ "kgj": "Gamale Kham",
+ "kgk": "Kaiwá",
+ "kgl": "Kunggari",
+ "kgm": "Karipúna",
+ "kgn": "Karingani",
+ "kgo": "Krongo",
+ "kgp": "Kaingang",
+ "kgq": "Kamoro",
+ "kgr": "Abun",
+ "kgs": "Kumbainggar",
+ "kgt": "Somyev",
+ "kgu": "Kobol",
+ "kgv": "Karas",
+ "kgw": "Karon Dori",
+ "kgx": "Kamaru",
+ "kgy": "Kyerung",
+ "kha": "Khasi",
+ "khb": "Lü",
+ "khc": "Tukang Besi North",
+ "khd": "Bädi Kanum",
+ "khe": "Korowai",
+ "khf": "Khuen",
+ "khg": "Khams Tibetan",
+ "khh": "Kehu",
+ "khi": "Khoisan languages",
+ "khj": "Kuturmi",
+ "khk": "Halh Mongolian",
+ "khl": "Lusi",
+ "khn": "Khandesi",
+ "kho": "Khotanese; Sakan",
+ "khp": "Kapori; Kapauri",
+ "khq": "Koyra Chiini Songhay",
+ "khr": "Kharia",
+ "khs": "Kasua",
+ "kht": "Khamti",
+ "khu": "Nkhumbi",
+ "khv": "Khvarshi",
+ "khw": "Khowar",
+ "khx": "Kanu",
+ "khy": "Kele (Democratic Republic of Congo)",
+ "khz": "Keapara",
+ "ki": "Kikuyu; Gikuyu",
+ "kia": "Kim",
+ "kib": "Koalib",
+ "kic": "Kickapoo",
+ "kid": "Koshin",
+ "kie": "Kibet",
+ "kif": "Eastern Parbate Kham",
+ "kig": "Kimaama; Kimaghima",
+ "kih": "Kilmeri",
+ "kii": "Kitsai",
+ "kij": "Kilivila",
+ "kil": "Kariya",
+ "kim": "Karagas",
+ "kio": "Kiowa",
+ "kip": "Sheshi Kham",
+ "kiq": "Kosadle; Kosare",
+ "kis": "Kis",
+ "kit": "Agob",
+ "kiu": "Kirmanjki (individual language)",
+ "kiv": "Kimbu",
+ "kiw": "Northeast Kiwai",
+ "kix": "Khiamniungan Naga",
+ "kiy": "Kirikiri",
+ "kiz": "Kisi",
+ "kj": "Kuanyama; Kwanyama",
+ "kja": "Mlap",
+ "kjb": "Q'anjob'al; Kanjobal",
+ "kjc": "Coastal Konjo",
+ "kjd": "Southern Kiwai",
+ "kje": "Kisar",
+ "kjg": "Khmu",
+ "kjh": "Khakas",
+ "kji": "Zabana",
+ "kjj": "Khinalugh",
+ "kjk": "Highland Konjo",
+ "kjl": "Western Parbate Kham",
+ "kjm": "Kháng",
+ "kjn": "Kunjen",
+ "kjo": "Harijan Kinnauri",
+ "kjp": "Pwo Eastern Karen",
+ "kjq": "Western Keres",
+ "kjr": "Kurudu",
+ "kjs": "East Kewa",
+ "kjt": "Phrae Pwo Karen",
+ "kju": "Kashaya",
+ "kjv": "Kaikavian Literary Language",
+ "kjx": "Ramopa",
+ "kjy": "Erave",
+ "kjz": "Bumthangkha",
+ "kk": "Kazakh",
+ "kka": "Kakanda",
+ "kkb": "Kwerisa",
+ "kkc": "Odoodee",
+ "kkd": "Kinuku",
+ "kke": "Kakabe",
+ "kkf": "Kalaktang Monpa",
+ "kkg": "Mabaka Valley Kalinga",
+ "kkh": "Khün",
+ "kki": "Kagulu",
+ "kkj": "Kako",
+ "kkk": "Kokota",
+ "kkl": "Kosarek Yale",
+ "kkm": "Kiong",
+ "kkn": "Kon Keu",
+ "kko": "Karko",
+ "kkp": "Gugubera; Koko-Bera",
+ "kkq": "Kaeku",
+ "kkr": "Kir-Balar",
+ "kks": "Giiwo",
+ "kkt": "Koi",
+ "kku": "Tumi",
+ "kkv": "Kangean",
+ "kkw": "Teke-Kukuya",
+ "kkx": "Kohin",
+ "kky": "Guugu Yimidhirr; Guguyimidjir",
+ "kkz": "Kaska",
+ "kl": "Kalaallisut; Greenlandic",
+ "kla": "Klamath-Modoc",
+ "klb": "Kiliwa",
+ "klc": "Kolbila",
+ "kld": "Gamilaraay",
+ "kle": "Kulung (Nepal)",
+ "klf": "Kendeje",
+ "klg": "Tagakaulo",
+ "klh": "Weliki",
+ "kli": "Kalumpang",
+ "klj": "Khalaj",
+ "klk": "Kono (Nigeria)",
+ "kll": "Kagan Kalagan",
+ "klm": "Migum",
+ "kln": "Kalenjin",
+ "klo": "Kapya",
+ "klp": "Kamasa",
+ "klq": "Rumu",
+ "klr": "Khaling",
+ "kls": "Kalasha",
+ "klt": "Nukna",
+ "klu": "Klao",
+ "klv": "Maskelynes",
+ "klw": "Tado; Lindu",
+ "klx": "Koluwawa",
+ "kly": "Kalao",
+ "klz": "Kabola",
+ "km": "Khmer; Central Khmer",
+ "kma": "Konni",
+ "kmb": "Kimbundu",
+ "kmc": "Southern Dong",
+ "kmd": "Majukayang Kalinga",
+ "kme": "Bakole",
+ "kmf": "Kare (Papua New Guinea)",
+ "kmg": "Kâte",
+ "kmh": "Kalam",
+ "kmi": "Kami (Nigeria)",
+ "kmj": "Kumarbhag Paharia",
+ "kmk": "Limos Kalinga",
+ "kml": "Tanudan Kalinga",
+ "kmm": "Kom (India)",
+ "kmn": "Awtuw",
+ "kmo": "Kwoma",
+ "kmp": "Gimme",
+ "kmq": "Kwama",
+ "kmr": "Northern Kurdish",
+ "kms": "Kamasau",
+ "kmt": "Kemtuik",
+ "kmu": "Kanite",
+ "kmv": "Karipúna Creole French",
+ "kmw": "Komo (Democratic Republic of Congo)",
+ "kmx": "Waboda",
+ "kmy": "Koma",
+ "kmz": "Khorasani Turkish",
+ "kn": "Kannada",
+ "kna": "Dera (Nigeria)",
+ "knb": "Lubuagan Kalinga",
+ "knc": "Central Kanuri",
+ "knd": "Konda",
+ "kne": "Kankanaey",
+ "knf": "Mankanya",
+ "kng": "Koongo",
+ "kni": "Kanufi",
+ "knj": "Western Kanjobal",
+ "knk": "Kuranko",
+ "knl": "Keninjal",
+ "knm": "Kanamarí",
+ "knn": "Konkani (individual language)",
+ "kno": "Kono (Sierra Leone)",
+ "knp": "Kwanja",
+ "knq": "Kintaq",
+ "knr": "Kaningra",
+ "kns": "Kensiu",
+ "knt": "Panoan Katukína",
+ "knu": "Kono (Guinea)",
+ "knv": "Tabo",
+ "knw": "Kung-Ekoka",
+ "knx": "Kendayan; Salako",
+ "kny": "Kanyok",
+ "knz": "Kalamsé",
+ "ko": "Korean",
+ "koa": "Konomala",
+ "koc": "Kpati",
+ "kod": "Kodi",
+ "koe": "Kacipo-Bale Suri",
+ "kof": "Kubi",
+ "kog": "Cogui; Kogi",
+ "koh": "Koyo",
+ "koi": "Komi-Permyak",
+ "kok": "Konkani (macrolanguage)",
+ "kol": "Kol (Papua New Guinea)",
+ "koo": "Konzo",
+ "kop": "Waube",
+ "koq": "Kota (Gabon)",
+ "kos": "Kosraean",
+ "kot": "Lagwan",
+ "kou": "Koke",
+ "kov": "Kudu-Camo",
+ "kow": "Kugama",
+ "koy": "Koyukon",
+ "koz": "Korak",
+ "kpa": "Kutto",
+ "kpb": "Mullu Kurumba",
+ "kpc": "Curripaco",
+ "kpd": "Koba",
+ "kpe": "Kpelle",
+ "kpf": "Komba",
+ "kpg": "Kapingamarangi",
+ "kph": "Kplang",
+ "kpi": "Kofei",
+ "kpj": "Karajá",
+ "kpk": "Kpan",
+ "kpl": "Kpala",
+ "kpm": "Koho",
+ "kpn": "Kepkiriwát",
+ "kpo": "Ikposo",
+ "kpq": "Korupun-Sela",
+ "kpr": "Korafe-Yegha",
+ "kps": "Tehit",
+ "kpt": "Karata",
+ "kpu": "Kafoa",
+ "kpv": "Komi-Zyrian",
+ "kpw": "Kobon",
+ "kpx": "Mountain Koiali",
+ "kpy": "Koryak",
+ "kpz": "Kupsabiny",
+ "kqa": "Mum",
+ "kqb": "Kovai",
+ "kqc": "Doromu-Koki",
+ "kqd": "Koy Sanjaq Surat",
+ "kqe": "Kalagan",
+ "kqf": "Kakabai",
+ "kqg": "Khe",
+ "kqh": "Kisankasa",
+ "kqi": "Koitabu",
+ "kqj": "Koromira",
+ "kqk": "Kotafon Gbe",
+ "kql": "Kyenele",
+ "kqm": "Khisa",
+ "kqn": "Kaonde",
+ "kqo": "Eastern Krahn",
+ "kqp": "Kimré",
+ "kqq": "Krenak",
+ "kqr": "Kimaragang",
+ "kqs": "Northern Kissi",
+ "kqt": "Klias River Kadazan",
+ "kqu": "Seroa",
+ "kqv": "Okolod",
+ "kqw": "Kandas",
+ "kqx": "Mser",
+ "kqy": "Koorete",
+ "kqz": "Korana",
+ "kr": "Kanuri",
+ "kra": "Kumhali",
+ "krb": "Karkin",
+ "krc": "Karachay-Balkar",
+ "krd": "Kairui-Midiki",
+ "kre": "Panará",
+ "krf": "Koro (Vanuatu)",
+ "krh": "Kurama",
+ "kri": "Krio",
+ "krj": "Kinaray-A",
+ "krk": "Kerek",
+ "krl": "Karelian",
+ "krn": "Sapo",
+ "kro": "Kru languages",
+ "krp": "Korop",
+ "krr": "Krung",
+ "krs": "Gbaya (Sudan)",
+ "krt": "Tumari Kanuri",
+ "kru": "Kurukh",
+ "krv": "Kavet",
+ "krw": "Western Krahn",
+ "krx": "Karon",
+ "kry": "Kryts",
+ "krz": "Sota Kanum",
+ "ks": "Kashmiri",
+ "ksa": "Shuwa-Zamani",
+ "ksb": "Shambala",
+ "ksc": "Southern Kalinga",
+ "ksd": "Kuanua",
+ "kse": "Kuni",
+ "ksf": "Bafia",
+ "ksg": "Kusaghe",
+ "ksh": "Kölsch",
+ "ksi": "Krisa; I'saka",
+ "ksj": "Uare",
+ "ksk": "Kansa",
+ "ksl": "Kumalu",
+ "ksm": "Kumba",
+ "ksn": "Kasiguranin",
+ "kso": "Kofa",
+ "ksp": "Kaba",
+ "ksq": "Kwaami",
+ "ksr": "Borong",
+ "kss": "Southern Kisi",
+ "kst": "Winyé",
+ "ksu": "Khamyang",
+ "ksv": "Kusu",
+ "ksw": "S'gaw Karen",
+ "ksx": "Kedang",
+ "ksy": "Kharia Thar",
+ "ksz": "Kodaku",
+ "kta": "Katua",
+ "ktb": "Kambaata",
+ "ktc": "Kholok",
+ "ktd": "Kokata; Kukatha",
+ "kte": "Nubri",
+ "ktf": "Kwami",
+ "ktg": "Kalkutung",
+ "kth": "Karanga",
+ "kti": "North Muyu",
+ "ktj": "Plapo Krumen",
+ "ktk": "Kaniet",
+ "ktl": "Koroshi",
+ "ktm": "Kurti",
+ "ktn": "Karitiâna",
+ "kto": "Kuot",
+ "ktp": "Kaduo",
+ "ktq": "Katabaga",
+ "kts": "South Muyu",
+ "ktt": "Ketum",
+ "ktu": "Kituba (Democratic Republic of Congo)",
+ "ktv": "Eastern Katu",
+ "ktw": "Kato",
+ "ktx": "Kaxararí",
+ "kty": "Kango (Bas-Uélé District)",
+ "ktz": "Juǀʼhoan; Juǀʼhoansi",
+ "ku": "Kurdish",
+ "kub": "Kutep",
+ "kuc": "Kwinsu",
+ "kud": "'Auhelawa",
+ "kue": "Kuman (Papua New Guinea)",
+ "kuf": "Western Katu",
+ "kug": "Kupa",
+ "kuh": "Kushi",
+ "kui": "Kuikúro-Kalapálo; Kalapalo",
+ "kuj": "Kuria",
+ "kuk": "Kepo'",
+ "kul": "Kulere",
+ "kum": "Kumyk",
+ "kun": "Kunama",
+ "kuo": "Kumukio",
+ "kup": "Kunimaipa",
+ "kuq": "Karipuna",
+ "kus": "Kusaal",
+ "kut": "Kutenai",
+ "kuu": "Upper Kuskokwim",
+ "kuv": "Kur",
+ "kuw": "Kpagua",
+ "kux": "Kukatja",
+ "kuy": "Kuuku-Ya'u",
+ "kuz": "Kunza",
+ "kv": "Komi",
+ "kva": "Bagvalal",
+ "kvb": "Kubu",
+ "kvc": "Kove",
+ "kvd": "Kui (Indonesia)",
+ "kve": "Kalabakan",
+ "kvf": "Kabalai",
+ "kvg": "Kuni-Boazi",
+ "kvh": "Komodo",
+ "kvi": "Kwang",
+ "kvj": "Psikye",
+ "kvk": "Korean Sign Language",
+ "kvl": "Kayaw",
+ "kvm": "Kendem",
+ "kvn": "Border Kuna",
+ "kvo": "Dobel",
+ "kvp": "Kompane",
+ "kvq": "Geba Karen",
+ "kvr": "Kerinci",
+ "kvt": "Lahta Karen; Lahta",
+ "kvu": "Yinbaw Karen",
+ "kvv": "Kola",
+ "kvw": "Wersing",
+ "kvx": "Parkari Koli",
+ "kvy": "Yintale Karen; Yintale",
+ "kvz": "Tsakwambo; Tsaukambo",
+ "kw": "Cornish",
+ "kwa": "Dâw",
+ "kwb": "Kwa",
+ "kwc": "Likwala",
+ "kwd": "Kwaio",
+ "kwe": "Kwerba",
+ "kwf": "Kwara'ae",
+ "kwg": "Sara Kaba Deme",
+ "kwh": "Kowiai",
+ "kwi": "Awa-Cuaiquer",
+ "kwj": "Kwanga",
+ "kwk": "Kwakiutl",
+ "kwl": "Kofyar",
+ "kwm": "Kwambi",
+ "kwn": "Kwangali",
+ "kwo": "Kwomtari",
+ "kwp": "Kodia",
+ "kwr": "Kwer",
+ "kws": "Kwese",
+ "kwt": "Kwesten",
+ "kwu": "Kwakum",
+ "kwv": "Sara Kaba Náà",
+ "kww": "Kwinti",
+ "kwx": "Khirwar",
+ "kwy": "San Salvador Kongo",
+ "kwz": "Kwadi",
+ "kxa": "Kairiru",
+ "kxb": "Krobu",
+ "kxc": "Konso; Khonso",
+ "kxd": "Brunei",
+ "kxf": "Manumanaw Karen; Manumanaw",
+ "kxh": "Karo (Ethiopia)",
+ "kxi": "Keningau Murut",
+ "kxj": "Kulfa",
+ "kxk": "Zayein Karen",
+ "kxm": "Northern Khmer",
+ "kxn": "Kanowit-Tanjong Melanau",
+ "kxo": "Kanoé",
+ "kxp": "Wadiyara Koli",
+ "kxq": "Smärky Kanum",
+ "kxr": "Koro (Papua New Guinea)",
+ "kxs": "Kangjia",
+ "kxt": "Koiwat",
+ "kxv": "Kuvi",
+ "kxw": "Konai",
+ "kxx": "Likuba",
+ "kxy": "Kayong",
+ "kxz": "Kerewo",
+ "ky": "Kirghiz; Kyrgyz",
+ "kya": "Kwaya",
+ "kyb": "Butbut Kalinga",
+ "kyc": "Kyaka",
+ "kyd": "Karey",
+ "kye": "Krache",
+ "kyf": "Kouya",
+ "kyg": "Keyagana",
+ "kyh": "Karok",
+ "kyi": "Kiput",
+ "kyj": "Karao",
+ "kyk": "Kamayo",
+ "kyl": "Kalapuya",
+ "kym": "Kpatili",
+ "kyn": "Northern Binukidnon",
+ "kyo": "Kelon",
+ "kyp": "Kang",
+ "kyq": "Kenga",
+ "kyr": "Kuruáya",
+ "kys": "Baram Kayan",
+ "kyt": "Kayagar",
+ "kyu": "Western Kayah",
+ "kyv": "Kayort",
+ "kyw": "Kudmali",
+ "kyx": "Rapoisi",
+ "kyy": "Kambaira",
+ "kyz": "Kayabí",
+ "kza": "Western Karaboro",
+ "kzb": "Kaibobo",
+ "kzc": "Bondoukou Kulango",
+ "kzd": "Kadai",
+ "kze": "Kosena",
+ "kzf": "Da'a Kaili",
+ "kzg": "Kikai",
+ "kzi": "Kelabit",
+ "kzk": "Kazukuru",
+ "kzl": "Kayeli",
+ "kzm": "Kais",
+ "kzn": "Kokola",
+ "kzo": "Kaningi",
+ "kzp": "Kaidipang",
+ "kzq": "Kaike",
+ "kzr": "Karang",
+ "kzs": "Sugut Dusun",
+ "kzu": "Kayupulau",
+ "kzv": "Komyandaret",
+ "kzw": "Karirí-Xocó",
+ "kzx": "Kamarian",
+ "kzy": "Kango (Tshopo District)",
+ "kzz": "Kalabra",
+ "la": "Latin",
+ "laa": "Southern Subanen",
+ "lab": "Linear A",
+ "lac": "Lacandon",
+ "lad": "Ladino",
+ "lae": "Pattani",
+ "laf": "Lafofa",
+ "lag": "Langi",
+ "lah": "Lahnda",
+ "lai": "Lambya",
+ "laj": "Lango (Uganda)",
+ "lal": "Lalia",
+ "lam": "Lamba",
+ "lan": "Laru",
+ "lap": "Laka (Chad)",
+ "laq": "Qabiao",
+ "lar": "Larteh",
+ "las": "Lama (Togo)",
+ "lau": "Laba",
+ "law": "Lauje",
+ "lax": "Tiwa",
+ "lay": "Lama Bai",
+ "laz": "Aribwatsa",
+ "lb": "Luxembourgish; Letzeburgesch",
+ "lbb": "Label",
+ "lbc": "Lakkia",
+ "lbe": "Lak",
+ "lbf": "Tinani",
+ "lbg": "Laopang",
+ "lbi": "La'bi",
+ "lbj": "Ladakhi",
+ "lbk": "Central Bontok",
+ "lbl": "Libon Bikol",
+ "lbm": "Lodhi",
+ "lbn": "Rmeet",
+ "lbo": "Laven",
+ "lbq": "Wampar",
+ "lbr": "Lohorung",
+ "lbs": "Libyan Sign Language",
+ "lbt": "Lachi",
+ "lbu": "Labu",
+ "lbv": "Lavatbura-Lamusong",
+ "lbw": "Tolaki",
+ "lbx": "Lawangan",
+ "lby": "Lamalama; Lamu-Lamu",
+ "lbz": "Lardil",
+ "lcc": "Legenyem",
+ "lcd": "Lola",
+ "lce": "Loncong; Sekak",
+ "lcf": "Lubu",
+ "lch": "Luchazi",
+ "lcl": "Lisela",
+ "lcm": "Tungag",
+ "lcp": "Western Lawa",
+ "lcq": "Luhu",
+ "lcs": "Lisabata-Nuniali",
+ "lda": "Kla-Dan",
+ "ldb": "Dũya",
+ "ldd": "Luri",
+ "ldg": "Lenyima",
+ "ldh": "Lamja-Dengsa-Tola",
+ "ldi": "Laari",
+ "ldj": "Lemoro",
+ "ldk": "Leelau",
+ "ldl": "Kaan",
+ "ldm": "Landoma",
+ "ldn": "Láadan",
+ "ldo": "Loo",
+ "ldp": "Tso",
+ "ldq": "Lufu",
+ "lea": "Lega-Shabunda",
+ "leb": "Lala-Bisa",
+ "lec": "Leco",
+ "led": "Lendu",
+ "lee": "Lyélé",
+ "lef": "Lelemi",
+ "leh": "Lenje",
+ "lei": "Lemio",
+ "lej": "Lengola",
+ "lek": "Leipon",
+ "lel": "Lele (Democratic Republic of Congo)",
+ "lem": "Nomaande",
+ "len": "Lenca",
+ "leo": "Leti (Cameroon)",
+ "lep": "Lepcha",
+ "leq": "Lembena",
+ "ler": "Lenkau",
+ "les": "Lese",
+ "let": "Lesing-Gelimi; Amio-Gelimi",
+ "leu": "Kara (Papua New Guinea)",
+ "lev": "Lamma",
+ "lew": "Ledo Kaili",
+ "lex": "Luang",
+ "ley": "Lemolang",
+ "lez": "Lezghian",
+ "lfa": "Lefa",
+ "lfn": "Lingua Franca Nova",
+ "lg": "Ganda; Luganda",
+ "lga": "Lungga",
+ "lgb": "Laghu",
+ "lgg": "Lugbara",
+ "lgh": "Laghuu",
+ "lgi": "Lengilu",
+ "lgk": "Lingarak; Neverver",
+ "lgl": "Wala",
+ "lgm": "Lega-Mwenga",
+ "lgn": "T'apo; Opuuo",
+ "lgo": "Lango (South Sudan)",
+ "lgq": "Logba",
+ "lgr": "Lengo",
+ "lgt": "Pahi",
+ "lgu": "Longgu",
+ "lgz": "Ligenza",
+ "lha": "Laha (Viet Nam)",
+ "lhh": "Laha (Indonesia)",
+ "lhi": "Lahu Shi",
+ "lhl": "Lahul Lohar",
+ "lhm": "Lhomi",
+ "lhn": "Lahanan",
+ "lhp": "Lhokpu",
+ "lhs": "Mlahsö",
+ "lht": "Lo-Toga",
+ "lhu": "Lahu",
+ "li": "Limburgan; Limburger; Limburgish",
+ "lia": "West-Central Limba",
+ "lib": "Likum",
+ "lic": "Hlai",
+ "lid": "Nyindrou",
+ "lie": "Likila",
+ "lif": "Limbu",
+ "lig": "Ligbi",
+ "lih": "Lihir",
+ "lij": "Ligurian",
+ "lik": "Lika",
+ "lil": "Lillooet",
+ "lio": "Liki",
+ "lip": "Sekpele",
+ "liq": "Libido",
+ "lir": "Liberian English",
+ "lis": "Lisu",
+ "liu": "Logorik",
+ "liv": "Liv",
+ "liw": "Col",
+ "lix": "Liabuku",
+ "liy": "Banda-Bambari",
+ "liz": "Libinza",
+ "lja": "Golpa",
+ "lje": "Rampi",
+ "lji": "Laiyolo",
+ "ljl": "Li'o",
+ "ljp": "Lampung Api",
+ "ljw": "Yirandali",
+ "ljx": "Yuru",
+ "lka": "Lakalei",
+ "lkb": "Kabras; Lukabaras",
+ "lkc": "Kucong",
+ "lkd": "Lakondê",
+ "lke": "Kenyi",
+ "lkh": "Lakha",
+ "lki": "Laki",
+ "lkj": "Remun",
+ "lkl": "Laeko-Libuat",
+ "lkm": "Kalaamaya",
+ "lkn": "Lakon; Vure",
+ "lko": "Khayo; Olukhayo",
+ "lkr": "Päri",
+ "lks": "Kisa; Olushisa",
+ "lkt": "Lakota",
+ "lku": "Kungkari",
+ "lky": "Lokoya",
+ "lla": "Lala-Roba",
+ "llb": "Lolo",
+ "llc": "Lele (Guinea)",
+ "lld": "Ladin",
+ "lle": "Lele (Papua New Guinea)",
+ "llf": "Hermit",
+ "llg": "Lole",
+ "llh": "Lamu",
+ "lli": "Teke-Laali",
+ "llj": "Ladji Ladji",
+ "llk": "Lelak",
+ "lll": "Lilau",
+ "llm": "Lasalimu",
+ "lln": "Lele (Chad)",
+ "llp": "North Efate",
+ "llq": "Lolak",
+ "lls": "Lithuanian Sign Language",
+ "llu": "Lau",
+ "llx": "Lauan",
+ "lma": "East Limba",
+ "lmb": "Merei",
+ "lmc": "Limilngan",
+ "lmd": "Lumun",
+ "lme": "Pévé",
+ "lmf": "South Lembata",
+ "lmg": "Lamogai",
+ "lmh": "Lambichhong",
+ "lmi": "Lombi",
+ "lmj": "West Lembata",
+ "lmk": "Lamkang",
+ "lml": "Hano",
+ "lmn": "Lambadi",
+ "lmo": "Lombard",
+ "lmp": "Limbum",
+ "lmq": "Lamatuka",
+ "lmr": "Lamalera",
+ "lmu": "Lamenu",
+ "lmv": "Lomaiviti",
+ "lmw": "Lake Miwok",
+ "lmx": "Laimbue",
+ "lmy": "Lamboya",
+ "ln": "Lingala",
+ "lna": "Langbashe",
+ "lnb": "Mbalanhu",
+ "lnd": "Lundayeh; Lun Bawang",
+ "lng": "Langobardic",
+ "lnh": "Lanoh",
+ "lni": "Daantanai'",
+ "lnj": "Leningitij",
+ "lnl": "South Central Banda",
+ "lnm": "Langam",
+ "lnn": "Lorediakarkar",
+ "lns": "Lamnso'",
+ "lnu": "Longuda",
+ "lnw": "Lanima",
+ "lnz": "Lonzo",
+ "lo": "Lao",
+ "loa": "Loloda",
+ "lob": "Lobi",
+ "loc": "Inonhan",
+ "loe": "Saluan",
+ "lof": "Logol",
+ "log": "Logo",
+ "loh": "Narim",
+ "loi": "Loma (Côte d'Ivoire)",
+ "loj": "Lou",
+ "lok": "Loko",
+ "lol": "Mongo",
+ "lom": "Loma (Liberia)",
+ "lon": "Malawi Lomwe",
+ "loo": "Lombo",
+ "lop": "Lopa",
+ "loq": "Lobala",
+ "lor": "Téén",
+ "los": "Loniu",
+ "lot": "Otuho",
+ "lou": "Louisiana Creole",
+ "lov": "Lopi",
+ "low": "Tampias Lobu",
+ "lox": "Loun",
+ "loy": "Loke",
+ "loz": "Lozi",
+ "lpa": "Lelepa",
+ "lpe": "Lepki",
+ "lpn": "Long Phuri Naga",
+ "lpo": "Lipo",
+ "lpx": "Lopit",
+ "lqr": "Logir",
+ "lra": "Rara Bakati'",
+ "lrc": "Northern Luri",
+ "lre": "Laurentian",
+ "lrg": "Laragia",
+ "lri": "Marachi; Olumarachi",
+ "lrk": "Loarki",
+ "lrl": "Lari",
+ "lrm": "Marama; Olumarama",
+ "lrn": "Lorang",
+ "lro": "Laro",
+ "lrr": "Southern Yamphu",
+ "lrt": "Larantuka Malay",
+ "lrv": "Larevat",
+ "lrz": "Lemerig",
+ "lsa": "Lasgerdi",
+ "lsb": "Burundian Sign Language; Langue des Signes Burundaise",
+ "lsc": "Albarradas Sign Language; Lengua de señas Albarradas",
+ "lsd": "Lishana Deni",
+ "lse": "Lusengo",
+ "lsh": "Lish",
+ "lsi": "Lashi",
+ "lsl": "Latvian Sign Language",
+ "lsm": "Saamia; Olusamia",
+ "lsn": "Tibetan Sign Language",
+ "lso": "Laos Sign Language",
+ "lsp": "Panamanian Sign Language; Lengua de Señas Panameñas",
+ "lsr": "Aruop",
+ "lss": "Lasi",
+ "lst": "Trinidad and Tobago Sign Language",
+ "lsv": "Sivia Sign Language",
+ "lsw": "Seychelles Sign Language; Lalang Siny Seselwa; Langue des Signes Seychelloise",
+ "lsy": "Mauritian Sign Language",
+ "lt": "Lithuanian",
+ "ltc": "Late Middle Chinese",
+ "ltg": "Latgalian",
+ "lth": "Thur",
+ "lti": "Leti (Indonesia)",
+ "ltn": "Latundê",
+ "lto": "Tsotso; Olutsotso",
+ "lts": "Tachoni; Lutachoni",
+ "ltu": "Latu",
+ "lu": "Luba-Katanga",
+ "lua": "Luba-Lulua",
+ "luc": "Aringa",
+ "lud": "Ludian",
+ "lue": "Luvale",
+ "luf": "Laua",
+ "lui": "Luiseno",
+ "luj": "Luna",
+ "luk": "Lunanakha",
+ "lul": "Olu'bo",
+ "lum": "Luimbi",
+ "lun": "Lunda",
+ "luo": "Luo (Kenya and Tanzania); Dholuo",
+ "lup": "Lumbu",
+ "luq": "Lucumi",
+ "lur": "Laura",
+ "lus": "Lushai",
+ "lut": "Lushootseed",
+ "luu": "Lumba-Yakkha",
+ "luv": "Luwati",
+ "luw": "Luo (Cameroon)",
+ "luy": "Luyia; Oluluyia",
+ "luz": "Southern Luri",
+ "lv": "Latvian",
+ "lva": "Maku'a",
+ "lvi": "Lavi",
+ "lvk": "Lavukaleve",
+ "lvs": "Standard Latvian",
+ "lvu": "Levuka",
+ "lwa": "Lwalu",
+ "lwe": "Lewo Eleng",
+ "lwg": "Wanga; Oluwanga",
+ "lwh": "White Lachi",
+ "lwl": "Eastern Lawa",
+ "lwm": "Laomian",
+ "lwo": "Luwo",
+ "lws": "Malawian Sign Language",
+ "lwt": "Lewotobi",
+ "lwu": "Lawu",
+ "lww": "Lewo",
+ "lxm": "Lakurumau",
+ "lya": "Layakha",
+ "lyg": "Lyngngam",
+ "lyn": "Luyana",
+ "lzh": "Literary Chinese",
+ "lzl": "Litzlitz",
+ "lzn": "Leinong Naga",
+ "lzz": "Laz",
+ "maa": "San Jerónimo Tecóatl Mazatec",
+ "mab": "Yutanduchi Mixtec",
+ "mad": "Madurese",
+ "mae": "Bo-Rukul",
+ "maf": "Mafa",
+ "mag": "Magahi",
+ "mai": "Maithili",
+ "maj": "Jalapa De Díaz Mazatec",
+ "mak": "Makasar",
+ "mam": "Mam",
+ "man": "Mandingo; Manding",
+ "map": "Austronesian languages",
+ "maq": "Chiquihuitlán Mazatec",
+ "mas": "Masai",
+ "mat": "San Francisco Matlatzinca",
+ "mau": "Huautla Mazatec",
+ "mav": "Sateré-Mawé",
+ "maw": "Mampruli",
+ "max": "North Moluccan Malay",
+ "maz": "Central Mazahua",
+ "mba": "Higaonon",
+ "mbb": "Western Bukidnon Manobo",
+ "mbc": "Macushi",
+ "mbd": "Dibabawon Manobo",
+ "mbe": "Molale",
+ "mbf": "Baba Malay",
+ "mbh": "Mangseng",
+ "mbi": "Ilianen Manobo",
+ "mbj": "Nadëb",
+ "mbk": "Malol",
+ "mbl": "Maxakalí",
+ "mbm": "Ombamba",
+ "mbn": "Macaguán",
+ "mbo": "Mbo (Cameroon)",
+ "mbp": "Malayo",
+ "mbq": "Maisin",
+ "mbr": "Nukak Makú",
+ "mbs": "Sarangani Manobo",
+ "mbt": "Matigsalug Manobo",
+ "mbu": "Mbula-Bwazza",
+ "mbv": "Mbulungish",
+ "mbw": "Maring",
+ "mbx": "Mari (East Sepik Province)",
+ "mby": "Memoni",
+ "mbz": "Amoltepec Mixtec",
+ "mca": "Maca",
+ "mcb": "Machiguenga",
+ "mcc": "Bitur",
+ "mcd": "Sharanahua",
+ "mce": "Itundujia Mixtec",
+ "mcf": "Matsés",
+ "mcg": "Mapoyo",
+ "mch": "Maquiritari",
+ "mci": "Mese",
+ "mcj": "Mvanip",
+ "mck": "Mbunda",
+ "mcl": "Macaguaje",
+ "mcm": "Malaccan Creole Portuguese",
+ "mcn": "Masana",
+ "mco": "Coatlán Mixe",
+ "mcp": "Makaa",
+ "mcq": "Ese",
+ "mcr": "Menya",
+ "mcs": "Mambai",
+ "mct": "Mengisa",
+ "mcu": "Cameroon Mambila",
+ "mcv": "Minanibai",
+ "mcw": "Mawa (Chad)",
+ "mcx": "Mpiemo",
+ "mcy": "South Watut",
+ "mcz": "Mawan",
+ "mda": "Mada (Nigeria)",
+ "mdb": "Morigi",
+ "mdc": "Male (Papua New Guinea)",
+ "mdd": "Mbum",
+ "mde": "Maba (Chad)",
+ "mdf": "Moksha",
+ "mdg": "Massalat",
+ "mdh": "Maguindanaon",
+ "mdi": "Mamvu",
+ "mdj": "Mangbetu",
+ "mdk": "Mangbutu",
+ "mdl": "Maltese Sign Language",
+ "mdm": "Mayogo",
+ "mdn": "Mbati",
+ "mdp": "Mbala",
+ "mdq": "Mbole",
+ "mdr": "Mandar",
+ "mds": "Maria (Papua New Guinea)",
+ "mdt": "Mbere",
+ "mdu": "Mboko",
+ "mdv": "Santa Lucía Monteverde Mixtec",
+ "mdw": "Mbosi",
+ "mdx": "Dizin",
+ "mdy": "Male (Ethiopia)",
+ "mdz": "Suruí Do Pará",
+ "mea": "Menka",
+ "meb": "Ikobi",
+ "mec": "Marra",
+ "med": "Melpa",
+ "mee": "Mengen",
+ "mef": "Megam",
+ "meh": "Southwestern Tlaxiaco Mixtec",
+ "mei": "Midob",
+ "mej": "Meyah",
+ "mek": "Mekeo",
+ "mel": "Central Melanau",
+ "mem": "Mangala",
+ "men": "Mende (Sierra Leone)",
+ "meo": "Kedah Malay",
+ "mep": "Miriwoong",
+ "meq": "Merey",
+ "mer": "Meru",
+ "mes": "Masmaje",
+ "met": "Mato",
+ "meu": "Motu",
+ "mev": "Mano",
+ "mew": "Maaka",
+ "mey": "Hassaniyya",
+ "mez": "Menominee",
+ "mfa": "Pattani Malay",
+ "mfb": "Bangka",
+ "mfc": "Mba",
+ "mfd": "Mendankwe-Nkwen",
+ "mfe": "Morisyen",
+ "mff": "Naki",
+ "mfg": "Mogofin",
+ "mfh": "Matal",
+ "mfi": "Wandala",
+ "mfj": "Mefele",
+ "mfk": "North Mofu",
+ "mfl": "Putai",
+ "mfm": "Marghi South",
+ "mfn": "Cross River Mbembe",
+ "mfo": "Mbe",
+ "mfp": "Makassar Malay",
+ "mfq": "Moba",
+ "mfr": "Marrithiyel",
+ "mfs": "Mexican Sign Language",
+ "mft": "Mokerang",
+ "mfu": "Mbwela",
+ "mfv": "Mandjak",
+ "mfw": "Mulaha",
+ "mfx": "Melo",
+ "mfy": "Mayo",
+ "mfz": "Mabaan",
+ "mg": "Malagasy",
+ "mga": "Middle Irish (900-1200)",
+ "mgb": "Mararit",
+ "mgc": "Morokodo",
+ "mgd": "Moru",
+ "mge": "Mango",
+ "mgf": "Maklew",
+ "mgg": "Mpumpong",
+ "mgh": "Makhuwa-Meetto",
+ "mgi": "Lijili",
+ "mgj": "Abureni",
+ "mgk": "Mawes",
+ "mgl": "Maleu-Kilenge",
+ "mgm": "Mambae",
+ "mgn": "Mbangi",
+ "mgo": "Meta'",
+ "mgp": "Eastern Magar",
+ "mgq": "Malila",
+ "mgr": "Mambwe-Lungu",
+ "mgs": "Manda (Tanzania)",
+ "mgt": "Mongol",
+ "mgu": "Mailu",
+ "mgv": "Matengo",
+ "mgw": "Matumbi",
+ "mgy": "Mbunga",
+ "mgz": "Mbugwe",
+ "mh": "Marshallese",
+ "mha": "Manda (India)",
+ "mhb": "Mahongwe",
+ "mhc": "Mocho",
+ "mhd": "Mbugu",
+ "mhe": "Besisi; Mah Meri",
+ "mhf": "Mamaa",
+ "mhg": "Margu",
+ "mhi": "Ma'di",
+ "mhj": "Mogholi",
+ "mhk": "Mungaka",
+ "mhl": "Mauwake",
+ "mhm": "Makhuwa-Moniga",
+ "mhn": "Mócheno",
+ "mho": "Mashi (Zambia)",
+ "mhp": "Balinese Malay",
+ "mhq": "Mandan",
+ "mhr": "Eastern Mari",
+ "mhs": "Buru (Indonesia)",
+ "mht": "Mandahuaca",
+ "mhu": "Digaro-Mishmi; Darang Deng",
+ "mhw": "Mbukushu",
+ "mhx": "Maru; Lhaovo",
+ "mhy": "Ma'anyan",
+ "mhz": "Mor (Mor Islands)",
+ "mi": "Maori",
+ "mia": "Miami",
+ "mib": "Atatláhuca Mixtec",
+ "mic": "Mi'kmaq; Micmac",
+ "mid": "Mandaic",
+ "mie": "Ocotepec Mixtec",
+ "mif": "Mofu-Gudur",
+ "mig": "San Miguel El Grande Mixtec",
+ "mih": "Chayuco Mixtec",
+ "mii": "Chigmecatitlán Mixtec",
+ "mij": "Abar; Mungbam",
+ "mik": "Mikasuki",
+ "mil": "Peñoles Mixtec",
+ "mim": "Alacatlatzala Mixtec",
+ "min": "Minangkabau",
+ "mio": "Pinotepa Nacional Mixtec",
+ "mip": "Apasco-Apoala Mixtec",
+ "miq": "Mískito",
+ "mir": "Isthmus Mixe",
+ "mit": "Southern Puebla Mixtec",
+ "miu": "Cacaloxtepec Mixtec",
+ "miw": "Akoye",
+ "mix": "Mixtepec Mixtec",
+ "miy": "Ayutla Mixtec",
+ "miz": "Coatzospan Mixtec",
+ "mjb": "Makalero",
+ "mjc": "San Juan Colorado Mixtec",
+ "mjd": "Northwest Maidu",
+ "mje": "Muskum",
+ "mjg": "Tu",
+ "mjh": "Mwera (Nyasa)",
+ "mji": "Kim Mun",
+ "mjj": "Mawak",
+ "mjk": "Matukar",
+ "mjl": "Mandeali",
+ "mjm": "Medebur",
+ "mjn": "Ma (Papua New Guinea)",
+ "mjo": "Malankuravan",
+ "mjp": "Malapandaram",
+ "mjq": "Malaryan",
+ "mjr": "Malavedan",
+ "mjs": "Miship",
+ "mjt": "Sauria Paharia",
+ "mju": "Manna-Dora",
+ "mjv": "Mannan",
+ "mjw": "Karbi",
+ "mjx": "Mahali",
+ "mjy": "Mahican",
+ "mjz": "Majhi",
+ "mk": "Macedonian",
+ "mka": "Mbre",
+ "mkb": "Mal Paharia",
+ "mkc": "Siliput",
+ "mke": "Mawchi",
+ "mkf": "Miya",
+ "mkg": "Mak (China)",
+ "mkh": "Mon-Khmer languages",
+ "mki": "Dhatki",
+ "mkj": "Mokilese",
+ "mkk": "Byep",
+ "mkl": "Mokole",
+ "mkm": "Moklen",
+ "mkn": "Kupang Malay",
+ "mko": "Mingang Doso",
+ "mkp": "Moikodi",
+ "mkq": "Bay Miwok",
+ "mkr": "Malas",
+ "mks": "Silacayoapan Mixtec",
+ "mkt": "Vamale",
+ "mku": "Konyanka Maninka",
+ "mkv": "Mafea",
+ "mkw": "Kituba (Congo)",
+ "mkx": "Kinamiging Manobo",
+ "mky": "East Makian",
+ "mkz": "Makasae",
+ "ml": "Malayalam",
+ "mla": "Malo",
+ "mlb": "Mbule",
+ "mlc": "Cao Lan",
+ "mle": "Manambu",
+ "mlf": "Mal",
+ "mlh": "Mape",
+ "mli": "Malimpung",
+ "mlj": "Miltu",
+ "mlk": "Ilwana; Kiwilwana",
+ "mll": "Malua Bay",
+ "mlm": "Mulam",
+ "mln": "Malango",
+ "mlo": "Mlomp",
+ "mlp": "Bargam",
+ "mlq": "Western Maninkakan",
+ "mlr": "Vame",
+ "mls": "Masalit",
+ "mlu": "To'abaita",
+ "mlv": "Motlav; Mwotlap",
+ "mlw": "Moloko",
+ "mlx": "Malfaxal; Naha'ai",
+ "mlz": "Malaynon",
+ "mma": "Mama",
+ "mmb": "Momina",
+ "mmc": "Michoacán Mazahua",
+ "mmd": "Maonan",
+ "mme": "Mae",
+ "mmf": "Mundat",
+ "mmg": "North Ambrym",
+ "mmh": "Mehináku",
+ "mmi": "Musar",
+ "mmj": "Majhwar",
+ "mmk": "Mukha-Dora",
+ "mml": "Man Met",
+ "mmm": "Maii",
+ "mmn": "Mamanwa",
+ "mmo": "Mangga Buang",
+ "mmp": "Siawi",
+ "mmq": "Musak",
+ "mmr": "Western Xiangxi Miao",
+ "mmt": "Malalamai",
+ "mmu": "Mmaala",
+ "mmv": "Miriti",
+ "mmw": "Emae",
+ "mmx": "Madak",
+ "mmy": "Migaama",
+ "mmz": "Mabaale",
+ "mn": "Mongolian",
+ "mna": "Mbula",
+ "mnb": "Muna",
+ "mnc": "Manchu",
+ "mnd": "Mondé",
+ "mne": "Naba",
+ "mnf": "Mundani",
+ "mng": "Eastern Mnong",
+ "mnh": "Mono (Democratic Republic of Congo)",
+ "mni": "Manipuri",
+ "mnj": "Munji",
+ "mnk": "Mandinka",
+ "mnl": "Tiale",
+ "mnm": "Mapena",
+ "mnn": "Southern Mnong",
+ "mno": "Manobo languages",
+ "mnp": "Min Bei Chinese",
+ "mnq": "Minriq",
+ "mnr": "Mono (USA)",
+ "mns": "Mansi",
+ "mnu": "Mer",
+ "mnv": "Rennell-Bellona",
+ "mnw": "Mon",
+ "mnx": "Manikion",
+ "mny": "Manyawa",
+ "mnz": "Moni",
+ "moa": "Mwan",
+ "moc": "Mocoví",
+ "mod": "Mobilian",
+ "moe": "Innu; Montagnais",
+ "mog": "Mongondow",
+ "moh": "Mohawk",
+ "moi": "Mboi",
+ "moj": "Monzombo",
+ "mok": "Morori",
+ "mom": "Mangue",
+ "moo": "Monom",
+ "mop": "Mopán Maya",
+ "moq": "Mor (Bomberai Peninsula)",
+ "mor": "Moro",
+ "mos": "Mossi",
+ "mot": "Barí",
+ "mou": "Mogum",
+ "mov": "Mohave",
+ "mow": "Moi (Congo)",
+ "mox": "Molima",
+ "moy": "Shekkacho",
+ "moz": "Mukulu; Gergiko",
+ "mpa": "Mpoto",
+ "mpb": "Malak Malak; Mullukmulluk",
+ "mpc": "Mangarrayi",
+ "mpd": "Machinere",
+ "mpe": "Majang",
+ "mpg": "Marba",
+ "mph": "Maung",
+ "mpi": "Mpade",
+ "mpj": "Martu Wangka; Wangkajunga",
+ "mpk": "Mbara (Chad)",
+ "mpl": "Middle Watut",
+ "mpm": "Yosondúa Mixtec",
+ "mpn": "Mindiri",
+ "mpo": "Miu",
+ "mpp": "Migabac",
+ "mpq": "Matís",
+ "mpr": "Vangunu",
+ "mps": "Dadibi",
+ "mpt": "Mian",
+ "mpu": "Makuráp",
+ "mpv": "Mungkip",
+ "mpw": "Mapidian",
+ "mpx": "Misima-Panaeati",
+ "mpy": "Mapia",
+ "mpz": "Mpi",
+ "mqa": "Maba (Indonesia)",
+ "mqb": "Mbuko",
+ "mqc": "Mangole",
+ "mqe": "Matepi",
+ "mqf": "Momuna",
+ "mqg": "Kota Bangun Kutai Malay",
+ "mqh": "Tlazoyaltepec Mixtec",
+ "mqi": "Mariri",
+ "mqj": "Mamasa",
+ "mqk": "Rajah Kabunsuwan Manobo",
+ "mql": "Mbelime",
+ "mqm": "South Marquesan",
+ "mqn": "Moronene",
+ "mqo": "Modole",
+ "mqp": "Manipa",
+ "mqq": "Minokok",
+ "mqr": "Mander",
+ "mqs": "West Makian",
+ "mqt": "Mok",
+ "mqu": "Mandari",
+ "mqv": "Mosimo",
+ "mqw": "Murupi",
+ "mqx": "Mamuju",
+ "mqy": "Manggarai",
+ "mqz": "Pano",
+ "mr": "Marathi",
+ "mra": "Mlabri",
+ "mrb": "Marino",
+ "mrc": "Maricopa",
+ "mrd": "Western Magar",
+ "mre": "Martha's Vineyard Sign Language",
+ "mrf": "Elseng",
+ "mrg": "Mising",
+ "mrh": "Mara Chin",
+ "mrj": "Western Mari",
+ "mrk": "Hmwaveke",
+ "mrl": "Mortlockese",
+ "mrm": "Merlav; Mwerlap",
+ "mrn": "Cheke Holo",
+ "mro": "Mru",
+ "mrp": "Morouas",
+ "mrq": "North Marquesan",
+ "mrr": "Maria (India)",
+ "mrs": "Maragus",
+ "mrt": "Marghi Central",
+ "mru": "Mono (Cameroon)",
+ "mrv": "Mangareva",
+ "mrw": "Maranao",
+ "mrx": "Maremgi; Dineor",
+ "mry": "Mandaya",
+ "mrz": "Marind",
+ "ms": "Malay (macrolanguage)",
+ "msb": "Masbatenyo",
+ "msc": "Sankaran Maninka",
+ "msd": "Yucatec Maya Sign Language",
+ "mse": "Musey",
+ "msf": "Mekwei",
+ "msg": "Moraid",
+ "msh": "Masikoro Malagasy",
+ "msi": "Sabah Malay",
+ "msj": "Ma (Democratic Republic of Congo)",
+ "msk": "Mansaka",
+ "msl": "Molof; Poule",
+ "msm": "Agusan Manobo",
+ "msn": "Vurës",
+ "mso": "Mombum",
+ "msp": "Maritsauá",
+ "msq": "Caac",
+ "msr": "Mongolian Sign Language",
+ "mss": "West Masela",
+ "msu": "Musom",
+ "msv": "Maslam",
+ "msw": "Mansoanka",
+ "msx": "Moresada",
+ "msy": "Aruamu",
+ "msz": "Momare",
+ "mt": "Maltese",
+ "mta": "Cotabato Manobo",
+ "mtb": "Anyin Morofo",
+ "mtc": "Munit",
+ "mtd": "Mualang",
+ "mte": "Mono (Solomon Islands)",
+ "mtf": "Murik (Papua New Guinea)",
+ "mtg": "Una",
+ "mth": "Munggui",
+ "mti": "Maiwa (Papua New Guinea)",
+ "mtj": "Moskona",
+ "mtk": "Mbe'",
+ "mtl": "Montol",
+ "mtm": "Mator",
+ "mtn": "Matagalpa",
+ "mto": "Totontepec Mixe",
+ "mtp": "Wichí Lhamtés Nocten",
+ "mtq": "Muong",
+ "mtr": "Mewari",
+ "mts": "Yora",
+ "mtt": "Mota",
+ "mtu": "Tututepec Mixtec",
+ "mtv": "Asaro'o",
+ "mtw": "Southern Binukidnon",
+ "mtx": "Tidaá Mixtec",
+ "mty": "Nabi",
+ "mua": "Mundang",
+ "mub": "Mubi",
+ "muc": "Ajumbu",
+ "mud": "Mednyj Aleut",
+ "mue": "Media Lengua",
+ "mug": "Musgu",
+ "muh": "Mündü",
+ "mui": "Musi",
+ "muj": "Mabire",
+ "muk": "Mugom",
+ "mum": "Maiwala",
+ "mun": "Munda languages",
+ "muo": "Nyong",
+ "mup": "Malvi",
+ "muq": "Eastern Xiangxi Miao",
+ "mur": "Murle",
+ "mus": "Creek",
+ "mut": "Western Muria",
+ "muu": "Yaaku",
+ "muv": "Muthuvan",
+ "mux": "Bo-Ung",
+ "muy": "Muyang",
+ "muz": "Mursi",
+ "mva": "Manam",
+ "mvb": "Mattole",
+ "mvd": "Mamboru",
+ "mve": "Marwari (Pakistan)",
+ "mvf": "Peripheral Mongolian",
+ "mvg": "Yucuañe Mixtec",
+ "mvh": "Mulgi",
+ "mvi": "Miyako",
+ "mvk": "Mekmek",
+ "mvl": "Mbara (Australia)",
+ "mvn": "Minaveha",
+ "mvo": "Marovo",
+ "mvp": "Duri",
+ "mvq": "Moere",
+ "mvr": "Marau",
+ "mvs": "Massep",
+ "mvt": "Mpotovoro",
+ "mvu": "Marfa",
+ "mvv": "Tagal Murut",
+ "mvw": "Machinga",
+ "mvx": "Meoswar",
+ "mvy": "Indus Kohistani",
+ "mvz": "Mesqan",
+ "mwa": "Mwatebu",
+ "mwb": "Juwal",
+ "mwc": "Are",
+ "mwe": "Mwera (Chimwera)",
+ "mwf": "Murrinh-Patha",
+ "mwg": "Aiklep",
+ "mwh": "Mouk-Aria",
+ "mwi": "Labo; Ninde",
+ "mwk": "Kita Maninkakan",
+ "mwl": "Mirandese",
+ "mwm": "Sar",
+ "mwn": "Nyamwanga",
+ "mwo": "Central Maewo",
+ "mwp": "Kala Lagaw Ya",
+ "mwq": "Mün Chin",
+ "mwr": "Marwari",
+ "mws": "Mwimbi-Muthambi",
+ "mwt": "Moken",
+ "mwu": "Mittu",
+ "mwv": "Mentawai",
+ "mww": "Hmong Daw",
+ "mwz": "Moingi",
+ "mxa": "Northwest Oaxaca Mixtec",
+ "mxb": "Tezoatlán Mixtec",
+ "mxc": "Manyika",
+ "mxd": "Modang",
+ "mxe": "Mele-Fila",
+ "mxf": "Malgbe",
+ "mxg": "Mbangala",
+ "mxh": "Mvuba",
+ "mxi": "Mozarabic",
+ "mxj": "Miju-Mishmi; Geman Deng",
+ "mxk": "Monumbo",
+ "mxl": "Maxi Gbe",
+ "mxm": "Meramera",
+ "mxn": "Moi (Indonesia)",
+ "mxo": "Mbowe",
+ "mxp": "Tlahuitoltepec Mixe",
+ "mxq": "Juquila Mixe",
+ "mxr": "Murik (Malaysia)",
+ "mxs": "Huitepec Mixtec",
+ "mxt": "Jamiltepec Mixtec",
+ "mxu": "Mada (Cameroon)",
+ "mxv": "Metlatónoc Mixtec",
+ "mxw": "Namo",
+ "mxx": "Mahou; Mawukakan",
+ "mxy": "Southeastern Nochixtlán Mixtec",
+ "mxz": "Central Masela",
+ "my": "Burmese",
+ "myb": "Mbay",
+ "myc": "Mayeka",
+ "mye": "Myene",
+ "myf": "Bambassi",
+ "myg": "Manta",
+ "myh": "Makah",
+ "myj": "Mangayat",
+ "myk": "Mamara Senoufo",
+ "myl": "Moma",
+ "mym": "Me'en",
+ "myn": "Mayan languages",
+ "myo": "Anfillo",
+ "myp": "Pirahã",
+ "myr": "Muniche",
+ "mys": "Mesmes",
+ "myu": "Mundurukú",
+ "myv": "Erzya",
+ "myw": "Muyuw",
+ "myx": "Masaaba",
+ "myy": "Macuna",
+ "myz": "Classical Mandaic",
+ "mza": "Santa María Zacatepec Mixtec",
+ "mzb": "Tumzabt",
+ "mzc": "Madagascar Sign Language",
+ "mzd": "Malimba",
+ "mze": "Morawa",
+ "mzg": "Monastic Sign Language",
+ "mzh": "Wichí Lhamtés Güisnay",
+ "mzi": "Ixcatlán Mazatec",
+ "mzj": "Manya",
+ "mzk": "Nigeria Mambila",
+ "mzl": "Mazatlán Mixe",
+ "mzm": "Mumuye",
+ "mzn": "Mazanderani",
+ "mzo": "Matipuhy",
+ "mzp": "Movima",
+ "mzq": "Mori Atas",
+ "mzr": "Marúbo",
+ "mzs": "Macanese",
+ "mzt": "Mintil",
+ "mzu": "Inapang",
+ "mzv": "Manza",
+ "mzw": "Deg",
+ "mzx": "Mawayana",
+ "mzy": "Mozambican Sign Language",
+ "mzz": "Maiadomu",
+ "na": "Nauru",
+ "naa": "Namla",
+ "nab": "Southern Nambikuára",
+ "nac": "Narak",
+ "nae": "Naka'ela",
+ "naf": "Nabak",
+ "nag": "Naga Pidgin",
+ "nah": "Nahuatl languages",
+ "nai": "North American Indian languages",
+ "naj": "Nalu",
+ "nak": "Nakanai",
+ "nal": "Nalik",
+ "nam": "Ngan'gityemerri",
+ "nan": "Min Nan Chinese",
+ "nao": "Naaba",
+ "nap": "Neapolitan",
+ "naq": "Khoekhoe; Nama (Namibia)",
+ "nar": "Iguta",
+ "nas": "Naasioi",
+ "nat": "Ca̱hungwa̱rya̱; Hungworo",
+ "naw": "Nawuri",
+ "nax": "Nakwi",
+ "nay": "Ngarrindjeri",
+ "naz": "Coatepec Nahuatl",
+ "nb": "Norwegian Bokmål",
+ "nba": "Nyemba",
+ "nbb": "Ndoe",
+ "nbc": "Chang Naga",
+ "nbd": "Ngbinda",
+ "nbe": "Konyak Naga",
+ "nbg": "Nagarchal",
+ "nbh": "Ngamo",
+ "nbi": "Mao Naga",
+ "nbj": "Ngarinyman",
+ "nbk": "Nake",
+ "nbm": "Ngbaka Ma'bo",
+ "nbn": "Kuri",
+ "nbo": "Nkukoli",
+ "nbp": "Nnam",
+ "nbq": "Nggem",
+ "nbr": "Numana",
+ "nbs": "Namibian Sign Language",
+ "nbt": "Na",
+ "nbu": "Rongmei Naga",
+ "nbv": "Ngamambo",
+ "nbw": "Southern Ngbandi",
+ "nby": "Ningera",
+ "nca": "Iyo",
+ "ncb": "Central Nicobarese",
+ "ncc": "Ponam",
+ "ncd": "Nachering",
+ "nce": "Yale",
+ "ncf": "Notsi",
+ "ncg": "Nisga'a",
+ "nch": "Central Huasteca Nahuatl",
+ "nci": "Classical Nahuatl",
+ "ncj": "Northern Puebla Nahuatl",
+ "nck": "Na-kara",
+ "ncl": "Michoacán Nahuatl",
+ "ncm": "Nambo",
+ "ncn": "Nauna",
+ "nco": "Sibe",
+ "ncq": "Northern Katang",
+ "ncr": "Ncane",
+ "ncs": "Nicaraguan Sign Language",
+ "nct": "Chothe Naga",
+ "ncu": "Chumburung",
+ "ncx": "Central Puebla Nahuatl",
+ "ncz": "Natchez",
+ "nd": "North Ndebele",
+ "nda": "Ndasa",
+ "ndb": "Kenswei Nsei",
+ "ndc": "Ndau",
+ "ndd": "Nde-Nsele-Nta",
+ "ndf": "Nadruvian",
+ "ndg": "Ndengereko",
+ "ndh": "Ndali",
+ "ndi": "Samba Leko",
+ "ndj": "Ndamba",
+ "ndk": "Ndaka",
+ "ndl": "Ndolo",
+ "ndm": "Ndam",
+ "ndn": "Ngundi",
+ "ndp": "Ndo",
+ "ndq": "Ndombe",
+ "ndr": "Ndoola",
+ "nds": "Low German; Low Saxon",
+ "ndt": "Ndunga",
+ "ndu": "Dugun",
+ "ndv": "Ndut",
+ "ndw": "Ndobo",
+ "ndx": "Nduga",
+ "ndy": "Lutos",
+ "ndz": "Ndogo",
+ "ne": "Nepali (macrolanguage)",
+ "nea": "Eastern Ngad'a",
+ "neb": "Toura (Côte d'Ivoire)",
+ "nec": "Nedebang",
+ "ned": "Nde-Gbite",
+ "nee": "Nêlêmwa-Nixumwak",
+ "nef": "Nefamese",
+ "neg": "Negidal",
+ "neh": "Nyenkha",
+ "nei": "Neo-Hittite",
+ "nej": "Neko",
+ "nek": "Neku",
+ "nem": "Nemi",
+ "nen": "Nengone",
+ "neo": "Ná-Meo",
+ "neq": "North Central Mixe",
+ "ner": "Yahadian",
+ "nes": "Bhoti Kinnauri",
+ "net": "Nete",
+ "neu": "Neo",
+ "nev": "Nyaheun",
+ "new": "Newari; Nepal Bhasa",
+ "nex": "Neme",
+ "ney": "Neyo",
+ "nez": "Nez Perce",
+ "nfa": "Dhao",
+ "nfd": "Ahwai",
+ "nfl": "Ayiwo; Äiwoo",
+ "nfr": "Nafaanra",
+ "nfu": "Mfumte",
+ "ng": "Ndonga",
+ "nga": "Ngbaka",
+ "ngb": "Northern Ngbandi",
+ "ngc": "Ngombe (Democratic Republic of Congo)",
+ "ngd": "Ngando (Central African Republic)",
+ "nge": "Ngemba",
+ "ngf": "Trans-New Guinea languages",
+ "ngg": "Ngbaka Manza",
+ "ngh": "Nǁng",
+ "ngi": "Ngizim",
+ "ngj": "Ngie",
+ "ngk": "Dalabon",
+ "ngl": "Lomwe",
+ "ngm": "Ngatik Men's Creole",
+ "ngn": "Ngwo",
+ "ngp": "Ngulu",
+ "ngq": "Ngurimi; Ngoreme",
+ "ngr": "Engdewu",
+ "ngs": "Gvoko",
+ "ngt": "Kriang; Ngeq",
+ "ngu": "Guerrero Nahuatl",
+ "ngv": "Nagumi",
+ "ngw": "Ngwaba",
+ "ngx": "Nggwahyi",
+ "ngy": "Tibea",
+ "ngz": "Ngungwel",
+ "nha": "Nhanda",
+ "nhb": "Beng",
+ "nhc": "Tabasco Nahuatl",
+ "nhd": "Chiripá; Ava Guaraní",
+ "nhe": "Eastern Huasteca Nahuatl",
+ "nhf": "Nhuwala",
+ "nhg": "Tetelcingo Nahuatl",
+ "nhh": "Nahari",
+ "nhi": "Zacatlán-Ahuacatlán-Tepetzintla Nahuatl",
+ "nhk": "Isthmus-Cosoleacaque Nahuatl",
+ "nhm": "Morelos Nahuatl",
+ "nhn": "Central Nahuatl",
+ "nho": "Takuu",
+ "nhp": "Isthmus-Pajapan Nahuatl",
+ "nhq": "Huaxcaleca Nahuatl",
+ "nhr": "Naro",
+ "nht": "Ometepec Nahuatl",
+ "nhu": "Noone",
+ "nhv": "Temascaltepec Nahuatl",
+ "nhw": "Western Huasteca Nahuatl",
+ "nhx": "Isthmus-Mecayapan Nahuatl",
+ "nhy": "Northern Oaxaca Nahuatl",
+ "nhz": "Santa María La Alta Nahuatl",
+ "nia": "Nias",
+ "nib": "Nakame",
+ "nic": "Niger-Kordofanian languages",
+ "nid": "Ngandi",
+ "nie": "Niellim",
+ "nif": "Nek",
+ "nig": "Ngalakgan",
+ "nih": "Nyiha (Tanzania)",
+ "nii": "Nii",
+ "nij": "Ngaju",
+ "nik": "Southern Nicobarese",
+ "nil": "Nila",
+ "nim": "Nilamba",
+ "nin": "Ninzo",
+ "nio": "Nganasan",
+ "niq": "Nandi",
+ "nir": "Nimboran",
+ "nis": "Nimi",
+ "nit": "Southeastern Kolami",
+ "niu": "Niuean",
+ "niv": "Gilyak",
+ "niw": "Nimo",
+ "nix": "Hema",
+ "niy": "Ngiti",
+ "niz": "Ningil",
+ "nja": "Nzanyi",
+ "njb": "Nocte Naga",
+ "njd": "Ndonde Hamba",
+ "njh": "Lotha Naga",
+ "nji": "Gudanji",
+ "njj": "Njen",
+ "njl": "Njalgulgule",
+ "njm": "Angami Naga",
+ "njn": "Liangmai Naga",
+ "njo": "Ao Naga",
+ "njr": "Njerep",
+ "njs": "Nisa",
+ "njt": "Ndyuka-Trio Pidgin",
+ "nju": "Ngadjunmaya",
+ "njx": "Kunyi",
+ "njy": "Njyem",
+ "njz": "Nyishi",
+ "nka": "Nkoya",
+ "nkb": "Khoibu Naga",
+ "nkc": "Nkongho",
+ "nkd": "Koireng",
+ "nke": "Duke",
+ "nkf": "Inpui Naga",
+ "nkg": "Nekgini",
+ "nkh": "Khezha Naga",
+ "nki": "Thangal Naga",
+ "nkj": "Nakai",
+ "nkk": "Nokuku",
+ "nkm": "Namat",
+ "nkn": "Nkangala",
+ "nko": "Nkonya",
+ "nkp": "Niuatoputapu",
+ "nkq": "Nkami",
+ "nkr": "Nukuoro",
+ "nks": "North Asmat",
+ "nkt": "Nyika (Tanzania)",
+ "nku": "Bouna Kulango",
+ "nkv": "Nyika (Malawi and Zambia)",
+ "nkw": "Nkutu",
+ "nkx": "Nkoroo",
+ "nkz": "Nkari",
+ "nl": "Dutch; Flemish",
+ "nla": "Ngombale",
+ "nlc": "Nalca",
+ "nle": "East Nyala",
+ "nlg": "Gela",
+ "nli": "Grangali",
+ "nlj": "Nyali",
+ "nlk": "Ninia Yali",
+ "nll": "Nihali",
+ "nlm": "Mankiyali",
+ "nlo": "Ngul",
+ "nlq": "Lao Naga",
+ "nlu": "Nchumbulu",
+ "nlv": "Orizaba Nahuatl",
+ "nlw": "Walangama",
+ "nlx": "Nahali",
+ "nly": "Nyamal",
+ "nlz": "Nalögo",
+ "nma": "Maram Naga",
+ "nmb": "Big Nambas; V'ënen Taut",
+ "nmc": "Ngam",
+ "nmd": "Ndumu",
+ "nme": "Mzieme Naga",
+ "nmf": "Tangkhul Naga (India)",
+ "nmg": "Kwasio",
+ "nmh": "Monsang Naga",
+ "nmi": "Nyam",
+ "nmj": "Ngombe (Central African Republic)",
+ "nmk": "Namakura",
+ "nml": "Ndemli",
+ "nmm": "Manangba",
+ "nmn": "ǃXóõ",
+ "nmo": "Moyon Naga",
+ "nmp": "Nimanbur",
+ "nmq": "Nambya",
+ "nmr": "Nimbari",
+ "nms": "Letemboi",
+ "nmt": "Namonuito",
+ "nmu": "Northeast Maidu",
+ "nmv": "Ngamini",
+ "nmw": "Nimoa; Rifao",
+ "nmx": "Nama (Papua New Guinea)",
+ "nmy": "Namuyi",
+ "nmz": "Nawdm",
+ "nn": "Norwegian Nynorsk",
+ "nna": "Nyangumarta",
+ "nnb": "Nande",
+ "nnc": "Nancere",
+ "nnd": "West Ambae",
+ "nne": "Ngandyera",
+ "nnf": "Ngaing",
+ "nng": "Maring Naga",
+ "nnh": "Ngiemboon",
+ "nni": "North Nuaulu",
+ "nnj": "Nyangatom",
+ "nnk": "Nankina",
+ "nnl": "Northern Rengma Naga",
+ "nnm": "Namia",
+ "nnn": "Ngete",
+ "nnp": "Wancho Naga",
+ "nnq": "Ngindo",
+ "nnr": "Narungga",
+ "nnt": "Nanticoke",
+ "nnu": "Dwang",
+ "nnv": "Nugunu (Australia)",
+ "nnw": "Southern Nuni",
+ "nny": "Nyangga",
+ "nnz": "Nda'nda'",
+ "no": "Norwegian",
+ "noa": "Woun Meu",
+ "noc": "Nuk",
+ "nod": "Northern Thai",
+ "noe": "Nimadi",
+ "nof": "Nomane",
+ "nog": "Nogai",
+ "noh": "Nomu",
+ "noi": "Noiri",
+ "noj": "Nonuya",
+ "nok": "Nooksack",
+ "nol": "Nomlaki",
+ "nom": "Nocamán",
+ "non": "Old Norse",
+ "nop": "Numanggang",
+ "noq": "Ngongo",
+ "nos": "Eastern Nisu",
+ "not": "Nomatsiguenga",
+ "nou": "Ewage-Notu",
+ "nov": "Novial",
+ "now": "Nyambo",
+ "noy": "Noy",
+ "noz": "Nayi",
+ "npa": "Nar Phu",
+ "npb": "Nupbikha",
+ "npg": "Ponyo-Gongwang Naga",
+ "nph": "Phom Naga",
+ "npi": "Nepali (individual language)",
+ "npl": "Southeastern Puebla Nahuatl",
+ "npn": "Mondropolon",
+ "npo": "Pochuri Naga",
+ "nps": "Nipsan",
+ "npu": "Puimei Naga",
+ "npx": "Noipx",
+ "npy": "Napu",
+ "nqg": "Southern Nago",
+ "nqk": "Kura Ede Nago",
+ "nql": "Ngendelengo",
+ "nqm": "Ndom",
+ "nqn": "Nen",
+ "nqo": "N'Ko; N’Ko",
+ "nqq": "Kyan-Karyaw Naga",
+ "nqt": "Nteng",
+ "nqy": "Akyaung Ari Naga",
+ "nr": "South Ndebele",
+ "nra": "Ngom",
+ "nrb": "Nara",
+ "nrc": "Noric",
+ "nre": "Southern Rengma Naga",
+ "nrf": "Jèrriais; Guernésiais",
+ "nrg": "Narango",
+ "nri": "Chokri Naga",
+ "nrk": "Ngarla",
+ "nrl": "Ngarluma",
+ "nrm": "Narom",
+ "nrn": "Norn",
+ "nrp": "North Picene",
+ "nrr": "Norra; Nora",
+ "nrt": "Northern Kalapuya",
+ "nru": "Narua",
+ "nrx": "Ngurmbur",
+ "nrz": "Lala",
+ "nsa": "Sangtam Naga",
+ "nsb": "Lower Nossob",
+ "nsc": "Nshi",
+ "nsd": "Southern Nisu",
+ "nse": "Nsenga",
+ "nsf": "Northwestern Nisu",
+ "nsg": "Ngasa",
+ "nsh": "Ngoshie",
+ "nsi": "Nigerian Sign Language",
+ "nsk": "Naskapi",
+ "nsl": "Norwegian Sign Language",
+ "nsm": "Sumi Naga",
+ "nsn": "Nehan",
+ "nso": "Pedi; Northern Sotho; Sepedi",
+ "nsp": "Nepalese Sign Language",
+ "nsq": "Northern Sierra Miwok",
+ "nsr": "Maritime Sign Language",
+ "nss": "Nali",
+ "nst": "Tase Naga",
+ "nsu": "Sierra Negra Nahuatl",
+ "nsv": "Southwestern Nisu",
+ "nsw": "Navut",
+ "nsx": "Nsongo",
+ "nsy": "Nasal",
+ "nsz": "Nisenan",
+ "ntd": "Northern Tidung",
+ "nte": "Nathembo",
+ "ntg": "Ngantangarra",
+ "nti": "Natioro",
+ "ntj": "Ngaanyatjarra",
+ "ntk": "Ikoma-Nata-Isenye",
+ "ntm": "Nateni",
+ "nto": "Ntomba",
+ "ntp": "Northern Tepehuan",
+ "ntr": "Delo",
+ "ntu": "Natügu",
+ "ntw": "Nottoway",
+ "ntx": "Tangkhul Naga (Myanmar)",
+ "nty": "Mantsi",
+ "ntz": "Natanzi",
+ "nua": "Yuanga",
+ "nub": "Nubian languages",
+ "nuc": "Nukuini",
+ "nud": "Ngala",
+ "nue": "Ngundu",
+ "nuf": "Nusu",
+ "nug": "Nungali",
+ "nuh": "Ndunda",
+ "nui": "Ngumbi",
+ "nuj": "Nyole",
+ "nuk": "Nuu-chah-nulth; Nuuchahnulth",
+ "nul": "Nusa Laut",
+ "num": "Niuafo'ou",
+ "nun": "Anong",
+ "nuo": "Nguôn",
+ "nup": "Nupe-Nupe-Tako",
+ "nuq": "Nukumanu",
+ "nur": "Nukuria",
+ "nus": "Nuer",
+ "nut": "Nung (Viet Nam)",
+ "nuu": "Ngbundu",
+ "nuv": "Northern Nuni",
+ "nuw": "Nguluwan",
+ "nux": "Mehek",
+ "nuy": "Nunggubuyu",
+ "nuz": "Tlamacazapa Nahuatl",
+ "nv": "Navajo; Navaho",
+ "nvh": "Nasarian",
+ "nvm": "Namiae",
+ "nvo": "Nyokon",
+ "nwa": "Nawathinehena",
+ "nwb": "Nyabwa",
+ "nwc": "Classical Newari; Classical Nepal Bhasa; Old Newari",
+ "nwe": "Ngwe",
+ "nwg": "Ngayawung",
+ "nwi": "Southwest Tanna",
+ "nwm": "Nyamusa-Molo",
+ "nwo": "Nauo",
+ "nwr": "Nawaru",
+ "nww": "Ndwewe",
+ "nwx": "Middle Newar",
+ "nwy": "Nottoway-Meherrin",
+ "nxa": "Nauete",
+ "nxd": "Ngando (Democratic Republic of Congo)",
+ "nxe": "Nage",
+ "nxg": "Ngad'a",
+ "nxi": "Nindi",
+ "nxk": "Koki Naga",
+ "nxl": "South Nuaulu",
+ "nxm": "Numidian",
+ "nxn": "Ngawun",
+ "nxo": "Ndambomo",
+ "nxq": "Naxi",
+ "nxr": "Ninggerum",
+ "nxx": "Nafri",
+ "ny": "Nyanja; Chewa; Chichewa",
+ "nyb": "Nyangbo",
+ "nyc": "Nyanga-li",
+ "nyd": "Nyore; Olunyole",
+ "nye": "Nyengo",
+ "nyf": "Giryama; Kigiryama",
+ "nyg": "Nyindu",
+ "nyh": "Nyikina",
+ "nyi": "Ama (Sudan)",
+ "nyj": "Nyanga",
+ "nyk": "Nyaneka",
+ "nyl": "Nyeu",
+ "nym": "Nyamwezi",
+ "nyn": "Nyankole",
+ "nyo": "Nyoro",
+ "nyp": "Nyang'i",
+ "nyq": "Nayini",
+ "nyr": "Nyiha (Malawi)",
+ "nys": "Nyungar",
+ "nyt": "Nyawaygi",
+ "nyu": "Nyungwe",
+ "nyv": "Nyulnyul",
+ "nyw": "Nyaw",
+ "nyx": "Nganyaywana",
+ "nyy": "Nyakyusa-Ngonde",
+ "nza": "Tigon Mbembe",
+ "nzb": "Njebi",
+ "nzd": "Nzadi",
+ "nzi": "Nzima",
+ "nzk": "Nzakara",
+ "nzm": "Zeme Naga",
+ "nzs": "New Zealand Sign Language",
+ "nzu": "Teke-Nzikou",
+ "nzy": "Nzakambay",
+ "nzz": "Nanga Dama Dogon",
+ "oaa": "Orok",
+ "oac": "Oroch",
+ "oar": "Old Aramaic (up to 700 BCE); Ancient Aramaic (up to 700 BCE)",
+ "oav": "Old Avar",
+ "obi": "Obispeño",
+ "obk": "Southern Bontok",
+ "obl": "Oblo",
+ "obm": "Moabite",
+ "obo": "Obo Manobo",
+ "obr": "Old Burmese",
+ "obt": "Old Breton",
+ "obu": "Obulom",
+ "oc": "Occitan (post 1500)",
+ "oca": "Ocaina",
+ "och": "Old Chinese",
+ "ocm": "Old Cham",
+ "oco": "Old Cornish",
+ "ocu": "Atzingo Matlatzinca",
+ "oda": "Odut",
+ "odk": "Od",
+ "odt": "Old Dutch",
+ "odu": "Odual",
+ "ofo": "Ofo",
+ "ofs": "Old Frisian",
+ "ofu": "Efutop",
+ "ogb": "Ogbia",
+ "ogc": "Ogbah",
+ "oge": "Old Georgian",
+ "ogg": "Ogbogolo",
+ "ogo": "Khana",
+ "ogu": "Ogbronuagum",
+ "oht": "Old Hittite",
+ "ohu": "Old Hungarian",
+ "oia": "Oirata",
+ "oie": "Okolie",
+ "oin": "Inebu One",
+ "oj": "Ojibwa",
+ "ojb": "Northwestern Ojibwa",
+ "ojc": "Central Ojibwa",
+ "ojg": "Eastern Ojibwa",
+ "ojp": "Old Japanese",
+ "ojs": "Severn Ojibwa",
+ "ojv": "Ontong Java",
+ "ojw": "Western Ojibwa",
+ "oka": "Okanagan",
+ "okb": "Okobo",
+ "okc": "Kobo",
+ "okd": "Okodia",
+ "oke": "Okpe (Southwestern Edo)",
+ "okg": "Koko Babangk",
+ "okh": "Koresh-e Rostam",
+ "oki": "Okiek",
+ "okj": "Oko-Juwoi",
+ "okk": "Kwamtim One",
+ "okl": "Old Kentish Sign Language",
+ "okm": "Middle Korean (10th-16th cent.)",
+ "okn": "Oki-No-Erabu",
+ "oko": "Old Korean (3rd-9th cent.)",
+ "okr": "Kirike",
+ "oks": "Oko-Eni-Osayen",
+ "oku": "Oku",
+ "okv": "Orokaiva",
+ "okx": "Okpe (Northwestern Edo)",
+ "okz": "Old Khmer",
+ "ola": "Walungge",
+ "old": "Mochi",
+ "ole": "Olekha",
+ "olk": "Olkol",
+ "olm": "Oloma",
+ "olo": "Livvi",
+ "olr": "Olrat",
+ "olt": "Old Lithuanian",
+ "olu": "Kuvale",
+ "om": "Oromo",
+ "oma": "Omaha-Ponca",
+ "omb": "East Ambae",
+ "omc": "Mochica",
+ "omg": "Omagua",
+ "omi": "Omi",
+ "omk": "Omok",
+ "oml": "Ombo",
+ "omn": "Minoan",
+ "omo": "Utarmbung",
+ "omp": "Old Manipuri",
+ "omq": "Oto-Manguean languages",
+ "omr": "Old Marathi",
+ "omt": "Omotik",
+ "omu": "Omurano",
+ "omv": "Omotic languages",
+ "omw": "South Tairora",
+ "omx": "Old Mon",
+ "omy": "Old Malay",
+ "ona": "Ona",
+ "onb": "Lingao",
+ "one": "Oneida",
+ "ong": "Olo",
+ "oni": "Onin",
+ "onj": "Onjob",
+ "onk": "Kabore One",
+ "onn": "Onobasulu",
+ "ono": "Onondaga",
+ "onp": "Sartang",
+ "onr": "Northern One",
+ "ons": "Ono",
+ "ont": "Ontenu",
+ "onu": "Unua",
+ "onw": "Old Nubian",
+ "onx": "Onin Based Pidgin",
+ "ood": "Tohono O'odham",
+ "oog": "Ong",
+ "oon": "Önge",
+ "oor": "Oorlams",
+ "oos": "Old Ossetic",
+ "opa": "Okpamheri",
+ "opk": "Kopkaka",
+ "opm": "Oksapmin",
+ "opo": "Opao",
+ "opt": "Opata",
+ "opy": "Ofayé",
+ "or": "Oriya (macrolanguage); Odia (macrolanguage)",
+ "ora": "Oroha",
+ "orc": "Orma",
+ "ore": "Orejón",
+ "org": "Oring",
+ "orh": "Oroqen",
+ "orn": "Orang Kanaq",
+ "oro": "Orokolo",
+ "orr": "Oruma",
+ "ors": "Orang Seletar",
+ "ort": "Adivasi Oriya",
+ "oru": "Ormuri",
+ "orv": "Old Russian",
+ "orw": "Oro Win",
+ "orx": "Oro",
+ "ory": "Odia (individual language); Oriya (individual language)",
+ "orz": "Ormu",
+ "os": "Ossetian; Ossetic",
+ "osa": "Osage",
+ "osc": "Oscan",
+ "osi": "Osing",
+ "osn": "Old Sundanese",
+ "oso": "Ososo",
+ "osp": "Old Spanish",
+ "ost": "Osatu",
+ "osu": "Southern One",
+ "osx": "Old Saxon",
+ "ota": "Ottoman Turkish (1500-1928)",
+ "otb": "Old Tibetan",
+ "otd": "Ot Danum",
+ "ote": "Mezquital Otomi",
+ "oti": "Oti",
+ "otk": "Old Turkish",
+ "otl": "Tilapa Otomi",
+ "otm": "Eastern Highland Otomi",
+ "otn": "Tenango Otomi",
+ "oto": "Otomian languages",
+ "otq": "Querétaro Otomi",
+ "otr": "Otoro",
+ "ots": "Estado de México Otomi",
+ "ott": "Temoaya Otomi",
+ "otu": "Otuke",
+ "otw": "Ottawa",
+ "otx": "Texcatepec Otomi",
+ "oty": "Old Tamil",
+ "otz": "Ixtenco Otomi",
+ "oua": "Tagargrent",
+ "oub": "Glio-Oubi",
+ "oue": "Oune",
+ "oui": "Old Uighur",
+ "oum": "Ouma",
+ "ovd": "Elfdalian; Övdalian",
+ "owi": "Owiniga",
+ "owl": "Old Welsh",
+ "oyb": "Oy",
+ "oyd": "Oyda",
+ "oym": "Wayampi",
+ "oyy": "Oya'oya",
+ "ozm": "Koonzime",
+ "pa": "Panjabi; Punjabi",
+ "paa": "Papuan languages",
+ "pab": "Parecís",
+ "pac": "Pacoh",
+ "pad": "Paumarí",
+ "pae": "Pagibete",
+ "paf": "Paranawát",
+ "pag": "Pangasinan",
+ "pah": "Tenharim",
+ "pai": "Pe",
+ "pak": "Parakanã",
+ "pal": "Pahlavi",
+ "pam": "Pampanga; Kapampangan",
+ "pao": "Northern Paiute",
+ "pap": "Papiamento",
+ "paq": "Parya",
+ "par": "Panamint; Timbisha",
+ "pas": "Papasena",
+ "pau": "Palauan",
+ "pav": "Pakaásnovos",
+ "paw": "Pawnee",
+ "pax": "Pankararé",
+ "pay": "Pech",
+ "paz": "Pankararú",
+ "pbb": "Páez",
+ "pbc": "Patamona",
+ "pbe": "Mezontla Popoloca",
+ "pbf": "Coyotepec Popoloca",
+ "pbg": "Paraujano",
+ "pbh": "E'ñapa Woromaipu",
+ "pbi": "Parkwa",
+ "pbl": "Mak (Nigeria)",
+ "pbm": "Puebla Mazatec",
+ "pbn": "Kpasam",
+ "pbo": "Papel",
+ "pbp": "Badyara",
+ "pbr": "Pangwa",
+ "pbs": "Central Pame",
+ "pbt": "Southern Pashto",
+ "pbu": "Northern Pashto",
+ "pbv": "Pnar",
+ "pby": "Pyu (Papua New Guinea)",
+ "pca": "Santa Inés Ahuatempan Popoloca",
+ "pcb": "Pear",
+ "pcc": "Bouyei",
+ "pcd": "Picard",
+ "pce": "Ruching Palaung",
+ "pcf": "Paliyan",
+ "pcg": "Paniya",
+ "pch": "Pardhan",
+ "pci": "Duruwa",
+ "pcj": "Parenga",
+ "pck": "Paite Chin",
+ "pcl": "Pardhi",
+ "pcm": "Nigerian Pidgin",
+ "pcn": "Piti",
+ "pcp": "Pacahuara",
+ "pcw": "Pyapun",
+ "pda": "Anam",
+ "pdc": "Pennsylvania German",
+ "pdi": "Pa Di",
+ "pdn": "Podena; Fedan",
+ "pdo": "Padoe",
+ "pdt": "Plautdietsch",
+ "pdu": "Kayan",
+ "pea": "Peranakan Indonesian",
+ "peb": "Eastern Pomo",
+ "ped": "Mala (Papua New Guinea)",
+ "pee": "Taje",
+ "pef": "Northeastern Pomo",
+ "peg": "Pengo",
+ "peh": "Bonan",
+ "pei": "Chichimeca-Jonaz",
+ "pej": "Northern Pomo",
+ "pek": "Penchal",
+ "pel": "Pekal",
+ "pem": "Phende",
+ "peo": "Old Persian (ca. 600-400 B.C.)",
+ "pep": "Kunja",
+ "peq": "Southern Pomo",
+ "pes": "Iranian Persian",
+ "pev": "Pémono",
+ "pex": "Petats",
+ "pey": "Petjo",
+ "pez": "Eastern Penan",
+ "pfa": "Pááfang",
+ "pfe": "Pere",
+ "pfl": "Pfaelzisch",
+ "pga": "Sudanese Creole Arabic",
+ "pgd": "Gāndhārī",
+ "pgg": "Pangwali",
+ "pgi": "Pagi",
+ "pgk": "Rerep",
+ "pgl": "Primitive Irish",
+ "pgn": "Paelignian",
+ "pgs": "Pangseng",
+ "pgu": "Pagu",
+ "pgz": "Papua New Guinean Sign Language",
+ "pha": "Pa-Hng",
+ "phd": "Phudagi",
+ "phg": "Phuong",
+ "phh": "Phukha",
+ "phi": "Philippine languages",
+ "phj": "Pahari",
+ "phk": "Phake",
+ "phl": "Phalura; Palula",
+ "phm": "Phimbi",
+ "phn": "Phoenician",
+ "pho": "Phunoi",
+ "phq": "Phana'",
+ "phr": "Pahari-Potwari",
+ "pht": "Phu Thai",
+ "phu": "Phuan",
+ "phv": "Pahlavani",
+ "phw": "Phangduwali",
+ "pi": "Pali",
+ "pia": "Pima Bajo",
+ "pib": "Yine",
+ "pic": "Pinji",
+ "pid": "Piaroa",
+ "pie": "Piro",
+ "pif": "Pingelapese",
+ "pig": "Pisabo",
+ "pih": "Pitcairn-Norfolk",
+ "pij": "Pijao",
+ "pil": "Yom",
+ "pim": "Powhatan",
+ "pin": "Piame",
+ "pio": "Piapoco",
+ "pip": "Pero",
+ "pir": "Piratapuyo",
+ "pis": "Pijin",
+ "pit": "Pitta Pitta",
+ "piu": "Pintupi-Luritja",
+ "piv": "Pileni; Vaeakau-Taumako",
+ "piw": "Pimbwe",
+ "pix": "Piu",
+ "piy": "Piya-Kwonci",
+ "piz": "Pije",
+ "pjt": "Pitjantjatjara",
+ "pka": "Ardhamāgadhī Prākrit",
+ "pkb": "Pokomo; Kipfokomo",
+ "pkc": "Paekche",
+ "pkg": "Pak-Tong",
+ "pkh": "Pankhu",
+ "pkn": "Pakanha",
+ "pko": "Pökoot",
+ "pkp": "Pukapuka",
+ "pkr": "Attapady Kurumba",
+ "pks": "Pakistan Sign Language",
+ "pkt": "Maleng",
+ "pku": "Paku",
+ "pl": "Polish",
+ "pla": "Miani",
+ "plb": "Polonombauk",
+ "plc": "Central Palawano",
+ "pld": "Polari",
+ "ple": "Palu'e",
+ "plf": "Central Malayo-Polynesian languages",
+ "plg": "Pilagá",
+ "plh": "Paulohi",
+ "plj": "Polci",
+ "plk": "Kohistani Shina",
+ "pll": "Shwe Palaung",
+ "pln": "Palenquero",
+ "plo": "Oluta Popoluca",
+ "plq": "Palaic",
+ "plr": "Palaka Senoufo",
+ "pls": "San Marcos Tlacoyalco Popoloca; San Marcos Tlalcoyalco Popoloca",
+ "plt": "Plateau Malagasy",
+ "plu": "Palikúr",
+ "plv": "Southwest Palawano",
+ "plw": "Brooke's Point Palawano",
+ "ply": "Bolyu",
+ "plz": "Paluan",
+ "pma": "Paama",
+ "pmb": "Pambia",
+ "pmd": "Pallanganmiddang",
+ "pme": "Pwaamei",
+ "pmf": "Pamona",
+ "pmh": "Māhārāṣṭri Prākrit",
+ "pmi": "Northern Pumi",
+ "pmj": "Southern Pumi",
+ "pmk": "Pamlico",
+ "pml": "Lingua Franca",
+ "pmm": "Pomo",
+ "pmn": "Pam",
+ "pmo": "Pom",
+ "pmq": "Northern Pame",
+ "pmr": "Paynamar",
+ "pms": "Piemontese",
+ "pmt": "Tuamotuan",
+ "pmw": "Plains Miwok",
+ "pmx": "Poumei Naga",
+ "pmy": "Papuan Malay",
+ "pmz": "Southern Pame",
+ "pna": "Punan Bah-Biau",
+ "pnb": "Western Panjabi",
+ "pnc": "Pannei",
+ "pnd": "Mpinda",
+ "pne": "Western Penan",
+ "png": "Pangu; Pongu",
+ "pnh": "Penrhyn",
+ "pni": "Aoheng",
+ "pnj": "Pinjarup",
+ "pnk": "Paunaka",
+ "pnl": "Paleni",
+ "pnm": "Punan Batu 1",
+ "pnn": "Pinai-Hagahai",
+ "pno": "Panobo",
+ "pnp": "Pancana",
+ "pnq": "Pana (Burkina Faso)",
+ "pnr": "Panim",
+ "pns": "Ponosakan",
+ "pnt": "Pontic",
+ "pnu": "Jiongnai Bunu",
+ "pnv": "Pinigura",
+ "pnw": "Banyjima; Panytyima",
+ "pnx": "Phong-Kniang",
+ "pny": "Pinyin",
+ "pnz": "Pana (Central African Republic)",
+ "poc": "Poqomam",
+ "poe": "San Juan Atzingo Popoloca",
+ "pof": "Poke",
+ "pog": "Potiguára",
+ "poh": "Poqomchi'",
+ "poi": "Highland Popoluca",
+ "pok": "Pokangá",
+ "pom": "Southeastern Pomo",
+ "pon": "Pohnpeian",
+ "poo": "Central Pomo",
+ "pop": "Pwapwâ",
+ "poq": "Texistepec Popoluca",
+ "pos": "Sayula Popoluca",
+ "pot": "Potawatomi",
+ "pov": "Upper Guinea Crioulo",
+ "pow": "San Felipe Otlaltepec Popoloca",
+ "pox": "Polabian",
+ "poy": "Pogolo",
+ "poz": "Malayo-Polynesian languages",
+ "ppe": "Papi",
+ "ppi": "Paipai",
+ "ppk": "Uma",
+ "ppl": "Pipil; Nicarao",
+ "ppm": "Papuma",
+ "ppn": "Papapana",
+ "ppo": "Folopa",
+ "ppp": "Pelende",
+ "ppq": "Pei",
+ "pps": "San Luís Temalacayuca Popoloca",
+ "ppt": "Pare",
+ "ppu": "Papora",
+ "pqa": "Pa'a",
+ "pqe": "Eastern Malayo-Polynesian languages",
+ "pqm": "Malecite-Passamaquoddy",
+ "pqw": "Western Malayo-Polynesian languages",
+ "pra": "Prakrit languages",
+ "prc": "Parachi",
+ "prd": "Parsi-Dari",
+ "pre": "Principense",
+ "prf": "Paranan",
+ "prg": "Prussian",
+ "prh": "Porohanon",
+ "pri": "Paicî",
+ "prk": "Parauk",
+ "prl": "Peruvian Sign Language",
+ "prm": "Kibiri",
+ "prn": "Prasuni",
+ "pro": "Old Provençal (to 1500); Old Occitan (to 1500)",
+ "prp": "Parsi",
+ "prq": "Ashéninka Perené",
+ "prr": "Puri",
+ "prs": "Dari; Afghan Persian",
+ "prt": "Phai",
+ "pru": "Puragi",
+ "prw": "Parawen",
+ "prx": "Purik",
+ "prz": "Providencia Sign Language",
+ "ps": "Pushto; Pashto",
+ "psa": "Asue Awyu",
+ "psc": "Iranian Sign Language; Persian Sign Language",
+ "psd": "Plains Indian Sign Language",
+ "pse": "Central Malay",
+ "psg": "Penang Sign Language",
+ "psh": "Southwest Pashai; Southwest Pashayi",
+ "psi": "Southeast Pashai; Southeast Pashayi",
+ "psl": "Puerto Rican Sign Language",
+ "psm": "Pauserna",
+ "psn": "Panasuan",
+ "pso": "Polish Sign Language",
+ "psp": "Philippine Sign Language",
+ "psq": "Pasi",
+ "psr": "Portuguese Sign Language",
+ "pss": "Kaulong",
+ "pst": "Central Pashto",
+ "psu": "Sauraseni Prākrit",
+ "psw": "Port Sandwich",
+ "psy": "Piscataway",
+ "pt": "Portuguese",
+ "pta": "Pai Tavytera",
+ "pth": "Pataxó Hã-Ha-Hãe",
+ "pti": "Pindiini; Wangkatha",
+ "ptn": "Patani",
+ "pto": "Zo'é",
+ "ptp": "Patep",
+ "ptq": "Pattapu",
+ "ptr": "Piamatsina",
+ "ptt": "Enrekang",
+ "ptu": "Bambam",
+ "ptv": "Port Vato",
+ "ptw": "Pentlatch",
+ "pty": "Pathiya",
+ "pua": "Western Highland Purepecha",
+ "pub": "Purum",
+ "puc": "Punan Merap",
+ "pud": "Punan Aput",
+ "pue": "Puelche",
+ "puf": "Punan Merah",
+ "pug": "Phuie",
+ "pui": "Puinave",
+ "puj": "Punan Tubu",
+ "pum": "Puma",
+ "puo": "Puoc",
+ "pup": "Pulabu",
+ "puq": "Puquina",
+ "pur": "Puruborá",
+ "put": "Putoh",
+ "puu": "Punu",
+ "puw": "Puluwatese",
+ "pux": "Puare",
+ "puy": "Purisimeño",
+ "pwa": "Pawaia",
+ "pwb": "Panawa",
+ "pwg": "Gapapaiwa",
+ "pwi": "Patwin",
+ "pwm": "Molbog",
+ "pwn": "Paiwan",
+ "pwo": "Pwo Western Karen",
+ "pwr": "Powari",
+ "pww": "Pwo Northern Karen",
+ "pxm": "Quetzaltepec Mixe",
+ "pye": "Pye Krumen",
+ "pym": "Fyam",
+ "pyn": "Poyanáwa",
+ "pys": "Paraguayan Sign Language; Lengua de Señas del Paraguay",
+ "pyu": "Puyuma",
+ "pyx": "Pyu (Myanmar)",
+ "pyy": "Pyen",
+ "pzh": "Pazeh",
+ "pzn": "Jejara Naga; Para Naga",
+ "qu": "Quechua",
+ "qua": "Quapaw",
+ "qub": "Huallaga Huánuco Quechua",
+ "quc": "K'iche'; Quiché",
+ "qud": "Calderón Highland Quichua",
+ "quf": "Lambayeque Quechua",
+ "qug": "Chimborazo Highland Quichua",
+ "quh": "South Bolivian Quechua",
+ "qui": "Quileute",
+ "quk": "Chachapoyas Quechua",
+ "qul": "North Bolivian Quechua",
+ "qum": "Sipacapense",
+ "qun": "Quinault",
+ "qup": "Southern Pastaza Quechua",
+ "quq": "Quinqui",
+ "qur": "Yanahuanca Pasco Quechua",
+ "qus": "Santiago del Estero Quichua",
+ "quv": "Sacapulteco",
+ "quw": "Tena Lowland Quichua",
+ "qux": "Yauyos Quechua",
+ "quy": "Ayacucho Quechua",
+ "quz": "Cusco Quechua",
+ "qva": "Ambo-Pasco Quechua",
+ "qvc": "Cajamarca Quechua",
+ "qve": "Eastern Apurímac Quechua",
+ "qvh": "Huamalíes-Dos de Mayo Huánuco Quechua",
+ "qvi": "Imbabura Highland Quichua",
+ "qvj": "Loja Highland Quichua",
+ "qvl": "Cajatambo North Lima Quechua",
+ "qvm": "Margos-Yarowilca-Lauricocha Quechua",
+ "qvn": "North Junín Quechua",
+ "qvo": "Napo Lowland Quechua",
+ "qvp": "Pacaraos Quechua",
+ "qvs": "San Martín Quechua",
+ "qvw": "Huaylla Wanca Quechua",
+ "qvy": "Queyu",
+ "qvz": "Northern Pastaza Quichua",
+ "qwa": "Corongo Ancash Quechua",
+ "qwc": "Classical Quechua",
+ "qwe": "Quechuan (family)",
+ "qwh": "Huaylas Ancash Quechua",
+ "qwm": "Kuman (Russia)",
+ "qws": "Sihuas Ancash Quechua",
+ "qwt": "Kwalhioqua-Tlatskanai",
+ "qxa": "Chiquián Ancash Quechua",
+ "qxc": "Chincha Quechua",
+ "qxh": "Panao Huánuco Quechua",
+ "qxl": "Salasaca Highland Quichua",
+ "qxn": "Northern Conchucos Ancash Quechua",
+ "qxo": "Southern Conchucos Ancash Quechua",
+ "qxp": "Puno Quechua",
+ "qxq": "Qashqa'i",
+ "qxr": "Cañar Highland Quichua",
+ "qxs": "Southern Qiang",
+ "qxt": "Santa Ana de Tusi Pasco Quechua",
+ "qxu": "Arequipa-La Unión Quechua",
+ "qxw": "Jauja Wanca Quechua",
+ "qya": "Quenya",
+ "qyp": "Quiripi",
+ "raa": "Dungmali",
+ "rab": "Camling",
+ "rac": "Rasawa",
+ "rad": "Rade",
+ "raf": "Western Meohang",
+ "rag": "Logooli; Lulogooli",
+ "rah": "Rabha",
+ "rai": "Ramoaaina",
+ "raj": "Rajasthani",
+ "rak": "Tulu-Bohuai",
+ "ral": "Ralte",
+ "ram": "Canela",
+ "ran": "Riantana",
+ "rao": "Rao",
+ "rap": "Rapanui",
+ "raq": "Saam",
+ "rar": "Rarotongan; Cook Islands Maori",
+ "ras": "Tegali",
+ "rat": "Razajerdi",
+ "rau": "Raute",
+ "rav": "Sampang",
+ "raw": "Rawang",
+ "rax": "Rang",
+ "ray": "Rapa",
+ "raz": "Rahambuu",
+ "rbb": "Rumai Palaung",
+ "rbk": "Northern Bontok",
+ "rbl": "Miraya Bikol",
+ "rbp": "Barababaraba",
+ "rcf": "Réunion Creole French",
+ "rdb": "Rudbari",
+ "rea": "Rerau",
+ "reb": "Rembong",
+ "ree": "Rejang Kayan",
+ "reg": "Kara (Tanzania)",
+ "rei": "Reli",
+ "rej": "Rejang",
+ "rel": "Rendille",
+ "rem": "Remo",
+ "ren": "Rengao",
+ "rer": "Rer Bare",
+ "res": "Reshe",
+ "ret": "Retta",
+ "rey": "Reyesano",
+ "rga": "Roria",
+ "rge": "Romano-Greek",
+ "rgk": "Rangkas",
+ "rgn": "Romagnol",
+ "rgr": "Resígaro",
+ "rgs": "Southern Roglai",
+ "rgu": "Ringgou",
+ "rhg": "Rohingya",
+ "rhp": "Yahang",
+ "ria": "Riang (India)",
+ "rib": "Bribri Sign Language",
+ "rif": "Tarifit",
+ "ril": "Riang Lang; Riang (Myanmar)",
+ "rim": "Nyaturu",
+ "rin": "Nungu",
+ "rir": "Ribun",
+ "rit": "Ritharrngu",
+ "riu": "Riung",
+ "rjg": "Rajong",
+ "rji": "Raji",
+ "rjs": "Rajbanshi",
+ "rka": "Kraol",
+ "rkb": "Rikbaktsa",
+ "rkh": "Rakahanga-Manihiki",
+ "rki": "Rakhine",
+ "rkm": "Marka",
+ "rkt": "Rangpuri; Kamta",
+ "rkw": "Arakwal",
+ "rm": "Romansh",
+ "rma": "Rama",
+ "rmb": "Rembarrnga",
+ "rmc": "Carpathian Romani",
+ "rmd": "Traveller Danish",
+ "rme": "Angloromani",
+ "rmf": "Kalo Finnish Romani",
+ "rmg": "Traveller Norwegian",
+ "rmh": "Murkim",
+ "rmi": "Lomavren",
+ "rmk": "Romkun",
+ "rml": "Baltic Romani",
+ "rmm": "Roma",
+ "rmn": "Balkan Romani",
+ "rmo": "Sinte Romani",
+ "rmp": "Rempi",
+ "rmq": "Caló",
+ "rms": "Romanian Sign Language",
+ "rmt": "Domari",
+ "rmu": "Tavringer Romani",
+ "rmv": "Romanova",
+ "rmw": "Welsh Romani",
+ "rmx": "Romam",
+ "rmy": "Vlax Romani",
+ "rmz": "Marma",
+ "rn": "Rundi",
+ "rnb": "Brunca Sign Language",
+ "rnd": "Ruund",
+ "rng": "Ronga",
+ "rnl": "Ranglong",
+ "rnn": "Roon",
+ "rnp": "Rongpo",
+ "rnr": "Nari Nari",
+ "rnw": "Rungwa",
+ "ro": "Romanian; Moldavian; Moldovan",
+ "roa": "Romance languages",
+ "rob": "Tae'",
+ "roc": "Cacgia Roglai",
+ "rod": "Rogo",
+ "roe": "Ronji",
+ "rof": "Rombo",
+ "rog": "Northern Roglai",
+ "rol": "Romblomanon",
+ "rom": "Romany",
+ "roo": "Rotokas",
+ "rop": "Kriol",
+ "ror": "Rongga",
+ "rou": "Runga",
+ "row": "Dela-Oenale",
+ "rpn": "Repanbitip",
+ "rpt": "Rapting",
+ "rri": "Ririo",
+ "rro": "Waima",
+ "rrt": "Arritinngithigh",
+ "rsb": "Romano-Serbian",
+ "rsk": "Ruthenian; Rusyn",
+ "rsl": "Russian Sign Language",
+ "rsm": "Miriwoong Sign Language",
+ "rsn": "Rwandan Sign Language",
+ "rtc": "Rungtu Chin",
+ "rth": "Ratahan",
+ "rtm": "Rotuman",
+ "rts": "Yurats",
+ "rtw": "Rathawi",
+ "ru": "Russian",
+ "rub": "Gungu",
+ "ruc": "Ruuli",
+ "rue": "Rusyn",
+ "ruf": "Luguru",
+ "rug": "Roviana",
+ "ruh": "Ruga",
+ "rui": "Rufiji",
+ "ruk": "Che",
+ "ruo": "Istro Romanian",
+ "rup": "Macedo-Romanian; Aromanian; Arumanian",
+ "ruq": "Megleno Romanian",
+ "rut": "Rutul",
+ "ruu": "Lanas Lobu",
+ "ruy": "Mala (Nigeria)",
+ "ruz": "Ruma",
+ "rw": "Kinyarwanda",
+ "rwa": "Rawo",
+ "rwk": "Rwa",
+ "rwl": "Ruwila",
+ "rwm": "Amba (Uganda)",
+ "rwo": "Rawa",
+ "rwr": "Marwari (India)",
+ "rxd": "Ngardi",
+ "rxw": "Karuwali; Garuwali",
+ "ryn": "Northern Amami-Oshima",
+ "rys": "Yaeyama",
+ "ryu": "Central Okinawan",
+ "rzh": "Rāziḥī",
+ "sa": "Sanskrit",
+ "saa": "Saba",
+ "sab": "Buglere",
+ "sac": "Meskwaki",
+ "sad": "Sandawe",
+ "sae": "Sabanê",
+ "saf": "Safaliba",
+ "sah": "Yakut",
+ "sai": "South American Indian languages",
+ "saj": "Sahu",
+ "sak": "Sake",
+ "sal": "Salishan languages",
+ "sam": "Samaritan Aramaic",
+ "sao": "Sause",
+ "saq": "Samburu",
+ "sar": "Saraveca",
+ "sas": "Sasak",
+ "sat": "Santali",
+ "sau": "Saleman",
+ "sav": "Saafi-Saafi",
+ "saw": "Sawi",
+ "sax": "Sa",
+ "say": "Saya",
+ "saz": "Saurashtra",
+ "sba": "Ngambay",
+ "sbb": "Simbo",
+ "sbc": "Kele (Papua New Guinea)",
+ "sbd": "Southern Samo",
+ "sbe": "Saliba",
+ "sbf": "Chabu; Shabo",
+ "sbg": "Seget",
+ "sbh": "Sori-Harengan",
+ "sbi": "Seti",
+ "sbj": "Surbakhal",
+ "sbk": "Safwa",
+ "sbl": "Botolan Sambal",
+ "sbm": "Sagala",
+ "sbn": "Sindhi Bhil",
+ "sbo": "Sabüm",
+ "sbp": "Sangu (Tanzania)",
+ "sbq": "Sileibi",
+ "sbr": "Sembakung Murut",
+ "sbs": "Subiya",
+ "sbt": "Kimki",
+ "sbu": "Stod Bhoti",
+ "sbv": "Sabine",
+ "sbw": "Simba",
+ "sbx": "Seberuang",
+ "sby": "Soli",
+ "sbz": "Sara Kaba",
+ "sc": "Sardinian",
+ "scb": "Chut",
+ "sce": "Dongxiang",
+ "scf": "San Miguel Creole French",
+ "scg": "Sanggau",
+ "sch": "Sakachep",
+ "sci": "Sri Lankan Creole Malay",
+ "sck": "Sadri",
+ "scl": "Shina",
+ "scn": "Sicilian",
+ "sco": "Scots",
+ "scp": "Hyolmo; Helambu Sherpa",
+ "scq": "Sa'och",
+ "scs": "North Slavey",
+ "sct": "Southern Katang",
+ "scu": "Shumcho",
+ "scv": "Sheni",
+ "scw": "Sha",
+ "scx": "Sicel",
+ "sd": "Sindhi",
+ "sda": "Toraja-Sa'dan",
+ "sdb": "Shabak",
+ "sdc": "Sassarese Sardinian",
+ "sde": "Surubu",
+ "sdf": "Sarli",
+ "sdg": "Savi",
+ "sdh": "Southern Kurdish",
+ "sdj": "Suundi",
+ "sdk": "Sos Kundi",
+ "sdl": "Saudi Arabian Sign Language",
+ "sdn": "Gallurese Sardinian",
+ "sdo": "Bukar-Sadung Bidayuh",
+ "sdp": "Sherdukpen",
+ "sdq": "Semandang",
+ "sdr": "Oraon Sadri",
+ "sds": "Sened",
+ "sdt": "Shuadit",
+ "sdu": "Sarudu",
+ "sdv": "Eastern Sudanic languages",
+ "sdx": "Sibu Melanau",
+ "sdz": "Sallands",
+ "se": "Northern Sami",
+ "sea": "Semai",
+ "seb": "Shempire Senoufo",
+ "sec": "Sechelt",
+ "sed": "Sedang",
+ "see": "Seneca",
+ "sef": "Cebaara Senoufo",
+ "seg": "Segeju",
+ "seh": "Sena",
+ "sei": "Seri",
+ "sej": "Sene",
+ "sek": "Sekani",
+ "sel": "Selkup",
+ "sem": "Semitic languages",
+ "sen": "Nanerigé Sénoufo",
+ "seo": "Suarmin",
+ "sep": "Sìcìté Sénoufo",
+ "seq": "Senara Sénoufo",
+ "ser": "Serrano",
+ "ses": "Koyraboro Senni Songhai",
+ "set": "Sentani",
+ "seu": "Serui-Laut",
+ "sev": "Nyarafolo Senoufo",
+ "sew": "Sewa Bay",
+ "sey": "Secoya",
+ "sez": "Senthang Chin",
+ "sfb": "Langue des signes de Belgique Francophone; French Belgian Sign Language",
+ "sfe": "Eastern Subanen",
+ "sfm": "Small Flowery Miao",
+ "sfs": "South African Sign Language",
+ "sfw": "Sehwi",
+ "sg": "Sango",
+ "sga": "Old Irish (to 900)",
+ "sgb": "Mag-antsi Ayta",
+ "sgc": "Kipsigis",
+ "sgd": "Surigaonon",
+ "sge": "Segai",
+ "sgg": "Swiss-German Sign Language",
+ "sgh": "Shughni",
+ "sgi": "Suga",
+ "sgj": "Surgujia",
+ "sgk": "Sangkong",
+ "sgm": "Singa",
+ "sgn": "Sign languages",
+ "sgp": "Singpho",
+ "sgr": "Sangisari",
+ "sgs": "Samogitian",
+ "sgt": "Brokpake",
+ "sgu": "Salas",
+ "sgw": "Sebat Bet Gurage",
+ "sgx": "Sierra Leone Sign Language",
+ "sgy": "Sanglechi",
+ "sgz": "Sursurunga",
+ "sh": "Serbo-Croatian",
+ "sha": "Shall-Zwall",
+ "shb": "Ninam",
+ "shc": "Sonde",
+ "shd": "Kundal Shahi",
+ "she": "Sheko",
+ "shg": "Shua",
+ "shh": "Shoshoni",
+ "shi": "Tachelhit",
+ "shj": "Shatt",
+ "shk": "Shilluk",
+ "shl": "Shendu",
+ "shm": "Shahrudi",
+ "shn": "Shan",
+ "sho": "Shanga",
+ "shp": "Shipibo-Conibo",
+ "shq": "Sala",
+ "shr": "Shi",
+ "shs": "Shuswap",
+ "sht": "Shasta",
+ "shu": "Chadian Arabic",
+ "shv": "Shehri",
+ "shw": "Shwai",
+ "shx": "She",
+ "shy": "Tachawit",
+ "shz": "Syenara Senoufo",
+ "si": "Sinhala; Sinhalese",
+ "sia": "Akkala Sami",
+ "sib": "Sebop",
+ "sid": "Sidamo",
+ "sie": "Simaa",
+ "sif": "Siamou",
+ "sig": "Paasaal",
+ "sih": "Zire; Sîshëë",
+ "sii": "Shom Peng",
+ "sij": "Numbami",
+ "sik": "Sikiana",
+ "sil": "Tumulung Sisaala",
+ "sim": "Mende (Papua New Guinea)",
+ "sio": "Siouan languages",
+ "sip": "Sikkimese",
+ "siq": "Sonia",
+ "sir": "Siri",
+ "sis": "Siuslaw",
+ "sit": "Sino-Tibetan languages",
+ "siu": "Sinagen",
+ "siv": "Sumariup",
+ "siw": "Siwai",
+ "six": "Sumau",
+ "siy": "Sivandi",
+ "siz": "Siwi",
+ "sja": "Epena",
+ "sjb": "Sajau Basap",
+ "sjd": "Kildin Sami",
+ "sje": "Pite Sami",
+ "sjg": "Assangori",
+ "sjk": "Kemi Sami",
+ "sjl": "Sajalong; Miji",
+ "sjm": "Mapun",
+ "sjn": "Sindarin",
+ "sjo": "Xibe",
+ "sjp": "Surjapuri",
+ "sjr": "Siar-Lak",
+ "sjs": "Senhaja De Srair",
+ "sjt": "Ter Sami",
+ "sju": "Ume Sami",
+ "sjw": "Shawnee",
+ "sk": "Slovak",
+ "ska": "Skagit",
+ "skb": "Saek",
+ "skc": "Ma Manda",
+ "skd": "Southern Sierra Miwok",
+ "ske": "Seke (Vanuatu)",
+ "skf": "Sakirabiá",
+ "skg": "Sakalava Malagasy",
+ "skh": "Sikule",
+ "ski": "Sika",
+ "skj": "Seke (Nepal)",
+ "skm": "Kutong",
+ "skn": "Kolibugan Subanon",
+ "sko": "Seko Tengah",
+ "skp": "Sekapan",
+ "skq": "Sininkere",
+ "skr": "Saraiki; Seraiki",
+ "sks": "Maia",
+ "skt": "Sakata",
+ "sku": "Sakao",
+ "skv": "Skou",
+ "skw": "Skepi Creole Dutch",
+ "skx": "Seko Padang",
+ "sky": "Sikaiana",
+ "skz": "Sekar",
+ "sl": "Slovenian",
+ "sla": "Slavic languages",
+ "slc": "Sáliba",
+ "sld": "Sissala",
+ "sle": "Sholaga",
+ "slf": "Swiss-Italian Sign Language",
+ "slg": "Selungai Murut",
+ "slh": "Southern Puget Sound Salish",
+ "sli": "Lower Silesian",
+ "slj": "Salumá",
+ "sll": "Salt-Yui",
+ "slm": "Pangutaran Sama",
+ "sln": "Salinan",
+ "slp": "Lamaholot",
+ "slq": "Salchuq",
+ "slr": "Salar",
+ "sls": "Singapore Sign Language",
+ "slt": "Sila",
+ "slu": "Selaru",
+ "slw": "Sialum",
+ "slx": "Salampasu",
+ "sly": "Selayar",
+ "slz": "Ma'ya",
+ "sm": "Samoan",
+ "sma": "Southern Sami",
+ "smb": "Simbari",
+ "smc": "Som",
+ "smf": "Auwe",
+ "smg": "Simbali",
+ "smh": "Samei",
+ "smi": "Sami languages",
+ "smj": "Lule Sami",
+ "smk": "Bolinao",
+ "sml": "Central Sama",
+ "smm": "Musasa",
+ "smn": "Inari Sami",
+ "smp": "Samaritan",
+ "smq": "Samo",
+ "smr": "Simeulue",
+ "sms": "Skolt Sami",
+ "smt": "Simte",
+ "smu": "Somray",
+ "smv": "Samvedi",
+ "smw": "Sumbawa",
+ "smx": "Samba",
+ "smy": "Semnani",
+ "smz": "Simeku",
+ "sn": "Shona",
+ "snc": "Sinaugoro",
+ "sne": "Bau Bidayuh",
+ "snf": "Noon",
+ "sng": "Sanga (Democratic Republic of Congo)",
+ "sni": "Sensi",
+ "snj": "Riverain Sango",
+ "snk": "Soninke",
+ "snl": "Sangil",
+ "snm": "Southern Ma'di",
+ "snn": "Siona",
+ "sno": "Snohomish",
+ "snp": "Siane",
+ "snq": "Sangu (Gabon)",
+ "snr": "Sihan",
+ "sns": "South West Bay; Nahavaq",
+ "snu": "Senggi; Viid",
+ "snv": "Sa'ban",
+ "snw": "Selee",
+ "snx": "Sam",
+ "sny": "Saniyo-Hiyewe",
+ "snz": "Kou",
+ "so": "Somali",
+ "soa": "Thai Song",
+ "sob": "Sobei",
+ "soc": "So (Democratic Republic of Congo)",
+ "sod": "Songoora",
+ "soe": "Songomeno",
+ "sog": "Sogdian",
+ "soh": "Aka",
+ "soi": "Sonha",
+ "soj": "Soi",
+ "sok": "Sokoro",
+ "sol": "Solos",
+ "son": "Songhai languages",
+ "soo": "Songo",
+ "sop": "Songe",
+ "soq": "Kanasi",
+ "sor": "Somrai",
+ "sos": "Seeku",
+ "sou": "Southern Thai",
+ "sov": "Sonsorol",
+ "sow": "Sowanda",
+ "sox": "Swo",
+ "soy": "Miyobe",
+ "soz": "Temi",
+ "spb": "Sepa (Indonesia)",
+ "spc": "Sapé",
+ "spd": "Saep",
+ "spe": "Sepa (Papua New Guinea)",
+ "spg": "Sian",
+ "spi": "Saponi",
+ "spk": "Sengo",
+ "spl": "Selepet",
+ "spm": "Akukem",
+ "spn": "Sanapaná",
+ "spo": "Spokane",
+ "spp": "Supyire Senoufo",
+ "spq": "Loreto-Ucayali Spanish",
+ "spr": "Saparua",
+ "sps": "Saposa",
+ "spt": "Spiti Bhoti",
+ "spu": "Sapuan",
+ "spv": "Sambalpuri; Kosli",
+ "spx": "South Picene",
+ "spy": "Sabaot",
+ "sq": "Albanian",
+ "sqa": "Shama-Sambuga",
+ "sqh": "Shau",
+ "sqj": "Albanian languages",
+ "sqk": "Albanian Sign Language",
+ "sqm": "Suma",
+ "sqn": "Susquehannock",
+ "sqo": "Sorkhei",
+ "sqq": "Sou",
+ "sqr": "Siculo Arabic",
+ "sqs": "Sri Lankan Sign Language",
+ "sqt": "Soqotri",
+ "squ": "Squamish",
+ "sqx": "Kufr Qassem Sign Language (KQSL)",
+ "sr": "Serbian",
+ "sra": "Saruga",
+ "srb": "Sora",
+ "src": "Logudorese Sardinian",
+ "sre": "Sara",
+ "srf": "Nafi",
+ "srg": "Sulod",
+ "srh": "Sarikoli",
+ "sri": "Siriano",
+ "srk": "Serudung Murut",
+ "srl": "Isirawa",
+ "srm": "Saramaccan",
+ "srn": "Sranan Tongo",
+ "sro": "Campidanese Sardinian",
+ "srq": "Sirionó",
+ "srr": "Serer",
+ "srs": "Sarsi",
+ "srt": "Sauri",
+ "sru": "Suruí",
+ "srv": "Southern Sorsoganon",
+ "srw": "Serua",
+ "srx": "Sirmauri",
+ "sry": "Sera",
+ "srz": "Shahmirzadi",
+ "ss": "Swati",
+ "ssa": "Nilo-Saharan languages",
+ "ssb": "Southern Sama",
+ "ssc": "Suba-Simbiti",
+ "ssd": "Siroi",
+ "sse": "Balangingi; Bangingih Sama",
+ "ssf": "Thao",
+ "ssg": "Seimat",
+ "ssh": "Shihhi Arabic",
+ "ssi": "Sansi",
+ "ssj": "Sausi",
+ "ssk": "Sunam",
+ "ssl": "Western Sisaala",
+ "ssm": "Semnam",
+ "ssn": "Waata",
+ "sso": "Sissano",
+ "ssp": "Spanish Sign Language",
+ "ssq": "So'a",
+ "ssr": "Swiss-French Sign Language",
+ "sss": "Sô",
+ "sst": "Sinasina",
+ "ssu": "Susuami",
+ "ssv": "Shark Bay",
+ "ssx": "Samberigi",
+ "ssy": "Saho",
+ "ssz": "Sengseng",
+ "st": "Southern Sotho",
+ "sta": "Settla",
+ "stb": "Northern Subanen",
+ "std": "Sentinel",
+ "ste": "Liana-Seti",
+ "stf": "Seta",
+ "stg": "Trieng",
+ "sth": "Shelta",
+ "sti": "Bulo Stieng",
+ "stj": "Matya Samo",
+ "stk": "Arammba",
+ "stl": "Stellingwerfs",
+ "stm": "Setaman",
+ "stn": "Owa",
+ "sto": "Stoney",
+ "stp": "Southeastern Tepehuan",
+ "stq": "Saterfriesisch",
+ "str": "Straits Salish",
+ "sts": "Shumashti",
+ "stt": "Budeh Stieng",
+ "stu": "Samtao",
+ "stv": "Silt'e",
+ "stw": "Satawalese",
+ "sty": "Siberian Tatar",
+ "su": "Sundanese",
+ "sua": "Sulka",
+ "sub": "Suku",
+ "suc": "Western Subanon",
+ "sue": "Suena",
+ "sug": "Suganga",
+ "sui": "Suki",
+ "suj": "Shubi",
+ "suk": "Sukuma",
+ "suo": "Bouni",
+ "suq": "Tirmaga-Chai Suri; Suri",
+ "sur": "Mwaghavul",
+ "sus": "Susu",
+ "sut": "Subtiaba",
+ "suv": "Puroik",
+ "suw": "Sumbwa",
+ "sux": "Sumerian",
+ "suy": "Suyá",
+ "suz": "Sunwar",
+ "sv": "Swedish",
+ "sva": "Svan",
+ "svb": "Ulau-Suain",
+ "svc": "Vincentian Creole English",
+ "sve": "Serili",
+ "svk": "Slovakian Sign Language",
+ "svm": "Slavomolisano",
+ "svs": "Savosavo",
+ "svx": "Skalvian",
+ "sw": "Swahili (macrolanguage)",
+ "swb": "Maore Comorian",
+ "swc": "Congo Swahili",
+ "swf": "Sere",
+ "swg": "Swabian",
+ "swh": "Swahili (individual language); Kiswahili",
+ "swi": "Sui",
+ "swj": "Sira",
+ "swk": "Malawi Sena",
+ "swl": "Swedish Sign Language",
+ "swm": "Samosa",
+ "swn": "Sawknah",
+ "swo": "Shanenawa",
+ "swp": "Suau",
+ "swq": "Sharwa",
+ "swr": "Saweru",
+ "sws": "Seluwasan",
+ "swt": "Sawila",
+ "swu": "Suwawa",
+ "swv": "Shekhawati",
+ "sww": "Sowa",
+ "swx": "Suruahá",
+ "swy": "Sarua",
+ "sxb": "Suba",
+ "sxc": "Sicanian",
+ "sxe": "Sighu",
+ "sxg": "Shuhi; Shixing",
+ "sxk": "Southern Kalapuya",
+ "sxl": "Selian",
+ "sxm": "Samre",
+ "sxn": "Sangir",
+ "sxo": "Sorothaptic",
+ "sxr": "Saaroa",
+ "sxs": "Sasaru",
+ "sxu": "Upper Saxon",
+ "sxw": "Saxwe Gbe",
+ "sya": "Siang",
+ "syb": "Central Subanen",
+ "syc": "Classical Syriac",
+ "syd": "Samoyedic languages",
+ "syi": "Seki",
+ "syk": "Sukur",
+ "syl": "Sylheti",
+ "sym": "Maya Samo",
+ "syn": "Senaya",
+ "syo": "Suoy",
+ "syr": "Syriac",
+ "sys": "Sinyar",
+ "syw": "Kagate",
+ "syx": "Samay",
+ "syy": "Al-Sayyid Bedouin Sign Language",
+ "sza": "Semelai",
+ "szb": "Ngalum",
+ "szc": "Semaq Beri",
+ "szd": "Seru",
+ "sze": "Seze",
+ "szg": "Sengele",
+ "szl": "Silesian",
+ "szn": "Sula",
+ "szp": "Suabo",
+ "szs": "Solomon Islands Sign Language",
+ "szv": "Isu (Fako Division)",
+ "szw": "Sawai",
+ "szy": "Sakizaya",
+ "ta": "Tamil",
+ "taa": "Lower Tanana",
+ "tab": "Tabassaran",
+ "tac": "Lowland Tarahumara",
+ "tad": "Tause",
+ "tae": "Tariana",
+ "taf": "Tapirapé",
+ "tag": "Tagoi",
+ "tai": "Tai languages",
+ "taj": "Eastern Tamang",
+ "tak": "Tala",
+ "tal": "Tal",
+ "tan": "Tangale",
+ "tao": "Yami",
+ "tap": "Taabwa",
+ "taq": "Tamasheq",
+ "tar": "Central Tarahumara",
+ "tas": "Tay Boi",
+ "tau": "Upper Tanana",
+ "tav": "Tatuyo",
+ "taw": "Tai",
+ "tax": "Tamki",
+ "tay": "Atayal",
+ "taz": "Tocho",
+ "tba": "Aikanã",
+ "tbc": "Takia",
+ "tbd": "Kaki Ae",
+ "tbe": "Tanimbili",
+ "tbf": "Mandara",
+ "tbg": "North Tairora",
+ "tbh": "Dharawal; Thurawal",
+ "tbi": "Gaam",
+ "tbj": "Tiang",
+ "tbk": "Calamian Tagbanwa",
+ "tbl": "Tboli",
+ "tbm": "Tagbu",
+ "tbn": "Barro Negro Tunebo",
+ "tbo": "Tawala",
+ "tbp": "Taworta; Diebroud",
+ "tbq": "Tibeto-Burman languages",
+ "tbr": "Tumtum",
+ "tbs": "Tanguat",
+ "tbt": "Tembo (Kitembo)",
+ "tbu": "Tubar",
+ "tbv": "Tobo",
+ "tbw": "Tagbanwa",
+ "tbx": "Kapin",
+ "tby": "Tabaru",
+ "tbz": "Ditammari",
+ "tca": "Ticuna",
+ "tcb": "Tanacross",
+ "tcc": "Datooga",
+ "tcd": "Tafi",
+ "tce": "Southern Tutchone",
+ "tcf": "Malinaltepec Me'phaa; Malinaltepec Tlapanec",
+ "tcg": "Tamagario",
+ "tch": "Turks And Caicos Creole English",
+ "tci": "Wára",
+ "tck": "Tchitchege",
+ "tcl": "Taman (Myanmar)",
+ "tcm": "Tanahmerah",
+ "tcn": "Tichurong",
+ "tco": "Taungyo",
+ "tcp": "Tawr Chin",
+ "tcq": "Kaiy",
+ "tcs": "Torres Strait Creole; Yumplatok",
+ "tct": "T'en",
+ "tcu": "Southeastern Tarahumara",
+ "tcw": "Tecpatlán Totonac",
+ "tcx": "Toda",
+ "tcy": "Tulu",
+ "tcz": "Thado Chin",
+ "tda": "Tagdal",
+ "tdb": "Panchpargania",
+ "tdc": "Emberá-Tadó",
+ "tdd": "Tai Nüa",
+ "tde": "Tiranige Diga Dogon",
+ "tdf": "Talieng",
+ "tdg": "Western Tamang",
+ "tdh": "Thulung",
+ "tdi": "Tomadino",
+ "tdj": "Tajio",
+ "tdk": "Tambas",
+ "tdl": "Sur",
+ "tdm": "Taruma",
+ "tdn": "Tondano",
+ "tdo": "Teme",
+ "tdq": "Tita",
+ "tdr": "Todrah",
+ "tds": "Doutai",
+ "tdt": "Tetun Dili",
+ "tdv": "Toro",
+ "tdx": "Tandroy-Mahafaly Malagasy",
+ "tdy": "Tadyawan",
+ "te": "Telugu",
+ "tea": "Temiar",
+ "teb": "Tetete",
+ "tec": "Terik",
+ "ted": "Tepo Krumen",
+ "tee": "Huehuetla Tepehua",
+ "tef": "Teressa",
+ "teg": "Teke-Tege",
+ "teh": "Tehuelche",
+ "tei": "Torricelli",
+ "tek": "Ibali Teke",
+ "tem": "Timne",
+ "ten": "Tama (Colombia)",
+ "teo": "Teso",
+ "tep": "Tepecano",
+ "teq": "Temein",
+ "ter": "Tereno",
+ "tes": "Tengger",
+ "tet": "Tetum",
+ "teu": "Soo",
+ "tev": "Teor",
+ "tew": "Tewa (USA)",
+ "tex": "Tennet",
+ "tey": "Tulishi",
+ "tez": "Tetserret",
+ "tfi": "Tofin Gbe",
+ "tfn": "Tanaina",
+ "tfo": "Tefaro",
+ "tfr": "Teribe",
+ "tft": "Ternate",
+ "tg": "Tajik",
+ "tga": "Sagalla",
+ "tgb": "Tobilung",
+ "tgc": "Tigak",
+ "tgd": "Ciwogai",
+ "tge": "Eastern Gorkha Tamang",
+ "tgf": "Chalikha",
+ "tgh": "Tobagonian Creole English",
+ "tgi": "Lawunuia",
+ "tgj": "Tagin",
+ "tgn": "Tandaganon",
+ "tgo": "Sudest",
+ "tgp": "Tangoa",
+ "tgq": "Tring",
+ "tgr": "Tareng",
+ "tgs": "Nume",
+ "tgt": "Central Tagbanwa",
+ "tgu": "Tanggu",
+ "tgv": "Tingui-Boto",
+ "tgw": "Tagwana Senoufo",
+ "tgx": "Tagish",
+ "tgy": "Togoyo",
+ "tgz": "Tagalaka",
+ "th": "Thai",
+ "thd": "Kuuk Thaayorre; Thayore",
+ "the": "Chitwania Tharu",
+ "thf": "Thangmi",
+ "thh": "Northern Tarahumara",
+ "thi": "Tai Long",
+ "thk": "Tharaka; Kitharaka",
+ "thl": "Dangaura Tharu",
+ "thm": "Aheu",
+ "thn": "Thachanadan",
+ "thp": "Thompson",
+ "thq": "Kochila Tharu",
+ "thr": "Rana Tharu",
+ "ths": "Thakali",
+ "tht": "Tahltan",
+ "thu": "Thuri",
+ "thv": "Tahaggart Tamahaq",
+ "thy": "Tha",
+ "thz": "Tayart Tamajeq",
+ "ti": "Tigrinya",
+ "tia": "Tidikelt Tamazight",
+ "tic": "Tira",
+ "tif": "Tifal",
+ "tig": "Tigre",
+ "tih": "Timugon Murut",
+ "tii": "Tiene",
+ "tij": "Tilung",
+ "tik": "Tikar",
+ "til": "Tillamook",
+ "tim": "Timbe",
+ "tin": "Tindi",
+ "tio": "Teop",
+ "tip": "Trimuris",
+ "tiq": "Tiéfo",
+ "tis": "Masadiit Itneg",
+ "tit": "Tinigua",
+ "tiu": "Adasen",
+ "tiv": "Tiv",
+ "tiw": "Tiwi",
+ "tix": "Southern Tiwa",
+ "tiy": "Tiruray",
+ "tiz": "Tai Hongjin",
+ "tja": "Tajuasohn",
+ "tjg": "Tunjung",
+ "tji": "Northern Tujia",
+ "tjj": "Tjungundji",
+ "tjl": "Tai Laing",
+ "tjm": "Timucua",
+ "tjn": "Tonjon",
+ "tjo": "Temacine Tamazight",
+ "tjp": "Tjupany",
+ "tjs": "Southern Tujia",
+ "tju": "Tjurruru",
+ "tjw": "Djabwurrung",
+ "tk": "Turkmen",
+ "tka": "Truká",
+ "tkb": "Buksa",
+ "tkd": "Tukudede",
+ "tke": "Takwane",
+ "tkf": "Tukumanféd",
+ "tkg": "Tesaka Malagasy",
+ "tkl": "Tokelau",
+ "tkm": "Takelma",
+ "tkn": "Toku-No-Shima",
+ "tkp": "Tikopia",
+ "tkq": "Tee",
+ "tkr": "Tsakhur",
+ "tks": "Takestani",
+ "tkt": "Kathoriya Tharu",
+ "tku": "Upper Necaxa Totonac",
+ "tkv": "Mur Pano",
+ "tkw": "Teanu",
+ "tkx": "Tangko",
+ "tkz": "Takua",
+ "tl": "Tagalog",
+ "tla": "Southwestern Tepehuan",
+ "tlb": "Tobelo",
+ "tlc": "Yecuatla Totonac",
+ "tld": "Talaud",
+ "tlf": "Telefol",
+ "tlg": "Tofanma",
+ "tlh": "Klingon; tlhIngan Hol",
+ "tli": "Tlingit",
+ "tlj": "Talinga-Bwisi",
+ "tlk": "Taloki",
+ "tll": "Tetela",
+ "tlm": "Tolomako",
+ "tln": "Talondo'",
+ "tlo": "Talodi",
+ "tlp": "Filomena Mata-Coahuitlán Totonac",
+ "tlq": "Tai Loi",
+ "tlr": "Talise",
+ "tls": "Tambotalo",
+ "tlt": "Sou Nama; Teluti",
+ "tlu": "Tulehu",
+ "tlv": "Taliabu",
+ "tlx": "Khehek",
+ "tly": "Talysh",
+ "tma": "Tama (Chad)",
+ "tmb": "Katbol; Avava",
+ "tmc": "Tumak",
+ "tmd": "Haruai",
+ "tme": "Tremembé",
+ "tmf": "Toba-Maskoy",
+ "tmg": "Ternateño",
+ "tmh": "Tamashek",
+ "tmi": "Tutuba",
+ "tmj": "Samarokena",
+ "tmk": "Northwestern Tamang",
+ "tml": "Tamnim Citak",
+ "tmm": "Tai Thanh",
+ "tmn": "Taman (Indonesia)",
+ "tmo": "Temoq",
+ "tmq": "Tumleo",
+ "tmr": "Jewish Babylonian Aramaic (ca. 200-1200 CE)",
+ "tms": "Tima",
+ "tmt": "Tasmate",
+ "tmu": "Iau",
+ "tmv": "Tembo (Motembo)",
+ "tmw": "Temuan",
+ "tmy": "Tami",
+ "tmz": "Tamanaku",
+ "tn": "Tswana",
+ "tna": "Tacana",
+ "tnb": "Western Tunebo",
+ "tnc": "Tanimuca-Retuarã",
+ "tnd": "Angosturas Tunebo",
+ "tng": "Tobanga",
+ "tnh": "Maiani",
+ "tni": "Tandia",
+ "tnk": "Kwamera",
+ "tnl": "Lenakel",
+ "tnm": "Tabla",
+ "tnn": "North Tanna",
+ "tno": "Toromono",
+ "tnp": "Whitesands",
+ "tnq": "Taino",
+ "tnr": "Ménik",
+ "tns": "Tenis",
+ "tnt": "Tontemboan",
+ "tnu": "Tay Khang",
+ "tnv": "Tangchangya",
+ "tnw": "Tonsawang",
+ "tnx": "Tanema",
+ "tny": "Tongwe",
+ "tnz": "Ten'edn",
+ "to": "Tonga (Tonga Islands)",
+ "tob": "Toba",
+ "toc": "Coyutla Totonac",
+ "tod": "Toma",
+ "tof": "Gizrra",
+ "tog": "Tonga (Nyasa)",
+ "toh": "Gitonga",
+ "toi": "Tonga (Zambia)",
+ "toj": "Tojolabal",
+ "tok": "Toki Pona",
+ "tol": "Tolowa",
+ "tom": "Tombulu",
+ "too": "Xicotepec De Juárez Totonac",
+ "top": "Papantla Totonac",
+ "toq": "Toposa",
+ "tor": "Togbo-Vara Banda",
+ "tos": "Highland Totonac",
+ "tou": "Tho",
+ "tov": "Upper Taromi",
+ "tow": "Jemez",
+ "tox": "Tobian",
+ "toy": "Topoiyo",
+ "toz": "To",
+ "tpa": "Taupota",
+ "tpc": "Azoyú Me'phaa; Azoyú Tlapanec",
+ "tpe": "Tippera",
+ "tpf": "Tarpia",
+ "tpg": "Kula",
+ "tpi": "Tok Pisin",
+ "tpj": "Tapieté",
+ "tpk": "Tupinikin",
+ "tpl": "Tlacoapa Me'phaa; Tlacoapa Tlapanec",
+ "tpm": "Tampulma",
+ "tpn": "Tupinambá",
+ "tpo": "Tai Pao",
+ "tpp": "Pisaflores Tepehua",
+ "tpq": "Tukpa",
+ "tpr": "Tuparí",
+ "tpt": "Tlachichilco Tepehua",
+ "tpu": "Tampuan",
+ "tpv": "Tanapag",
+ "tpw": "Tupí",
+ "tpx": "Acatepec Me'phaa; Acatepec Tlapanec",
+ "tpy": "Trumai",
+ "tpz": "Tinputz",
+ "tqb": "Tembé",
+ "tql": "Lehali",
+ "tqm": "Turumsa",
+ "tqn": "Tenino",
+ "tqo": "Toaripi",
+ "tqp": "Tomoip",
+ "tqq": "Tunni",
+ "tqr": "Torona",
+ "tqt": "Western Totonac",
+ "tqu": "Touo",
+ "tqw": "Tonkawa",
+ "tr": "Turkish",
+ "tra": "Tirahi",
+ "trb": "Terebu",
+ "trc": "Copala Triqui",
+ "trd": "Turi",
+ "tre": "East Tarangan",
+ "trf": "Trinidadian Creole English",
+ "trg": "Lishán Didán",
+ "trh": "Turaka",
+ "tri": "Trió",
+ "trj": "Toram",
+ "trk": "Turkic languages",
+ "trl": "Traveller Scottish",
+ "trm": "Tregami",
+ "trn": "Trinitario",
+ "tro": "Tarao Naga",
+ "trp": "Kok Borok",
+ "trq": "San Martín Itunyoso Triqui",
+ "trr": "Taushiro",
+ "trs": "Chicahuaxtla Triqui",
+ "trt": "Tunggare",
+ "tru": "Turoyo; Surayt",
+ "trv": "Sediq; Seediq; Taroko",
+ "trw": "Torwali",
+ "trx": "Tringgus-Sembaan Bidayuh",
+ "try": "Turung",
+ "trz": "Torá",
+ "ts": "Tsonga",
+ "tsa": "Tsaangi",
+ "tsb": "Tsamai",
+ "tsc": "Tswa",
+ "tsd": "Tsakonian",
+ "tse": "Tunisian Sign Language",
+ "tsg": "Tausug",
+ "tsh": "Tsuvan",
+ "tsi": "Tsimshian",
+ "tsj": "Tshangla",
+ "tsk": "Tseku",
+ "tsl": "Ts'ün-Lao",
+ "tsm": "Turkish Sign Language; Türk İşaret Dili",
+ "tsp": "Northern Toussian",
+ "tsq": "Thai Sign Language",
+ "tsr": "Akei",
+ "tss": "Taiwan Sign Language",
+ "tst": "Tondi Songway Kiini",
+ "tsu": "Tsou",
+ "tsv": "Tsogo",
+ "tsw": "Tsishingini",
+ "tsx": "Mubami",
+ "tsy": "Tebul Sign Language",
+ "tsz": "Purepecha",
+ "tt": "Tatar",
+ "tta": "Tutelo",
+ "ttb": "Gaa",
+ "ttc": "Tektiteko",
+ "ttd": "Tauade",
+ "tte": "Bwanabwana",
+ "ttf": "Tuotomb",
+ "ttg": "Tutong",
+ "tth": "Upper Ta'oih",
+ "tti": "Tobati",
+ "ttj": "Tooro",
+ "ttk": "Totoro",
+ "ttl": "Totela",
+ "ttm": "Northern Tutchone",
+ "ttn": "Towei",
+ "tto": "Lower Ta'oih",
+ "ttp": "Tombelala",
+ "ttq": "Tawallammat Tamajaq",
+ "ttr": "Tera",
+ "tts": "Northeastern Thai",
+ "ttt": "Muslim Tat",
+ "ttu": "Torau",
+ "ttv": "Titan",
+ "ttw": "Long Wat",
+ "tty": "Sikaritai",
+ "ttz": "Tsum",
+ "tua": "Wiarumus",
+ "tub": "Tübatulabal",
+ "tuc": "Mutu",
+ "tud": "Tuxá",
+ "tue": "Tuyuca",
+ "tuf": "Central Tunebo",
+ "tug": "Tunia",
+ "tuh": "Taulil",
+ "tui": "Tupuri",
+ "tuj": "Tugutil",
+ "tul": "Tula",
+ "tum": "Tumbuka",
+ "tun": "Tunica",
+ "tuo": "Tucano",
+ "tup": "Tupi languages",
+ "tuq": "Tedaga",
+ "tus": "Tuscarora",
+ "tut": "Altaic languages",
+ "tuu": "Tututni",
+ "tuv": "Turkana",
+ "tuw": "Tungus languages",
+ "tux": "Tuxináwa",
+ "tuy": "Tugen",
+ "tuz": "Turka",
+ "tva": "Vaghua",
+ "tvd": "Tsuvadi",
+ "tve": "Te'un",
+ "tvk": "Southeast Ambrym",
+ "tvl": "Tuvalu",
+ "tvm": "Tela-Masbuar",
+ "tvn": "Tavoyan",
+ "tvo": "Tidore",
+ "tvs": "Taveta",
+ "tvt": "Tutsa Naga",
+ "tvu": "Tunen",
+ "tvw": "Sedoa",
+ "tvx": "Taivoan",
+ "tvy": "Timor Pidgin",
+ "tw": "Twi",
+ "twa": "Twana",
+ "twb": "Western Tawbuid",
+ "twc": "Teshenawa",
+ "twd": "Twents",
+ "twe": "Tewa (Indonesia)",
+ "twf": "Northern Tiwa",
+ "twg": "Tereweng",
+ "twh": "Tai Dón",
+ "twl": "Tawara",
+ "twm": "Tawang Monpa",
+ "twn": "Twendi",
+ "two": "Tswapong",
+ "twp": "Ere",
+ "twq": "Tasawaq",
+ "twr": "Southwestern Tarahumara",
+ "twt": "Turiwára",
+ "twu": "Termanu",
+ "tww": "Tuwari",
+ "twx": "Tewe",
+ "twy": "Tawoyan",
+ "txa": "Tombonuo",
+ "txb": "Tokharian B",
+ "txc": "Tsetsaut",
+ "txe": "Totoli",
+ "txg": "Tangut",
+ "txh": "Thracian",
+ "txi": "Ikpeng",
+ "txj": "Tarjumo",
+ "txm": "Tomini",
+ "txn": "West Tarangan",
+ "txo": "Toto",
+ "txq": "Tii",
+ "txr": "Tartessian",
+ "txs": "Tonsea",
+ "txt": "Citak",
+ "txu": "Kayapó",
+ "txx": "Tatana",
+ "txy": "Tanosy Malagasy",
+ "ty": "Tahitian",
+ "tya": "Tauya",
+ "tye": "Kyanga",
+ "tyh": "O'du",
+ "tyi": "Teke-Tsaayi",
+ "tyj": "Tai Do; Tai Yo",
+ "tyl": "Thu Lao",
+ "tyn": "Kombai",
+ "typ": "Thaypan",
+ "tyr": "Tai Daeng",
+ "tys": "Tày Sa Pa",
+ "tyt": "Tày Tac",
+ "tyu": "Kua",
+ "tyv": "Tuvinian",
+ "tyx": "Teke-Tyee",
+ "tyy": "Tiyaa",
+ "tyz": "Tày",
+ "tza": "Tanzanian Sign Language",
+ "tzh": "Tzeltal",
+ "tzj": "Tz'utujil",
+ "tzl": "Talossan",
+ "tzm": "Central Atlas Tamazight",
+ "tzn": "Tugun",
+ "tzo": "Tzotzil",
+ "tzx": "Tabriak",
+ "uam": "Uamué",
+ "uan": "Kuan",
+ "uar": "Tairuma",
+ "uba": "Ubang",
+ "ubi": "Ubi",
+ "ubl": "Buhi'non Bikol",
+ "ubr": "Ubir",
+ "ubu": "Umbu-Ungu",
+ "uby": "Ubykh",
+ "uda": "Uda",
+ "ude": "Udihe",
+ "udg": "Muduga",
+ "udi": "Udi",
+ "udj": "Ujir",
+ "udl": "Wuzlam",
+ "udm": "Udmurt",
+ "udu": "Uduk",
+ "ues": "Kioko",
+ "ufi": "Ufim",
+ "ug": "Uighur; Uyghur",
+ "uga": "Ugaritic",
+ "ugb": "Kuku-Ugbanh",
+ "uge": "Ughele",
+ "ugh": "Kubachi",
+ "ugn": "Ugandan Sign Language",
+ "ugo": "Ugong",
+ "ugy": "Uruguayan Sign Language",
+ "uha": "Uhami",
+ "uhn": "Damal",
+ "uis": "Uisai",
+ "uiv": "Iyive",
+ "uji": "Tanjijili",
+ "uk": "Ukrainian",
+ "uka": "Kaburi",
+ "ukg": "Ukuriguma",
+ "ukh": "Ukhwejo",
+ "uki": "Kui (India)",
+ "ukk": "Muak Sa-aak",
+ "ukl": "Ukrainian Sign Language",
+ "ukp": "Ukpe-Bayobiri",
+ "ukq": "Ukwa",
+ "uks": "Urubú-Kaapor Sign Language; Kaapor Sign Language",
+ "uku": "Ukue",
+ "ukv": "Kuku",
+ "ukw": "Ukwuani-Aboh-Ndoni",
+ "uky": "Kuuk-Yak",
+ "ula": "Fungwa",
+ "ulb": "Ulukwumi",
+ "ulc": "Ulch",
+ "ule": "Lule",
+ "ulf": "Usku; Afra",
+ "uli": "Ulithian",
+ "ulk": "Meriam Mir",
+ "ull": "Ullatan",
+ "ulm": "Ulumanda'",
+ "uln": "Unserdeutsch",
+ "ulu": "Uma' Lung",
+ "ulw": "Ulwa",
+ "uma": "Umatilla",
+ "umb": "Umbundu",
+ "umc": "Marrucinian",
+ "umd": "Umbindhamu",
+ "umg": "Morrobalama; Umbuygamu",
+ "umi": "Ukit",
+ "umm": "Umon",
+ "umn": "Makyan Naga",
+ "umo": "Umotína",
+ "ump": "Umpila",
+ "umr": "Umbugarla",
+ "ums": "Pendau",
+ "umu": "Munsee",
+ "una": "North Watut",
+ "und": "Undetermined",
+ "une": "Uneme",
+ "ung": "Ngarinyin",
+ "uni": "Uni",
+ "unk": "Enawené-Nawé",
+ "unm": "Unami",
+ "unn": "Kurnai",
+ "unr": "Mundari",
+ "unu": "Unubahe",
+ "unx": "Munda",
+ "unz": "Unde Kaili",
+ "uon": "Kulon",
+ "upi": "Umeda",
+ "upv": "Uripiv-Wala-Rano-Atchin",
+ "ur": "Urdu",
+ "ura": "Urarina",
+ "urb": "Urubú-Kaapor; Kaapor",
+ "urc": "Urningangg",
+ "ure": "Uru",
+ "urf": "Uradhi",
+ "urg": "Urigina",
+ "urh": "Urhobo",
+ "uri": "Urim",
+ "urj": "Uralic languages",
+ "urk": "Urak Lawoi'",
+ "url": "Urali",
+ "urm": "Urapmin",
+ "urn": "Uruangnirin",
+ "uro": "Ura (Papua New Guinea)",
+ "urp": "Uru-Pa-In",
+ "urr": "Lehalurup; Löyöp",
+ "urt": "Urat",
+ "uru": "Urumi",
+ "urv": "Uruava",
+ "urw": "Sop",
+ "urx": "Urimo",
+ "ury": "Orya",
+ "urz": "Uru-Eu-Wau-Wau",
+ "usa": "Usarufa",
+ "ush": "Ushojo",
+ "usi": "Usui",
+ "usk": "Usaghade",
+ "usp": "Uspanteco",
+ "uss": "us-Saare",
+ "usu": "Uya",
+ "uta": "Otank",
+ "ute": "Ute-Southern Paiute",
+ "uth": "ut-Hun",
+ "utp": "Amba (Solomon Islands)",
+ "utr": "Etulo",
+ "utu": "Utu",
+ "uum": "Urum",
+ "uur": "Ura (Vanuatu)",
+ "uuu": "U",
+ "uve": "West Uvean; Fagauvea",
+ "uvh": "Uri",
+ "uvl": "Lote",
+ "uwa": "Kuku-Uwanh",
+ "uya": "Doko-Uyanga",
+ "uz": "Uzbek",
+ "uzn": "Northern Uzbek",
+ "uzs": "Southern Uzbek",
+ "vaa": "Vaagri Booli",
+ "vae": "Vale",
+ "vaf": "Vafsi",
+ "vag": "Vagla",
+ "vah": "Varhadi-Nagpuri",
+ "vai": "Vai",
+ "vaj": "Sekele; Northwestern ǃKung; Vasekele",
+ "val": "Vehes",
+ "vam": "Vanimo",
+ "van": "Valman",
+ "vao": "Vao",
+ "vap": "Vaiphei",
+ "var": "Huarijio",
+ "vas": "Vasavi",
+ "vau": "Vanuma",
+ "vav": "Varli",
+ "vay": "Wayu",
+ "vbb": "Southeast Babar",
+ "vbk": "Southwestern Bontok",
+ "ve": "Venda",
+ "vec": "Venetian",
+ "ved": "Veddah",
+ "vel": "Veluws",
+ "vem": "Vemgo-Mabas",
+ "veo": "Ventureño",
+ "vep": "Veps",
+ "ver": "Mom Jango",
+ "vgr": "Vaghri",
+ "vgt": "Vlaamse Gebarentaal; Flemish Sign Language",
+ "vi": "Vietnamese",
+ "vic": "Virgin Islands Creole English",
+ "vid": "Vidunda",
+ "vif": "Vili",
+ "vig": "Viemo",
+ "vil": "Vilela",
+ "vin": "Vinza",
+ "vis": "Vishavan",
+ "vit": "Viti",
+ "viv": "Iduna",
+ "vka": "Kariyarra",
+ "vkj": "Kujarge",
+ "vkk": "Kaur",
+ "vkl": "Kulisusu",
+ "vkm": "Kamakan",
+ "vkn": "Koro Nulu",
+ "vko": "Kodeoha",
+ "vkp": "Korlai Creole Portuguese",
+ "vkt": "Tenggarong Kutai Malay",
+ "vku": "Kurrama",
+ "vkz": "Koro Zuba",
+ "vlp": "Valpei",
+ "vls": "Vlaams",
+ "vma": "Martuyhunira",
+ "vmb": "Barbaram",
+ "vmc": "Juxtlahuaca Mixtec",
+ "vmd": "Mudu Koraga",
+ "vme": "East Masela",
+ "vmf": "Mainfränkisch",
+ "vmg": "Lungalunga",
+ "vmh": "Maraghei",
+ "vmi": "Miwa",
+ "vmj": "Ixtayutla Mixtec",
+ "vmk": "Makhuwa-Shirima",
+ "vml": "Malgana",
+ "vmm": "Mitlatongo Mixtec",
+ "vmp": "Soyaltepec Mazatec",
+ "vmq": "Soyaltepec Mixtec",
+ "vmr": "Marenje",
+ "vms": "Moksela",
+ "vmu": "Muluridyi",
+ "vmv": "Valley Maidu",
+ "vmw": "Makhuwa",
+ "vmx": "Tamazola Mixtec",
+ "vmy": "Ayautla Mazatec",
+ "vmz": "Mazatlán Mazatec",
+ "vnk": "Vano; Lovono",
+ "vnm": "Vinmavis; Neve'ei",
+ "vnp": "Vunapu",
+ "vo": "Volapük",
+ "vor": "Voro",
+ "vot": "Votic",
+ "vra": "Vera'a",
+ "vro": "Võro",
+ "vrs": "Varisi",
+ "vrt": "Burmbar; Banam Bay",
+ "vsi": "Moldova Sign Language",
+ "vsl": "Venezuelan Sign Language",
+ "vsv": "Valencian Sign Language; Llengua de signes valenciana",
+ "vto": "Vitou",
+ "vum": "Vumbu",
+ "vun": "Vunjo",
+ "vut": "Vute",
+ "vwa": "Awa (China)",
+ "wa": "Walloon",
+ "waa": "Walla Walla",
+ "wab": "Wab",
+ "wac": "Wasco-Wishram",
+ "wad": "Wamesa; Wondama",
+ "wae": "Walser",
+ "waf": "Wakoná",
+ "wag": "Wa'ema",
+ "wah": "Watubela",
+ "wai": "Wares",
+ "waj": "Waffa",
+ "wak": "Wakashan languages",
+ "wal": "Wolaytta; Wolaitta",
+ "wam": "Wampanoag",
+ "wan": "Wan",
+ "wao": "Wappo",
+ "wap": "Wapishana",
+ "waq": "Wagiman",
+ "war": "Waray (Philippines)",
+ "was": "Washo",
+ "wat": "Kaninuwa",
+ "wau": "Waurá",
+ "wav": "Waka",
+ "waw": "Waiwai",
+ "wax": "Watam; Marangis",
+ "way": "Wayana",
+ "waz": "Wampur",
+ "wba": "Warao",
+ "wbb": "Wabo",
+ "wbe": "Waritai",
+ "wbf": "Wara",
+ "wbh": "Wanda",
+ "wbi": "Vwanji",
+ "wbj": "Alagwa",
+ "wbk": "Waigali",
+ "wbl": "Wakhi",
+ "wbm": "Wa",
+ "wbp": "Warlpiri",
+ "wbq": "Waddar",
+ "wbr": "Wagdi",
+ "wbs": "West Bengal Sign Language",
+ "wbt": "Warnman",
+ "wbv": "Wajarri",
+ "wbw": "Woi",
+ "wca": "Yanomámi",
+ "wci": "Waci Gbe",
+ "wdd": "Wandji",
+ "wdg": "Wadaginam",
+ "wdj": "Wadjiginy",
+ "wdk": "Wadikali",
+ "wdt": "Wendat",
+ "wdu": "Wadjigu",
+ "wdy": "Wadjabangayi",
+ "wea": "Wewaw",
+ "wec": "Wè Western",
+ "wed": "Wedau",
+ "weg": "Wergaia",
+ "weh": "Weh",
+ "wei": "Kiunum",
+ "wem": "Weme Gbe",
+ "wen": "Sorbian languages",
+ "weo": "Wemale",
+ "wep": "Westphalien",
+ "wer": "Weri",
+ "wes": "Cameroon Pidgin",
+ "wet": "Perai",
+ "weu": "Rawngtu Chin",
+ "wew": "Wejewa",
+ "wfg": "Yafi; Zorop",
+ "wga": "Wagaya",
+ "wgb": "Wagawaga",
+ "wgg": "Wangkangurru; Wangganguru",
+ "wgi": "Wahgi",
+ "wgo": "Waigeo",
+ "wgu": "Wirangu",
+ "wgy": "Warrgamay",
+ "wha": "Sou Upaa; Manusela",
+ "whg": "North Wahgi",
+ "whk": "Wahau Kenyah",
+ "whu": "Wahau Kayan",
+ "wib": "Southern Toussian",
+ "wic": "Wichita",
+ "wie": "Wik-Epa",
+ "wif": "Wik-Keyangan",
+ "wig": "Wik Ngathan",
+ "wih": "Wik-Me'anha",
+ "wii": "Minidien",
+ "wij": "Wik-Iiyanh",
+ "wik": "Wikalkan",
+ "wil": "Wilawila",
+ "wim": "Wik-Mungkan",
+ "win": "Ho-Chunk",
+ "wir": "Wiraféd",
+ "wiu": "Wiru",
+ "wiv": "Vitu",
+ "wiy": "Wiyot",
+ "wja": "Waja",
+ "wji": "Warji",
+ "wka": "Kw'adza",
+ "wkb": "Kumbaran",
+ "wkd": "Wakde; Mo",
+ "wkl": "Kalanadi",
+ "wkr": "Keerray-Woorroong",
+ "wku": "Kunduvadi",
+ "wkw": "Wakawaka",
+ "wky": "Wangkayutyuru",
+ "wla": "Walio",
+ "wlc": "Mwali Comorian",
+ "wle": "Wolane",
+ "wlg": "Kunbarlang",
+ "wlh": "Welaun",
+ "wli": "Waioli",
+ "wlk": "Wailaki",
+ "wll": "Wali (Sudan)",
+ "wlm": "Middle Welsh",
+ "wlo": "Wolio",
+ "wlr": "Wailapa",
+ "wls": "Wallisian",
+ "wlu": "Wuliwuli",
+ "wlv": "Wichí Lhamtés Vejoz",
+ "wlw": "Walak",
+ "wlx": "Wali (Ghana)",
+ "wly": "Waling",
+ "wma": "Mawa (Nigeria)",
+ "wmb": "Wambaya",
+ "wmc": "Wamas",
+ "wmd": "Mamaindé",
+ "wme": "Wambule",
+ "wmg": "Western Minyag",
+ "wmh": "Waima'a",
+ "wmi": "Wamin",
+ "wmm": "Maiwa (Indonesia)",
+ "wmn": "Waamwang",
+ "wmo": "Wom (Papua New Guinea)",
+ "wms": "Wambon",
+ "wmt": "Walmajarri",
+ "wmw": "Mwani",
+ "wmx": "Womo",
+ "wnb": "Wanambre",
+ "wnc": "Wantoat",
+ "wnd": "Wandarang",
+ "wne": "Waneci",
+ "wng": "Wanggom",
+ "wni": "Ndzwani Comorian",
+ "wnk": "Wanukaka",
+ "wnm": "Wanggamala",
+ "wnn": "Wunumara",
+ "wno": "Wano",
+ "wnp": "Wanap",
+ "wnu": "Usan",
+ "wnw": "Wintu",
+ "wny": "Wanyi; Waanyi",
+ "wo": "Wolof",
+ "woa": "Kuwema; Tyaraity",
+ "wob": "Wè Northern",
+ "woc": "Wogeo",
+ "wod": "Wolani",
+ "woe": "Woleaian",
+ "wof": "Gambian Wolof",
+ "wog": "Wogamusin",
+ "woi": "Kamang",
+ "wok": "Longto",
+ "wom": "Wom (Nigeria)",
+ "won": "Wongo",
+ "woo": "Manombai",
+ "wor": "Woria",
+ "wos": "Hanga Hundi",
+ "wow": "Wawonii",
+ "woy": "Weyto",
+ "wpc": "Maco",
+ "wrb": "Waluwarra; Warluwara",
+ "wrg": "Warungu; Gudjal",
+ "wrh": "Wiradjuri",
+ "wri": "Wariyangga",
+ "wrk": "Garrwa",
+ "wrl": "Warlmanpa",
+ "wrm": "Warumungu",
+ "wrn": "Warnang",
+ "wro": "Worrorra",
+ "wrp": "Waropen",
+ "wrr": "Wardaman",
+ "wrs": "Waris",
+ "wru": "Waru",
+ "wrv": "Waruna",
+ "wrw": "Gugu Warra",
+ "wrx": "Wae Rana",
+ "wry": "Merwari",
+ "wrz": "Waray (Australia)",
+ "wsa": "Warembori",
+ "wsg": "Adilabad Gondi",
+ "wsi": "Wusi",
+ "wsk": "Waskia",
+ "wsr": "Owenia",
+ "wss": "Wasa",
+ "wsu": "Wasu",
+ "wsv": "Wotapuri-Katarqalai",
+ "wtf": "Watiwa",
+ "wth": "Wathawurrung",
+ "wti": "Berta",
+ "wtk": "Watakataui",
+ "wtm": "Mewati",
+ "wtw": "Wotu",
+ "wua": "Wikngenchera",
+ "wub": "Wunambal",
+ "wud": "Wudu",
+ "wuh": "Wutunhua",
+ "wul": "Silimo",
+ "wum": "Wumbvu",
+ "wun": "Bungu",
+ "wur": "Wurrugu",
+ "wut": "Wutung",
+ "wuu": "Wu Chinese",
+ "wuv": "Wuvulu-Aua",
+ "wux": "Wulna",
+ "wuy": "Wauyai",
+ "wwa": "Waama",
+ "wwb": "Wakabunga",
+ "wwo": "Wetamut; Dorig",
+ "wwr": "Warrwa",
+ "www": "Wawa",
+ "wxa": "Waxianghua",
+ "wxw": "Wardandi",
+ "wyb": "Wangaaybuwan-Ngiyambaa",
+ "wyi": "Woiwurrung",
+ "wym": "Wymysorys",
+ "wyn": "Wyandot",
+ "wyr": "Wayoró",
+ "wyy": "Western Fijian",
+ "xaa": "Andalusian Arabic",
+ "xab": "Sambe",
+ "xac": "Kachari",
+ "xad": "Adai",
+ "xae": "Aequian",
+ "xag": "Aghwan",
+ "xai": "Kaimbé",
+ "xaj": "Ararandewára",
+ "xak": "Máku",
+ "xal": "Kalmyk; Oirat",
+ "xam": "ǀXam",
+ "xan": "Xamtanga",
+ "xao": "Khao",
+ "xap": "Apalachee",
+ "xaq": "Aquitanian",
+ "xar": "Karami",
+ "xas": "Kamas",
+ "xat": "Katawixi",
+ "xau": "Kauwera",
+ "xav": "Xavánte",
+ "xaw": "Kawaiisu",
+ "xay": "Kayan Mahakam",
+ "xbb": "Lower Burdekin",
+ "xbc": "Bactrian",
+ "xbd": "Bindal",
+ "xbe": "Bigambal",
+ "xbg": "Bunganditj",
+ "xbi": "Kombio",
+ "xbj": "Birrpayi",
+ "xbm": "Middle Breton",
+ "xbn": "Kenaboi",
+ "xbo": "Bolgarian",
+ "xbp": "Bibbulman",
+ "xbr": "Kambera",
+ "xbw": "Kambiwá",
+ "xby": "Batjala; Batyala",
+ "xcb": "Cumbric",
+ "xcc": "Camunic",
+ "xce": "Celtiberian",
+ "xcg": "Cisalpine Gaulish",
+ "xch": "Chemakum; Chimakum",
+ "xcl": "Classical Armenian",
+ "xcm": "Comecrudo",
+ "xcn": "Cotoname",
+ "xco": "Chorasmian",
+ "xcr": "Carian",
+ "xct": "Classical Tibetan",
+ "xcu": "Curonian",
+ "xcv": "Chuvantsy",
+ "xcw": "Coahuilteco",
+ "xcy": "Cayuse",
+ "xda": "Darkinyung",
+ "xdc": "Dacian",
+ "xdk": "Dharuk",
+ "xdm": "Edomite",
+ "xdo": "Kwandu",
+ "xdq": "Kaitag",
+ "xdy": "Malayic Dayak",
+ "xeb": "Eblan",
+ "xed": "Hdi",
+ "xeg": "ǁXegwi",
+ "xel": "Kelo",
+ "xem": "Kembayan",
+ "xep": "Epi-Olmec",
+ "xer": "Xerénte",
+ "xes": "Kesawai",
+ "xet": "Xetá",
+ "xeu": "Keoru-Ahia",
+ "xfa": "Faliscan",
+ "xga": "Galatian",
+ "xgb": "Gbin",
+ "xgd": "Gudang",
+ "xgf": "Gabrielino-Fernandeño",
+ "xgg": "Goreng",
+ "xgi": "Garingbal",
+ "xgl": "Galindan",
+ "xgm": "Dharumbal; Guwinmal",
+ "xgn": "Mongolian languages",
+ "xgr": "Garza",
+ "xgu": "Unggumi",
+ "xgw": "Guwa",
+ "xh": "Xhosa",
+ "xha": "Harami",
+ "xhc": "Hunnic",
+ "xhd": "Hadrami",
+ "xhe": "Khetrani",
+ "xhm": "Middle Khmer (1400 to 1850 CE)",
+ "xhr": "Hernican",
+ "xht": "Hattic",
+ "xhu": "Hurrian",
+ "xhv": "Khua",
+ "xib": "Iberian",
+ "xii": "Xiri",
+ "xil": "Illyrian",
+ "xin": "Xinca",
+ "xir": "Xiriâna",
+ "xis": "Kisan",
+ "xiv": "Indus Valley Language",
+ "xiy": "Xipaya",
+ "xjb": "Minjungbal",
+ "xjt": "Jaitmatang",
+ "xka": "Kalkoti",
+ "xkb": "Northern Nago",
+ "xkc": "Kho'ini",
+ "xkd": "Mendalam Kayan",
+ "xke": "Kereho",
+ "xkf": "Khengkha",
+ "xkg": "Kagoro",
+ "xki": "Kenyan Sign Language",
+ "xkj": "Kajali",
+ "xkk": "Kachok; Kaco'",
+ "xkl": "Mainstream Kenyah",
+ "xkn": "Kayan River Kayan",
+ "xko": "Kiorr",
+ "xkp": "Kabatei",
+ "xkq": "Koroni",
+ "xkr": "Xakriabá",
+ "xks": "Kumbewaha",
+ "xkt": "Kantosi",
+ "xku": "Kaamba",
+ "xkv": "Kgalagadi",
+ "xkw": "Kembra",
+ "xkx": "Karore",
+ "xky": "Uma' Lasan",
+ "xkz": "Kurtokha",
+ "xla": "Kamula",
+ "xlb": "Loup B",
+ "xlc": "Lycian",
+ "xld": "Lydian",
+ "xle": "Lemnian",
+ "xlg": "Ligurian (Ancient)",
+ "xli": "Liburnian",
+ "xln": "Alanic",
+ "xlo": "Loup A",
+ "xlp": "Lepontic",
+ "xls": "Lusitanian",
+ "xlu": "Cuneiform Luwian",
+ "xly": "Elymian",
+ "xma": "Mushungulu",
+ "xmb": "Mbonga",
+ "xmc": "Makhuwa-Marrevone",
+ "xmd": "Mbudum",
+ "xme": "Median",
+ "xmf": "Mingrelian",
+ "xmg": "Mengaka",
+ "xmh": "Kugu-Muminh",
+ "xmj": "Majera",
+ "xmk": "Ancient Macedonian",
+ "xml": "Malaysian Sign Language",
+ "xmm": "Manado Malay",
+ "xmn": "Manichaean Middle Persian",
+ "xmo": "Morerebi",
+ "xmp": "Kuku-Mu'inh",
+ "xmq": "Kuku-Mangk",
+ "xmr": "Meroitic",
+ "xms": "Moroccan Sign Language",
+ "xmt": "Matbat",
+ "xmu": "Kamu",
+ "xmv": "Antankarana Malagasy; Tankarana Malagasy",
+ "xmw": "Tsimihety Malagasy",
+ "xmx": "Salawati; Maden",
+ "xmy": "Mayaguduna",
+ "xmz": "Mori Bawah",
+ "xna": "Ancient North Arabian",
+ "xnb": "Kanakanabu",
+ "xnd": "Na-Dene languages",
+ "xng": "Middle Mongolian",
+ "xnh": "Kuanhua",
+ "xni": "Ngarigu",
+ "xnj": "Ngoni (Tanzania)",
+ "xnk": "Nganakarti",
+ "xnm": "Ngumbarl",
+ "xnn": "Northern Kankanay",
+ "xno": "Anglo-Norman",
+ "xnq": "Ngoni (Mozambique)",
+ "xnr": "Kangri",
+ "xns": "Kanashi",
+ "xnt": "Narragansett",
+ "xnu": "Nukunul",
+ "xny": "Nyiyaparli",
+ "xnz": "Kenzi; Mattoki",
+ "xoc": "O'chi'chi'",
+ "xod": "Kokoda",
+ "xog": "Soga",
+ "xoi": "Kominimung",
+ "xok": "Xokleng",
+ "xom": "Komo (Sudan)",
+ "xon": "Konkomba",
+ "xoo": "Xukurú",
+ "xop": "Kopar",
+ "xor": "Korubo",
+ "xow": "Kowaki",
+ "xpa": "Pirriya",
+ "xpb": "Northeastern Tasmanian; Pyemmairrener",
+ "xpc": "Pecheneg",
+ "xpd": "Oyster Bay Tasmanian",
+ "xpe": "Liberia Kpelle",
+ "xpf": "Southeast Tasmanian; Nuenonne",
+ "xpg": "Phrygian",
+ "xph": "North Midlands Tasmanian; Tyerrenoterpanner",
+ "xpi": "Pictish",
+ "xpj": "Mpalitjanh",
+ "xpk": "Kulina Pano",
+ "xpl": "Port Sorell Tasmanian",
+ "xpm": "Pumpokol",
+ "xpn": "Kapinawá",
+ "xpo": "Pochutec",
+ "xpp": "Puyo-Paekche",
+ "xpq": "Mohegan-Pequot",
+ "xpr": "Parthian",
+ "xps": "Pisidian",
+ "xpt": "Punthamara",
+ "xpu": "Punic",
+ "xpv": "Northern Tasmanian; Tommeginne",
+ "xpw": "Northwestern Tasmanian; Peerapper",
+ "xpx": "Southwestern Tasmanian; Toogee",
+ "xpy": "Puyo",
+ "xpz": "Bruny Island Tasmanian",
+ "xqa": "Karakhanid",
+ "xqt": "Qatabanian",
+ "xra": "Krahô",
+ "xrb": "Eastern Karaboro",
+ "xrd": "Gundungurra",
+ "xre": "Kreye",
+ "xrg": "Minang",
+ "xri": "Krikati-Timbira",
+ "xrm": "Armazic",
+ "xrn": "Arin",
+ "xrr": "Raetic",
+ "xrt": "Aranama-Tamique",
+ "xru": "Marriammu",
+ "xrw": "Karawa",
+ "xsa": "Sabaean",
+ "xsb": "Sambal",
+ "xsc": "Scythian",
+ "xsd": "Sidetic",
+ "xse": "Sempan",
+ "xsh": "Shamang",
+ "xsi": "Sio",
+ "xsj": "Subi",
+ "xsl": "South Slavey",
+ "xsm": "Kasem",
+ "xsn": "Sanga (Nigeria)",
+ "xso": "Solano",
+ "xsp": "Silopi",
+ "xsq": "Makhuwa-Saka",
+ "xsr": "Sherpa",
+ "xss": "Assan",
+ "xsu": "Sanumá",
+ "xsv": "Sudovian",
+ "xsy": "Saisiyat",
+ "xta": "Alcozauca Mixtec",
+ "xtb": "Chazumba Mixtec",
+ "xtc": "Katcha-Kadugli-Miri",
+ "xtd": "Diuxi-Tilantongo Mixtec",
+ "xte": "Ketengban",
+ "xtg": "Transalpine Gaulish",
+ "xth": "Yitha Yitha",
+ "xti": "Sinicahua Mixtec",
+ "xtj": "San Juan Teita Mixtec",
+ "xtl": "Tijaltepec Mixtec",
+ "xtm": "Magdalena Peñasco Mixtec",
+ "xtn": "Northern Tlaxiaco Mixtec",
+ "xto": "Tokharian A",
+ "xtp": "San Miguel Piedras Mixtec",
+ "xtq": "Tumshuqese",
+ "xtr": "Early Tripuri",
+ "xts": "Sindihui Mixtec",
+ "xtt": "Tacahua Mixtec",
+ "xtu": "Cuyamecalco Mixtec",
+ "xtv": "Thawa",
+ "xtw": "Tawandê",
+ "xty": "Yoloxochitl Mixtec",
+ "xua": "Alu Kurumba",
+ "xub": "Betta Kurumba",
+ "xud": "Umiida",
+ "xug": "Kunigami",
+ "xuj": "Jennu Kurumba",
+ "xul": "Ngunawal; Nunukul",
+ "xum": "Umbrian",
+ "xun": "Unggaranggu",
+ "xuo": "Kuo",
+ "xup": "Upper Umpqua",
+ "xur": "Urartian",
+ "xut": "Kuthant",
+ "xuu": "Kxoe; Khwedam",
+ "xve": "Venetic",
+ "xvi": "Kamviri",
+ "xvn": "Vandalic",
+ "xvo": "Volscian",
+ "xvs": "Vestinian",
+ "xwa": "Kwaza",
+ "xwc": "Woccon",
+ "xwd": "Wadi Wadi",
+ "xwe": "Xwela Gbe",
+ "xwg": "Kwegu",
+ "xwj": "Wajuk",
+ "xwk": "Wangkumara",
+ "xwl": "Western Xwla Gbe",
+ "xwo": "Written Oirat",
+ "xwr": "Kwerba Mamberamo",
+ "xwt": "Wotjobaluk",
+ "xww": "Wemba Wemba",
+ "xxb": "Boro (Ghana)",
+ "xxk": "Ke'o",
+ "xxm": "Minkin",
+ "xxr": "Koropó",
+ "xxt": "Tambora",
+ "xya": "Yaygir",
+ "xyb": "Yandjibara",
+ "xyj": "Mayi-Yapi",
+ "xyk": "Mayi-Kulan",
+ "xyl": "Yalakalore",
+ "xyt": "Mayi-Thakurti",
+ "xyy": "Yorta Yorta",
+ "xzh": "Zhang-Zhung",
+ "xzm": "Zemgalian",
+ "xzp": "Ancient Zapotec",
+ "yaa": "Yaminahua",
+ "yab": "Yuhup",
+ "yac": "Pass Valley Yali",
+ "yad": "Yagua",
+ "yae": "Pumé",
+ "yaf": "Yaka (Democratic Republic of Congo)",
+ "yag": "Yámana",
+ "yah": "Yazgulyam",
+ "yai": "Yagnobi",
+ "yaj": "Banda-Yangere",
+ "yak": "Yakama",
+ "yal": "Yalunka",
+ "yam": "Yamba",
+ "yan": "Mayangna",
+ "yao": "Yao",
+ "yap": "Yapese",
+ "yaq": "Yaqui",
+ "yar": "Yabarana",
+ "yas": "Nugunu (Cameroon)",
+ "yat": "Yambeta",
+ "yau": "Yuwana",
+ "yav": "Yangben",
+ "yaw": "Yawalapití",
+ "yax": "Yauma",
+ "yay": "Agwagwune",
+ "yaz": "Lokaa",
+ "yba": "Yala",
+ "ybb": "Yemba",
+ "ybe": "West Yugur",
+ "ybh": "Yakha",
+ "ybi": "Yamphu",
+ "ybj": "Hasha",
+ "ybk": "Bokha",
+ "ybl": "Yukuben",
+ "ybm": "Yaben",
+ "ybn": "Yabaâna",
+ "ybo": "Yabong",
+ "ybx": "Yawiyo",
+ "yby": "Yaweyuha",
+ "ych": "Chesu",
+ "ycl": "Lolopo",
+ "ycn": "Yucuna",
+ "ycp": "Chepya",
+ "yda": "Yanda",
+ "ydd": "Eastern Yiddish",
+ "yde": "Yangum Dey",
+ "ydg": "Yidgha",
+ "ydk": "Yoidik",
+ "yea": "Ravula",
+ "yec": "Yeniche",
+ "yee": "Yimas",
+ "yei": "Yeni",
+ "yej": "Yevanic",
+ "yel": "Yela",
+ "yer": "Tarok",
+ "yes": "Nyankpa",
+ "yet": "Yetfa",
+ "yeu": "Yerukula",
+ "yev": "Yapunda",
+ "yey": "Yeyi",
+ "yga": "Malyangapa",
+ "ygi": "Yiningayi",
+ "ygl": "Yangum Gel",
+ "ygm": "Yagomi",
+ "ygp": "Gepo",
+ "ygr": "Yagaria",
+ "ygs": "Yolŋu Sign Language",
+ "ygu": "Yugul",
+ "ygw": "Yagwoia",
+ "yha": "Baha Buyang",
+ "yhd": "Judeo-Iraqi Arabic",
+ "yhl": "Hlepho Phowa",
+ "yhs": "Yan-nhaŋu Sign Language",
+ "yi": "Yiddish",
+ "yia": "Yinggarda",
+ "yif": "Ache",
+ "yig": "Wusa Nasu",
+ "yih": "Western Yiddish",
+ "yii": "Yidiny",
+ "yij": "Yindjibarndi",
+ "yik": "Dongshanba Lalo",
+ "yil": "Yindjilandji",
+ "yim": "Yimchungru Naga",
+ "yin": "Riang Lai; Yinchia",
+ "yip": "Pholo",
+ "yiq": "Miqie",
+ "yir": "North Awyu",
+ "yis": "Yis",
+ "yit": "Eastern Lalu",
+ "yiu": "Awu",
+ "yiv": "Northern Nisu",
+ "yix": "Axi Yi",
+ "yiz": "Azhe",
+ "yka": "Yakan",
+ "ykg": "Northern Yukaghir",
+ "yki": "Yoke",
+ "ykk": "Yakaikeke",
+ "ykl": "Khlula",
+ "ykm": "Kap",
+ "ykn": "Kua-nsi",
+ "yko": "Yasa",
+ "ykr": "Yekora",
+ "ykt": "Kathu",
+ "yku": "Kuamasi",
+ "yky": "Yakoma",
+ "yla": "Yaul",
+ "ylb": "Yaleba",
+ "yle": "Yele",
+ "ylg": "Yelogu",
+ "yli": "Angguruk Yali",
+ "yll": "Yil",
+ "ylm": "Limi",
+ "yln": "Langnian Buyang",
+ "ylo": "Naluo Yi",
+ "ylr": "Yalarnnga",
+ "ylu": "Aribwaung",
+ "yly": "Nyâlayu; Nyelâyu",
+ "ymb": "Yambes",
+ "ymc": "Southern Muji",
+ "ymd": "Muda",
+ "yme": "Yameo",
+ "ymg": "Yamongeri",
+ "ymh": "Mili",
+ "ymi": "Moji",
+ "ymk": "Makwe",
+ "yml": "Iamalele",
+ "ymm": "Maay",
+ "ymn": "Yamna; Sunum",
+ "ymo": "Yangum Mon",
+ "ymp": "Yamap",
+ "ymq": "Qila Muji",
+ "ymr": "Malasar",
+ "yms": "Mysian",
+ "ymx": "Northern Muji",
+ "ymz": "Muzi",
+ "yna": "Aluo",
+ "ynd": "Yandruwandha",
+ "yne": "Lang'e",
+ "yng": "Yango",
+ "ynk": "Naukan Yupik",
+ "ynl": "Yangulam",
+ "ynn": "Yana",
+ "yno": "Yong",
+ "ynq": "Yendang",
+ "yns": "Yansi",
+ "ynu": "Yahuna",
+ "yo": "Yoruba",
+ "yob": "Yoba",
+ "yog": "Yogad",
+ "yoi": "Yonaguni",
+ "yok": "Yokuts",
+ "yol": "Yola",
+ "yom": "Yombe",
+ "yon": "Yongkom",
+ "yot": "Yotti",
+ "yox": "Yoron",
+ "yoy": "Yoy",
+ "ypa": "Phala",
+ "ypb": "Labo Phowa",
+ "ypg": "Phola",
+ "yph": "Phupha",
+ "ypk": "Yupik languages",
+ "ypm": "Phuma",
+ "ypn": "Ani Phowa",
+ "ypo": "Alo Phola",
+ "ypp": "Phupa",
+ "ypz": "Phuza",
+ "yra": "Yerakai",
+ "yrb": "Yareba",
+ "yre": "Yaouré",
+ "yrk": "Nenets",
+ "yrl": "Nhengatu",
+ "yrm": "Yirrk-Mel",
+ "yrn": "Yerong",
+ "yro": "Yaroamë",
+ "yrs": "Yarsun",
+ "yrw": "Yarawata",
+ "yry": "Yarluyandi",
+ "ysc": "Yassic",
+ "ysd": "Samatao",
+ "ysg": "Sonaga",
+ "ysl": "Yugoslavian Sign Language",
+ "ysm": "Myanmar Sign Language",
+ "ysn": "Sani",
+ "yso": "Nisi (China)",
+ "ysp": "Southern Lolopo",
+ "ysr": "Sirenik Yupik",
+ "yss": "Yessan-Mayo",
+ "ysy": "Sanie",
+ "yta": "Talu",
+ "ytl": "Tanglang",
+ "ytp": "Thopho",
+ "ytw": "Yout Wam",
+ "yty": "Yatay",
+ "yua": "Yucateco; Yucatec Maya",
+ "yub": "Yugambal",
+ "yuc": "Yuchi",
+ "yud": "Judeo-Tripolitanian Arabic",
+ "yue": "Yue Chinese; Cantonese",
+ "yuf": "Havasupai-Walapai-Yavapai",
+ "yug": "Yug",
+ "yui": "Yurutí",
+ "yuj": "Karkar-Yuri",
+ "yuk": "Yuki",
+ "yul": "Yulu",
+ "yum": "Quechan",
+ "yun": "Bena (Nigeria)",
+ "yup": "Yukpa",
+ "yuq": "Yuqui",
+ "yur": "Yurok",
+ "yut": "Yopno",
+ "yuw": "Yau (Morobe Province)",
+ "yux": "Southern Yukaghir",
+ "yuy": "East Yugur",
+ "yuz": "Yuracare",
+ "yva": "Yawa",
+ "yvt": "Yavitero",
+ "ywa": "Kalou",
+ "ywg": "Yinhawangka",
+ "ywl": "Western Lalu",
+ "ywn": "Yawanawa",
+ "ywq": "Wuding-Luquan Yi",
+ "ywr": "Yawuru",
+ "ywt": "Xishanba Lalo; Central Lalo",
+ "ywu": "Wumeng Nasu",
+ "yww": "Yawarawarga",
+ "yxa": "Mayawali",
+ "yxg": "Yagara",
+ "yxl": "Yardliyawarra",
+ "yxm": "Yinwum",
+ "yxu": "Yuyu",
+ "yxy": "Yabula Yabula",
+ "yyr": "Yir Yoront",
+ "yyu": "Yau (Sandaun Province)",
+ "yyz": "Ayizi",
+ "yzg": "E'ma Buyang",
+ "yzk": "Zokhuo",
+ "za": "Zhuang; Chuang",
+ "zaa": "Sierra de Juárez Zapotec",
+ "zab": "Western Tlacolula Valley Zapotec; San Juan Guelavía Zapotec",
+ "zac": "Ocotlán Zapotec",
+ "zad": "Cajonos Zapotec",
+ "zae": "Yareni Zapotec",
+ "zaf": "Ayoquesco Zapotec",
+ "zag": "Zaghawa",
+ "zah": "Zangwal",
+ "zai": "Isthmus Zapotec",
+ "zaj": "Zaramo",
+ "zak": "Zanaki",
+ "zal": "Zauzou",
+ "zam": "Miahuatlán Zapotec",
+ "zao": "Ozolotepec Zapotec",
+ "zap": "Zapotec",
+ "zaq": "Aloápam Zapotec",
+ "zar": "Rincón Zapotec",
+ "zas": "Santo Domingo Albarradas Zapotec",
+ "zat": "Tabaa Zapotec",
+ "zau": "Zangskari",
+ "zav": "Yatzachi Zapotec",
+ "zaw": "Mitla Zapotec",
+ "zax": "Xadani Zapotec",
+ "zay": "Zayse-Zergulla; Zaysete",
+ "zaz": "Zari",
+ "zba": "Balaibalan",
+ "zbc": "Central Berawan",
+ "zbe": "East Berawan",
+ "zbl": "Blissymbols; Bliss; Blissymbolics",
+ "zbt": "Batui",
+ "zbu": "Bu (Bauchi State)",
+ "zbw": "West Berawan",
+ "zca": "Coatecas Altas Zapotec",
+ "zcd": "Las Delicias Zapotec",
+ "zch": "Central Hongshuihe Zhuang",
+ "zdj": "Ngazidja Comorian",
+ "zea": "Zeeuws",
+ "zeg": "Zenag",
+ "zeh": "Eastern Hongshuihe Zhuang",
+ "zen": "Zenaga",
+ "zga": "Kinga",
+ "zgb": "Guibei Zhuang",
+ "zgh": "Standard Moroccan Tamazight",
+ "zgm": "Minz Zhuang",
+ "zgn": "Guibian Zhuang",
+ "zgr": "Magori",
+ "zh": "Chinese",
+ "zhb": "Zhaba",
+ "zhd": "Dai Zhuang",
+ "zhi": "Zhire",
+ "zhn": "Nong Zhuang",
+ "zhw": "Zhoa",
+ "zhx": "Chinese (family)",
+ "zia": "Zia",
+ "zib": "Zimbabwe Sign Language",
+ "zik": "Zimakani",
+ "zil": "Zialo",
+ "zim": "Mesme",
+ "zin": "Zinza",
+ "ziw": "Zigula",
+ "ziz": "Zizilivakan",
+ "zka": "Kaimbulawa",
+ "zkb": "Koibal",
+ "zkd": "Kadu",
+ "zkg": "Koguryo",
+ "zkh": "Khorezmian",
+ "zkk": "Karankawa",
+ "zkn": "Kanan",
+ "zko": "Kott",
+ "zkp": "São Paulo Kaingáng",
+ "zkr": "Zakhring",
+ "zkt": "Kitan",
+ "zku": "Kaurna",
+ "zkv": "Krevinian",
+ "zkz": "Khazar",
+ "zla": "Zula",
+ "zle": "East Slavic languages",
+ "zlj": "Liujiang Zhuang",
+ "zlm": "Malay (individual language)",
+ "zln": "Lianshan Zhuang",
+ "zlq": "Liuqian Zhuang",
+ "zls": "South Slavic languages",
+ "zlw": "West Slavic languages",
+ "zma": "Manda (Australia)",
+ "zmb": "Zimba",
+ "zmc": "Margany",
+ "zmd": "Maridan",
+ "zme": "Mangerr",
+ "zmf": "Mfinu",
+ "zmg": "Marti Ke",
+ "zmh": "Makolkol",
+ "zmi": "Negeri Sembilan Malay",
+ "zmj": "Maridjabin",
+ "zmk": "Mandandanyi",
+ "zml": "Matngala",
+ "zmm": "Marimanindji; Marramaninyshi",
+ "zmn": "Mbangwe",
+ "zmo": "Molo",
+ "zmp": "Mpuono",
+ "zmq": "Mituku",
+ "zmr": "Maranunggu",
+ "zms": "Mbesa",
+ "zmt": "Maringarr",
+ "zmu": "Muruwari",
+ "zmv": "Mbariman-Gudhinma",
+ "zmw": "Mbo (Democratic Republic of Congo)",
+ "zmx": "Bomitaba",
+ "zmy": "Mariyedi",
+ "zmz": "Mbandja",
+ "zna": "Zan Gula",
+ "znd": "Zande languages",
+ "zne": "Zande (individual language)",
+ "zng": "Mang",
+ "znk": "Manangkari",
+ "zns": "Mangas",
+ "zoc": "Copainalá Zoque",
+ "zoh": "Chimalapa Zoque",
+ "zom": "Zou",
+ "zoo": "Asunción Mixtepec Zapotec",
+ "zoq": "Tabasco Zoque",
+ "zor": "Rayón Zoque",
+ "zos": "Francisco León Zoque",
+ "zpa": "Lachiguiri Zapotec",
+ "zpb": "Yautepec Zapotec",
+ "zpc": "Choapan Zapotec",
+ "zpd": "Southeastern Ixtlán Zapotec",
+ "zpe": "Petapa Zapotec",
+ "zpf": "San Pedro Quiatoni Zapotec",
+ "zpg": "Guevea De Humboldt Zapotec",
+ "zph": "Totomachapan Zapotec",
+ "zpi": "Santa María Quiegolani Zapotec",
+ "zpj": "Quiavicuzas Zapotec",
+ "zpk": "Tlacolulita Zapotec",
+ "zpl": "Lachixío Zapotec",
+ "zpm": "Mixtepec Zapotec",
+ "zpn": "Santa Inés Yatzechi Zapotec",
+ "zpo": "Amatlán Zapotec",
+ "zpp": "El Alto Zapotec",
+ "zpq": "Zoogocho Zapotec",
+ "zpr": "Santiago Xanica Zapotec",
+ "zps": "Coatlán Zapotec",
+ "zpt": "San Vicente Coatlán Zapotec",
+ "zpu": "Yalálag Zapotec",
+ "zpv": "Chichicapan Zapotec",
+ "zpw": "Zaniza Zapotec",
+ "zpx": "San Baltazar Loxicha Zapotec",
+ "zpy": "Mazaltepec Zapotec",
+ "zpz": "Texmelucan Zapotec",
+ "zqe": "Qiubei Zhuang",
+ "zra": "Kara (Korea)",
+ "zrg": "Mirgan",
+ "zrn": "Zerenkel",
+ "zro": "Záparo",
+ "zrp": "Zarphatic",
+ "zrs": "Mairasi",
+ "zsa": "Sarasira",
+ "zsk": "Kaskean",
+ "zsl": "Zambian Sign Language",
+ "zsm": "Standard Malay",
+ "zsr": "Southern Rincon Zapotec",
+ "zsu": "Sukurum",
+ "zte": "Elotepec Zapotec",
+ "ztg": "Xanaguía Zapotec",
+ "ztl": "Lapaguía-Guivini Zapotec",
+ "ztm": "San Agustín Mixtepec Zapotec",
+ "ztn": "Santa Catarina Albarradas Zapotec",
+ "ztp": "Loxicha Zapotec",
+ "ztq": "Quioquitani-Quierí Zapotec",
+ "zts": "Tilquiapan Zapotec",
+ "ztt": "Tejalapan Zapotec",
+ "ztu": "Güilá Zapotec",
+ "ztx": "Zaachila Zapotec",
+ "zty": "Yatee Zapotec",
+ "zu": "Zulu",
+ "zua": "Zeem",
+ "zuh": "Tokano",
+ "zum": "Kumzari",
+ "zun": "Zuni",
+ "zuy": "Zumaya",
+ "zwa": "Zay",
+ "zyb": "Yongbei Zhuang",
+ "zyg": "Yang Zhuang",
+ "zyj": "Youjiang Zhuang",
+ "zyn": "Yongnan Zhuang",
+ "zyp": "Zyphe Chin",
+ "zza": "Zaza; Dimili; Dimli (macrolanguage); Kirdki; Kirmanjki (macrolanguage); Zazaki",
+ "zzj": "Zuojiang Zhuang"
+}
\ No newline at end of file
diff --git a/venv/lib/python3.10/site-packages/datasets/utils/resources/multilingualities.json b/venv/lib/python3.10/site-packages/datasets/utils/resources/multilingualities.json
new file mode 100644
index 0000000000000000000000000000000000000000..a35c79f03dfcf7c8a116b7fc8ee1b383ab5022fa
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/utils/resources/multilingualities.json
@@ -0,0 +1,6 @@
+{
+ "monolingual": "contains a single language",
+ "multilingual": "contains multiple languages",
+ "translation": "contains translated or aligned text",
+ "other": "other type of language distribution"
+}
diff --git a/venv/lib/python3.10/site-packages/datasets/utils/resources/readme_structure.yaml b/venv/lib/python3.10/site-packages/datasets/utils/resources/readme_structure.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5b781e11e8258a446874ebf96104f642d0c190cf
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/utils/resources/readme_structure.yaml
@@ -0,0 +1,116 @@
+name: "" # Filename comes here
+allow_empty: false
+allow_empty_text: true
+subsections:
+ - name: "Dataset Card for X" # First-level markdown heading
+ allow_empty: false
+ allow_empty_text: true
+ subsections:
+ - name: "Table of Contents"
+ allow_empty: false
+ allow_empty_text: false
+ subsections: null # meaning it should not be checked.
+ - name: "Dataset Description"
+ allow_empty: false
+ allow_empty_text: false
+ subsections:
+ - name: "Dataset Summary"
+ allow_empty: false
+ allow_empty_text: false
+ subsections: null
+ - name: "Supported Tasks and Leaderboards"
+ allow_empty: true
+ allow_empty_text: true
+ subsections: null
+ - name: Languages
+ allow_empty: true
+ allow_empty_text: true
+ subsections: null
+ - name: "Dataset Structure"
+ allow_empty: false
+ allow_empty_text: true
+ subsections:
+ - name: "Data Instances"
+ allow_empty: false
+ allow_empty_text: true
+ subsections: null
+ - name: "Data Fields"
+ allow_empty: false
+ allow_empty_text: true
+ subsections: null
+ - name: "Data Splits"
+ allow_empty: false
+ allow_empty_text: true
+ subsections: null
+ - name: "Dataset Creation"
+ allow_empty: false
+ allow_empty_text: true
+ subsections:
+ - name: "Curation Rationale"
+ allow_empty: true
+ allow_empty_text: true
+ subsections: null
+ - name: "Source Data"
+ allow_empty: false
+ allow_empty_text: true
+ subsections:
+ - name: "Initial Data Collection and Normalization"
+ allow_empty: true
+ allow_empty_text: true
+ subsections: null
+ - name: "Who are the source language producers?"
+ allow_empty: true
+ allow_empty_text: true
+ subsections: null
+ - name: "Annotations"
+ allow_empty: false
+ allow_empty_text: true
+ subsections:
+ - name: "Annotation process"
+ allow_empty: true
+ allow_empty_text: true
+ subsections: null
+ - name: "Who are the annotators?"
+ allow_empty: true
+ allow_empty_text: true
+ subsections: null
+ - name: "Personal and Sensitive Information"
+ allow_empty: true
+ allow_empty_text: true
+ subsections: null
+ - name: "Considerations for Using the Data"
+ allow_empty: true
+ allow_empty_text: true
+ subsections:
+ - name: "Social Impact of Dataset"
+ allow_empty: true
+ allow_empty_text: true
+ subsections: null
+ - name: "Discussion of Biases"
+ allow_empty: true
+ allow_empty_text: true
+ subsections: null
+ - name: "Other Known Limitations"
+ allow_empty: true
+ allow_empty_text: true
+ subsections: null
+ - name: "Additional Information"
+ allow_empty: true
+ allow_empty_text: true
+ subsections:
+ - name: "Dataset Curators"
+ allow_empty: true
+ allow_empty_text: true
+ subsections: null
+ - name: "Licensing Information"
+ allow_empty: true
+ allow_empty_text: true
+ subsections: null
+ - name: "Citation Information"
+ allow_empty: false
+ allow_empty_text: true
+ subsections: null
+ - name: "Contributions"
+ allow_empty: false
+ allow_empty_text: false
+ subsections: null
diff --git a/venv/lib/python3.10/site-packages/datasets/utils/resources/size_categories.json b/venv/lib/python3.10/site-packages/datasets/utils/resources/size_categories.json
new file mode 100644
index 0000000000000000000000000000000000000000..983ce0c10dbb2e2245f90ae47e9de4c1025d5bb1
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/utils/resources/size_categories.json
@@ -0,0 +1,14 @@
+[
+ "unknown",
+ "n<1K",
+ "1K1T"
+]
diff --git a/venv/lib/python3.10/site-packages/datasets/utils/sharding.py b/venv/lib/python3.10/site-packages/datasets/utils/sharding.py
new file mode 100644
index 0000000000000000000000000000000000000000..753bdf60c2a5b0b23f70893de5ba57a51462d126
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/utils/sharding.py
@@ -0,0 +1,92 @@
+import numpy as np
+
+
+def _number_of_shards_in_gen_kwargs(gen_kwargs: dict) -> int:
+ """Return the number of possible shards according to the input gen_kwargs"""
+ # Having lists of different sizes makes sharding ambigious, raise an error in this case
+ # until we decide how to define sharding without ambiguity for users
+ lists_lengths = {key: len(value) for key, value in gen_kwargs.items() if isinstance(value, list)}
+ if len(set(lists_lengths.values())) > 1:
+ raise RuntimeError(
+ "Sharding is ambiguous for this dataset: "
+ + "we found several data sources lists of different lengths, and we don't know over which list we should parallelize:\n"
+ + "\n".join(f"\t- key {key} has length {length}" for key, length in lists_lengths.items())
+ + "\nTo fix this, check the 'gen_kwargs' and make sure to use lists only for data sources, "
+ + "and use tuples otherwise. In the end there should only be one single list, or several lists with the same length."
+ )
+ max_length = max(lists_lengths.values(), default=0)
+ return max(1, max_length)
+
+
+def _distribute_shards(num_shards: int, max_num_jobs: int) -> list[range]:
+ """
+ Get the range of shard indices per job.
+ If num_shards>> _distribute_shards(2, max_num_jobs=4)
+ [range(0, 1), range(1, 2)]
+ >>> _distribute_shards(10, max_num_jobs=3)
+ [range(0, 4), range(4, 7), range(7, 10)]
+ ```
+ """
+ shards_indices_per_group = []
+ for group_idx in range(max_num_jobs):
+ num_shards_to_add = num_shards // max_num_jobs + (group_idx < (num_shards % max_num_jobs))
+ if num_shards_to_add == 0:
+ break
+ start = shards_indices_per_group[-1].stop if shards_indices_per_group else 0
+ shard_indices = range(start, start + num_shards_to_add)
+ shards_indices_per_group.append(shard_indices)
+ return shards_indices_per_group
+
+
+def _split_gen_kwargs(gen_kwargs: dict, max_num_jobs: int) -> list[dict]:
+ """Split the gen_kwargs into `max_num_job` gen_kwargs"""
+ # Having lists of different sizes makes sharding ambigious, raise an error in this case
+ num_shards = _number_of_shards_in_gen_kwargs(gen_kwargs)
+ if num_shards == 1:
+ return [dict(gen_kwargs)]
+ else:
+ shard_indices_per_group = _distribute_shards(num_shards=num_shards, max_num_jobs=max_num_jobs)
+ return [
+ {
+ key: [value[shard_idx] for shard_idx in shard_indices_per_group[group_idx]]
+ if isinstance(value, list)
+ else value
+ for key, value in gen_kwargs.items()
+ }
+ for group_idx in range(len(shard_indices_per_group))
+ ]
+
+
+def _merge_gen_kwargs(gen_kwargs_list: list[dict]) -> dict:
+ return {
+ key: [value for gen_kwargs in gen_kwargs_list for value in gen_kwargs[key]]
+ if isinstance(gen_kwargs_list[0][key], list)
+ else gen_kwargs_list[0][key]
+ for key in gen_kwargs_list[0]
+ }
+
+
+def _shuffle_gen_kwargs(rng: np.random.Generator, gen_kwargs: dict) -> dict:
+ """Return a shuffled copy of the input gen_kwargs"""
+ # We must shuffle all the lists, and lists of the same size must have the same shuffling.
+ # This way entangled lists of (shard, shard_metadata) are still in the right order.
+
+ # First, let's generate the shuffled indices per list size
+ list_sizes = {len(value) for value in gen_kwargs.values() if isinstance(value, list)}
+ indices_per_size = {}
+ for size in list_sizes:
+ indices_per_size[size] = list(range(size))
+ rng.shuffle(indices_per_size[size])
+ # Now let's copy the gen_kwargs and shuffle the lists based on their sizes
+ shuffled_kwargs = dict(gen_kwargs)
+ for key, value in shuffled_kwargs.items():
+ if isinstance(value, list):
+ shuffled_kwargs[key] = [value[i] for i in indices_per_size[len(value)]]
+ return shuffled_kwargs
diff --git a/venv/lib/python3.10/site-packages/datasets/utils/stratify.py b/venv/lib/python3.10/site-packages/datasets/utils/stratify.py
new file mode 100644
index 0000000000000000000000000000000000000000..6dab0469e8168148a81eb2c13f7c103b98d061e7
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/utils/stratify.py
@@ -0,0 +1,107 @@
+import numpy as np
+
+
+def approximate_mode(class_counts, n_draws, rng):
+ """Computes approximate mode of multivariate hypergeometric.
+ This is an approximation to the mode of the multivariate
+ hypergeometric given by class_counts and n_draws.
+ It shouldn't be off by more than one.
+ It is the mostly likely outcome of drawing n_draws many
+ samples from the population given by class_counts.
+ Args
+ ----------
+ class_counts : ndarray of int
+ Population per class.
+ n_draws : int
+ Number of draws (samples to draw) from the overall population.
+ rng : random state
+ Used to break ties.
+ Returns
+ -------
+ sampled_classes : ndarray of int
+ Number of samples drawn from each class.
+ np.sum(sampled_classes) == n_draws
+
+ """
+ # this computes a bad approximation to the mode of the
+ # multivariate hypergeometric given by class_counts and n_draws
+ continuous = n_draws * class_counts / class_counts.sum()
+ # floored means we don't overshoot n_samples, but probably undershoot
+ floored = np.floor(continuous)
+ # we add samples according to how much "left over" probability
+ # they had, until we arrive at n_samples
+ need_to_add = int(n_draws - floored.sum())
+ if need_to_add > 0:
+ remainder = continuous - floored
+ values = np.sort(np.unique(remainder))[::-1]
+ # add according to remainder, but break ties
+ # randomly to avoid biases
+ for value in values:
+ (inds,) = np.where(remainder == value)
+ # if we need_to_add less than what's in inds
+ # we draw randomly from them.
+ # if we need to add more, we add them all and
+ # go to the next value
+ add_now = min(len(inds), need_to_add)
+ inds = rng.choice(inds, size=add_now, replace=False)
+ floored[inds] += 1
+ need_to_add -= add_now
+ if need_to_add == 0:
+ break
+ return floored.astype(np.int64)
+
+
+def stratified_shuffle_split_generate_indices(y, n_train, n_test, rng, n_splits=10):
+ """
+
+ Provides train/test indices to split data in train/test sets.
+ It's reference is taken from StratifiedShuffleSplit implementation
+ of scikit-learn library.
+
+ Args
+ ----------
+
+ n_train : int,
+ represents the absolute number of train samples.
+
+ n_test : int,
+ represents the absolute number of test samples.
+
+ random_state : int or RandomState instance, default=None
+ Controls the randomness of the training and testing indices produced.
+ Pass an int for reproducible output across multiple function calls.
+
+ n_splits : int, default=10
+ Number of re-shuffling & splitting iterations.
+ """
+ classes, y_indices = np.unique(y, return_inverse=True)
+ n_classes = classes.shape[0]
+ class_counts = np.bincount(y_indices)
+ if np.min(class_counts) < 2:
+ raise ValueError("Minimum class count error")
+ if n_train < n_classes:
+ raise ValueError(
+ "The train_size = %d should be greater or equal to the number of classes = %d" % (n_train, n_classes)
+ )
+ if n_test < n_classes:
+ raise ValueError(
+ "The test_size = %d should be greater or equal to the number of classes = %d" % (n_test, n_classes)
+ )
+ class_indices = np.split(np.argsort(y_indices, kind="mergesort"), np.cumsum(class_counts)[:-1])
+ for _ in range(n_splits):
+ n_i = approximate_mode(class_counts, n_train, rng)
+ class_counts_remaining = class_counts - n_i
+ t_i = approximate_mode(class_counts_remaining, n_test, rng)
+
+ train = []
+ test = []
+
+ for i in range(n_classes):
+ permutation = rng.permutation(class_counts[i])
+ perm_indices_class_i = class_indices[i].take(permutation, mode="clip")
+ train.extend(perm_indices_class_i[: n_i[i]])
+ test.extend(perm_indices_class_i[n_i[i] : n_i[i] + t_i[i]])
+ train = rng.permutation(train)
+ test = rng.permutation(test)
+
+ yield train, test
diff --git a/venv/lib/python3.10/site-packages/datasets/utils/tf_utils.py b/venv/lib/python3.10/site-packages/datasets/utils/tf_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..10ae3ef64ae5e14339bf0e855c010253b92a7e02
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/utils/tf_utils.py
@@ -0,0 +1,564 @@
+# Copyright 2022 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""TF-specific utils import."""
+
+import os
+import warnings
+from functools import partial
+from math import ceil
+from uuid import uuid4
+
+import numpy as np
+import pyarrow as pa
+from multiprocess import get_context
+
+
+try:
+ from multiprocess.shared_memory import SharedMemory
+except ImportError:
+ SharedMemory = None # Version checks should prevent this being called on older Python versions
+
+from .. import config
+
+
+def minimal_tf_collate_fn(features):
+ if isinstance(features, dict): # case batch_size=None: nothing to collate
+ return features
+ elif config.TF_AVAILABLE:
+ import tensorflow as tf
+ else:
+ raise ImportError("Called a Tensorflow-specific function but Tensorflow is not installed.")
+
+ first = features[0]
+ batch = {}
+ for k, v in first.items():
+ if isinstance(v, np.ndarray):
+ batch[k] = np.stack([f[k] for f in features])
+ elif isinstance(v, tf.Tensor):
+ batch[k] = tf.stack([f[k] for f in features])
+ else:
+ batch[k] = np.array([f[k] for f in features])
+ return batch
+
+
+def minimal_tf_collate_fn_with_renaming(features):
+ batch = minimal_tf_collate_fn(features)
+ if "label" in batch:
+ batch["labels"] = batch["label"]
+ del batch["label"]
+ return batch
+
+
+def is_numeric_pa_type(pa_type):
+ if pa.types.is_list(pa_type):
+ return is_numeric_pa_type(pa_type.value_type)
+ return pa.types.is_integer(pa_type) or pa.types.is_floating(pa_type) or pa.types.is_decimal(pa_type)
+
+
+def np_get_batch(
+ indices, dataset, cols_to_retain, collate_fn, collate_fn_args, columns_to_np_types, return_dict=False
+):
+ if not isinstance(indices, np.ndarray):
+ indices = indices.numpy()
+
+ is_batched = True
+ # Optimization - if we're loading a sequential batch, do it with slicing instead of a list of indices
+ if isinstance(indices, np.integer):
+ batch = dataset[indices.item()]
+ is_batched = False
+ elif np.all(np.diff(indices) == 1):
+ batch = dataset[indices[0] : indices[-1] + 1]
+ elif isinstance(indices, np.ndarray):
+ batch = dataset[indices]
+ else:
+ raise RuntimeError(f"Unexpected type for indices: {type(indices)}")
+
+ if cols_to_retain is not None:
+ batch = {
+ key: value
+ for key, value in batch.items()
+ if key in cols_to_retain or key in ("label", "label_ids", "labels")
+ }
+
+ if is_batched:
+ actual_size = len(list(batch.values())[0]) # Get the length of one of the arrays, assume all same
+ # Our collators expect a list of dicts, not a dict of lists/arrays, so we invert
+ batch = [{key: value[i] for key, value in batch.items()} for i in range(actual_size)]
+ batch = collate_fn(batch, **collate_fn_args)
+
+ if return_dict:
+ out_batch = {}
+ for col, cast_dtype in columns_to_np_types.items():
+ # In case the collate_fn returns something strange
+ array = np.array(batch[col])
+ array = array.astype(cast_dtype)
+ out_batch[col] = array
+ else:
+ out_batch = []
+ for col, cast_dtype in columns_to_np_types.items():
+ # In case the collate_fn returns something strange
+ array = np.array(batch[col])
+ array = array.astype(cast_dtype)
+ out_batch.append(array)
+ return out_batch
+
+
+def dataset_to_tf(
+ dataset,
+ cols_to_retain,
+ collate_fn,
+ collate_fn_args,
+ columns_to_np_types,
+ output_signature,
+ shuffle,
+ batch_size,
+ drop_remainder,
+):
+ """Create a tf.data.Dataset from the underlying Dataset. This is a single-process method - the multiprocess
+ equivalent is multiprocess_dataset_to_tf.
+
+ Args:
+ dataset (`Dataset`): Dataset to wrap with tf.data.Dataset.
+ cols_to_retain (`List[str]`): Dataset column(s) to load in the
+ tf.data.Dataset. It is acceptable to include column names that are created by the `collate_fn` and
+ that do not exist in the original dataset.
+ collate_fn(`Callable`): A function or callable object (such as a `DataCollator`) that will collate
+ lists of samples into a batch.
+ collate_fn_args (`Dict`): A `dict` of keyword arguments to be passed to the
+ `collate_fn`. Can be empty.
+ columns_to_np_types (`Dict[str, np.dtype]`): A `dict` mapping column names to numpy dtypes.
+ output_signature (`Dict[str, tf.TensorSpec]`): A `dict` mapping column names to
+ `tf.TensorSpec` objects.
+ shuffle(`bool`): Shuffle the dataset order when loading. Recommended True for training, False for
+ validation/evaluation.
+ batch_size (`int`, default `None`): Size of batches to load from the dataset. Defaults to `None`, which implies that
+ the dataset won't be batched, but the returned dataset can be batched later with `tf_dataset.batch(batch_size)`.
+ drop_remainder(`bool`, default `None`): Drop the last incomplete batch when loading. If not provided,
+ defaults to the same setting as shuffle.
+
+ Returns:
+ `tf.data.Dataset`
+ """
+ if config.TF_AVAILABLE:
+ import tensorflow as tf
+ else:
+ raise ImportError("Called a Tensorflow-specific function but Tensorflow is not installed.")
+
+ # TODO Matt: When our minimum Python version is 3.8 or higher, we can delete all of this and move everything
+ # to the NumPy multiprocessing path.
+ if hasattr(tf, "random_index_shuffle"):
+ random_index_shuffle = tf.random_index_shuffle
+ elif hasattr(tf.random.experimental, "index_shuffle"):
+ random_index_shuffle = tf.random.experimental.index_shuffle
+ else:
+ if len(dataset) > 10_000_000:
+ warnings.warn(
+ "to_tf_dataset() can be memory-inefficient on versions of TensorFlow older than 2.9. "
+ "If you are iterating over a dataset with a very large number of samples, consider "
+ "upgrading to TF >= 2.9."
+ )
+ random_index_shuffle = None
+
+ getter_fn = partial(
+ np_get_batch,
+ dataset=dataset,
+ cols_to_retain=cols_to_retain,
+ collate_fn=collate_fn,
+ collate_fn_args=collate_fn_args,
+ columns_to_np_types=columns_to_np_types,
+ return_dict=False,
+ )
+
+ # This works because dictionaries always output in the same order
+ tout = [tf.dtypes.as_dtype(dtype) for dtype in columns_to_np_types.values()]
+
+ @tf.function(input_signature=[tf.TensorSpec(None, tf.int64)])
+ def fetch_function(indices):
+ output = tf.py_function(
+ getter_fn,
+ inp=[indices],
+ Tout=tout,
+ )
+ return {key: output[i] for i, key in enumerate(columns_to_np_types.keys())}
+
+ tf_dataset = tf.data.Dataset.range(len(dataset))
+
+ if shuffle and random_index_shuffle is not None:
+ base_seed = tf.fill((3,), value=tf.cast(-1, dtype=tf.int64))
+
+ def scan_random_index(state, index):
+ if tf.reduce_all(state == -1):
+ # This generates a new random seed once per epoch only,
+ # to ensure that we iterate over each sample exactly once per epoch
+ state = tf.random.uniform(shape=(3,), maxval=2**62, dtype=tf.int64)
+ shuffled_index = random_index_shuffle(index=index, seed=state, max_index=len(dataset) - 1)
+ return state, shuffled_index
+
+ tf_dataset = tf_dataset.scan(base_seed, scan_random_index)
+ elif shuffle:
+ tf_dataset = tf_dataset.shuffle(tf_dataset.cardinality())
+
+ if batch_size is not None:
+ tf_dataset = tf_dataset.batch(batch_size, drop_remainder=drop_remainder)
+
+ tf_dataset = tf_dataset.map(fetch_function)
+
+ if batch_size is not None:
+
+ def ensure_shapes(input_dict):
+ return {key: tf.ensure_shape(val, output_signature[key].shape) for key, val in input_dict.items()}
+
+ else:
+ # Ensure shape but remove batch dimension of output_signature[key].shape
+ def ensure_shapes(input_dict):
+ return {key: tf.ensure_shape(val, output_signature[key].shape[1:]) for key, val in input_dict.items()}
+
+ return tf_dataset.map(ensure_shapes)
+
+
+class SharedMemoryContext:
+ # This is a context manager for creating shared memory that ensures cleanup happens even if a process is interrupted
+ # The process that creates shared memory is always the one responsible for unlinking it in the end
+ def __init__(self):
+ self.created_shms = []
+ self.opened_shms = []
+
+ def get_shm(self, name, size, create):
+ shm = SharedMemory(size=int(size), name=name, create=create)
+ if create:
+ # We only unlink the ones we created in this context
+ self.created_shms.append(shm)
+ else:
+ # If we didn't create it, we only close it when done, we don't unlink it
+ self.opened_shms.append(shm)
+ return shm
+
+ def get_array(self, name, shape, dtype, create):
+ shm = self.get_shm(name=name, size=np.prod(shape) * np.dtype(dtype).itemsize, create=create)
+ return np.ndarray(shape, dtype=dtype, buffer=shm.buf)
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ for shm in self.created_shms:
+ shm.close()
+ shm.unlink()
+ for shm in self.opened_shms:
+ shm.close()
+
+
+class NumpyMultiprocessingGenerator:
+ def __init__(
+ self,
+ dataset,
+ cols_to_retain,
+ collate_fn,
+ collate_fn_args,
+ columns_to_np_types,
+ output_signature,
+ shuffle,
+ batch_size,
+ drop_remainder,
+ num_workers,
+ ):
+ self.dataset = dataset
+ self.cols_to_retain = cols_to_retain
+ self.collate_fn = collate_fn
+ self.collate_fn_args = collate_fn_args
+ self.string_columns = [col for col, dtype in columns_to_np_types.items() if dtype is np.str_]
+ # Strings will be converted to arrays of single unicode chars, so that we can have a constant itemsize
+ self.columns_to_np_types = {
+ col: dtype if col not in self.string_columns else np.dtype("U1")
+ for col, dtype in columns_to_np_types.items()
+ }
+ self.output_signature = output_signature
+ self.shuffle = shuffle
+ self.batch_size = batch_size
+ self.drop_remainder = drop_remainder
+ self.num_workers = num_workers
+ # Because strings are converted to characters, we need to add one extra dimension to the shape
+ self.columns_to_ranks = {
+ col: int(spec.shape.rank) if col not in self.string_columns else int(spec.shape.rank) + 1
+ for col, spec in output_signature.items()
+ }
+
+ def __iter__(self):
+ # Make sure we only spawn workers if they have work to do
+ num_workers = min(self.num_workers, int(ceil(len(self.dataset) / self.batch_size)))
+ # Do the shuffling in iter so that it's done at the start of each epoch
+ per_worker_batches, final_batch, final_batch_worker = self.distribute_batches(
+ self.dataset, self.batch_size, self.drop_remainder, num_workers, self.shuffle
+ )
+ ctx = get_context("spawn")
+ names = []
+ shape_arrays = []
+ workers = []
+ array_ready_events = [ctx.Event() for _ in range(num_workers)]
+ array_loaded_events = [ctx.Event() for _ in range(num_workers)]
+
+ base_args = {
+ "dataset": self.dataset,
+ "cols_to_retain": self.cols_to_retain,
+ "collate_fn": self.collate_fn,
+ "collate_fn_args": self.collate_fn_args,
+ "columns_to_np_types": self.columns_to_np_types,
+ "columns_to_ranks": self.columns_to_ranks,
+ "string_columns": self.string_columns,
+ }
+ with SharedMemoryContext() as shm_ctx:
+ for i in range(num_workers):
+ worker_random_id = str(uuid4())
+ worker_name = f"dw_{i}_{worker_random_id}"[:10]
+ names.append(worker_name)
+
+ worker_shape_arrays = {
+ col: shm_ctx.get_array(f"{worker_name}_{col}_shape", shape=(rank,), dtype=np.int64, create=True)
+ for col, rank in self.columns_to_ranks.items()
+ }
+ shape_arrays.append(worker_shape_arrays)
+
+ worker_indices = per_worker_batches[i]
+ if i == final_batch_worker and final_batch is not None:
+ final_batch_arg = final_batch
+ else:
+ final_batch_arg = None
+ worker_kwargs = {
+ "worker_name": worker_name,
+ "indices": worker_indices,
+ "extra_batch": final_batch_arg,
+ "array_ready_event": array_ready_events[i],
+ "array_loaded_event": array_loaded_events[i],
+ **base_args,
+ }
+ worker = ctx.Process(target=self.worker_loop, kwargs=worker_kwargs, daemon=True)
+ worker.start()
+ workers.append(worker)
+
+ end_signal_received = False
+ while not end_signal_received:
+ for i in range(num_workers):
+ if not array_ready_events[i].wait(timeout=60):
+ raise TimeoutError("Data loading worker timed out!")
+ array_ready_events[i].clear()
+ array_shapes = shape_arrays[i]
+ if any(np.any(shape < 0) for shape in array_shapes.values()):
+ # Child processes send negative array shapes to indicate
+ # that no more data is going to be sent
+ end_signal_received = True
+ break
+ # Matt: Because array shapes are variable we recreate the shared memory each iteration.
+ # I suspect repeatedly opening lots of shared memory is the bottleneck for the parent process.
+ # A future optimization, at the cost of some code complexity, could be to reuse shared memory
+ # between iterations, but this would require knowing in advance the maximum size, or having
+ # a system to only create a new memory block when a new maximum size is seen.
+ # Another potential optimization would be to figure out which memory copies are necessary,
+ # or whether we can yield objects straight out of shared memory.
+ with SharedMemoryContext() as batch_shm_ctx:
+ # This memory context only lasts long enough to copy everything out of the batch
+ arrays = {
+ col: batch_shm_ctx.get_array(
+ f"{names[i]}_{col}",
+ shape=shape,
+ dtype=self.columns_to_np_types[col],
+ create=False,
+ )
+ for col, shape in array_shapes.items()
+ }
+ # Copy everything out of shm because the memory
+ # will be unlinked by the child process at some point
+ arrays = {col: np.copy(arr) for col, arr in arrays.items()}
+ # Now we convert any unicode char arrays to strings
+ for string_col in self.string_columns:
+ arrays[string_col] = (
+ arrays[string_col].view(f"U{arrays[string_col].shape[-1]}").squeeze(-1)
+ )
+ yield arrays
+ array_loaded_events[i].set()
+ # Now we just do some cleanup
+ # Shared memory is cleaned up by the context manager, so we just make sure workers finish
+ for worker in workers:
+ worker.join()
+
+ def __call__(self):
+ return self
+
+ @staticmethod
+ def worker_loop(
+ dataset,
+ cols_to_retain,
+ collate_fn,
+ collate_fn_args,
+ columns_to_np_types,
+ columns_to_ranks,
+ string_columns,
+ indices,
+ extra_batch,
+ worker_name,
+ array_ready_event,
+ array_loaded_event,
+ ):
+ os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"
+
+ if config.TF_AVAILABLE:
+ import tensorflow as tf
+ else:
+ raise ImportError("Called a Tensorflow-specific function but Tensorflow is not installed.")
+
+ tf.config.set_visible_devices([], "GPU") # Make sure workers don't try to allocate GPU memory
+
+ def send_batch_to_parent(indices):
+ batch = np_get_batch(
+ indices=indices,
+ dataset=dataset,
+ cols_to_retain=cols_to_retain,
+ collate_fn=collate_fn,
+ collate_fn_args=collate_fn_args,
+ columns_to_np_types=columns_to_np_types,
+ return_dict=True,
+ )
+
+ # Now begins the fun part where we start shovelling shared memory at the parent process
+ out_arrays = {}
+ with SharedMemoryContext() as batch_shm_ctx:
+ # The batch shared memory context exists only as long as it takes for the parent process
+ # to read everything, after which it cleans everything up again
+ for col, cast_dtype in columns_to_np_types.items():
+ # Everything has to be np.array for this to work, even if the collate_fn is giving us tf.Tensor
+ array = batch[col]
+ if col in string_columns:
+ # We can't send unicode arrays over shared memory, so we convert to single chars ("U1")
+ # which have a fixed width of 4 bytes. The parent process will convert these back to strings.
+ array = array.view("U1").reshape(array.shape + (-1,))
+ shape_arrays[col][:] = array.shape
+ out_arrays[col] = batch_shm_ctx.get_array(
+ f"{worker_name}_{col}", shape=array.shape, dtype=cast_dtype, create=True
+ )
+ out_arrays[col][:] = array
+
+ array_ready_event.set()
+ array_loaded_event.wait()
+ array_loaded_event.clear()
+
+ with SharedMemoryContext() as shm_ctx:
+ shape_arrays = {
+ col: shm_ctx.get_array(f"{worker_name}_{col}_shape", shape=(rank,), dtype=np.int64, create=False)
+ for col, rank in columns_to_ranks.items()
+ }
+
+ for batch in indices:
+ send_batch_to_parent(batch)
+ if extra_batch is not None:
+ send_batch_to_parent(extra_batch)
+ # Now we send a batsignal to the parent process that we're done
+ for col, array in shape_arrays.items():
+ array[:] = -1
+ array_ready_event.set()
+
+ @staticmethod
+ def distribute_batches(dataset, batch_size, drop_remainder, num_workers, shuffle):
+ indices = np.arange(len(dataset))
+ if shuffle:
+ np.random.shuffle(indices)
+ num_samples = len(indices)
+ # We distribute the batches so that reading from the workers in round-robin order yields the exact
+ # order specified in indices. This is only important when shuffle is False, but we do it regardless.
+ incomplete_batch_cutoff = num_samples - (num_samples % batch_size)
+ indices, last_incomplete_batch = np.split(indices, [incomplete_batch_cutoff])
+ if drop_remainder or len(last_incomplete_batch) == 0:
+ last_incomplete_batch = None
+
+ indices = indices.reshape(-1, batch_size)
+ num_batches = len(indices)
+ final_batches_cutoff = num_batches - (num_batches % num_workers)
+ indices, final_batches = np.split(indices, [final_batches_cutoff])
+ indices = indices.reshape(-1, num_workers, batch_size)
+
+ per_worker_indices = np.split(indices, indices.shape[1], axis=1)
+ per_worker_indices = [np.squeeze(worker_indices, 1) for worker_indices in per_worker_indices]
+ # Distribute the final batches to the first workers
+ for i in range(len(final_batches)):
+ # len(final_batches) can be zero, and is always less than num_workers
+ per_worker_indices[i] = np.concatenate([per_worker_indices[i], final_batches[i].reshape(1, -1)], axis=0)
+ # Add the last incomplete batch to the next worker, which might be the first worker
+ if last_incomplete_batch is not None:
+ incomplete_batch_worker_idx = len(final_batches)
+ else:
+ incomplete_batch_worker_idx = None
+ return per_worker_indices, last_incomplete_batch, incomplete_batch_worker_idx
+
+
+def multiprocess_dataset_to_tf(
+ dataset,
+ cols_to_retain,
+ collate_fn,
+ collate_fn_args,
+ columns_to_np_types,
+ output_signature,
+ shuffle,
+ batch_size,
+ drop_remainder,
+ num_workers,
+):
+ """Create a tf.data.Dataset from the underlying Dataset. This is a multi-process method - the single-process
+ equivalent is dataset_to_tf.
+
+ Args:
+ dataset (`Dataset`): Dataset to wrap with tf.data.Dataset.
+ cols_to_retain (`List[str]`): Dataset column(s) to load in the
+ tf.data.Dataset. It is acceptable to include column names that are created by the `collate_fn` and
+ that do not exist in the original dataset.
+ collate_fn(`Callable`): A function or callable object (such as a `DataCollator`) that will collate
+ lists of samples into a batch.
+ collate_fn_args (`Dict`): A `dict` of keyword arguments to be passed to the
+ `collate_fn`. Can be empty.
+ columns_to_np_types (`Dict[str, np.dtype]`): A `dict` mapping column names to numpy dtypes.
+ output_signature (`Dict[str, tf.TensorSpec]`): A `dict` mapping column names to
+ `tf.TensorSpec` objects.
+ shuffle(`bool`): Shuffle the dataset order when loading. Recommended True for training, False for
+ validation/evaluation.
+ batch_size (`int`, default `None`): Size of batches to load from the dataset. Defaults to `None`, which implies that
+ the dataset won't be batched, but the returned dataset can be batched later with `tf_dataset.batch(batch_size)`.
+ drop_remainder(`bool`, default `None`): Drop the last incomplete batch when loading. If not provided,
+ defaults to the same setting as shuffle.
+ num_workers (`int`): Number of workers to use for loading the dataset. Should be >= 1.
+
+ Returns:
+ `tf.data.Dataset`
+ """
+ if config.TF_AVAILABLE:
+ import tensorflow as tf
+ else:
+ raise ImportError("Called a Tensorflow-specific function but Tensorflow is not installed.")
+
+ data_generator = NumpyMultiprocessingGenerator(
+ dataset=dataset,
+ cols_to_retain=cols_to_retain,
+ collate_fn=collate_fn,
+ collate_fn_args=collate_fn_args,
+ columns_to_np_types=columns_to_np_types,
+ output_signature=output_signature,
+ shuffle=shuffle,
+ batch_size=batch_size,
+ drop_remainder=drop_remainder,
+ num_workers=num_workers,
+ )
+
+ tf_dataset = tf.data.Dataset.from_generator(data_generator, output_signature=output_signature)
+ if drop_remainder:
+ dataset_length = int(len(dataset) // batch_size)
+ else:
+ dataset_length = int(ceil(len(dataset) / batch_size))
+ return tf_dataset.apply(tf.data.experimental.assert_cardinality(dataset_length))
diff --git a/venv/lib/python3.10/site-packages/datasets/utils/tqdm.py b/venv/lib/python3.10/site-packages/datasets/utils/tqdm.py
new file mode 100644
index 0000000000000000000000000000000000000000..e28a8ff7ccfda7d4fd9a6195636d181f285ceb65
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/utils/tqdm.py
@@ -0,0 +1,131 @@
+"""Utility helpers to handle progress bars in `datasets`.
+
+Example:
+ 1. Use `datasets.utils.tqdm` as you would use `tqdm.tqdm` or `tqdm.auto.tqdm`.
+ 2. To disable progress bars, either use `disable_progress_bars()` helper or set the
+ environment variable `HF_DATASETS_DISABLE_PROGRESS_BARS` to 1.
+ 3. To re-enable progress bars, use `enable_progress_bars()`.
+ 4. To check whether progress bars are disabled, use `are_progress_bars_disabled()`.
+
+NOTE: Environment variable `HF_DATASETS_DISABLE_PROGRESS_BARS` has the priority.
+
+Example:
+ ```py
+ from datasets.utils import (
+ are_progress_bars_disabled,
+ disable_progress_bars,
+ enable_progress_bars,
+ tqdm,
+ )
+
+ # Disable progress bars globally
+ disable_progress_bars()
+
+ # Use as normal `tqdm`
+ for _ in tqdm(range(5)):
+ do_something()
+
+ # Still not showing progress bars, as `disable=False` is overwritten to `True`.
+ for _ in tqdm(range(5), disable=False):
+ do_something()
+
+ are_progress_bars_disabled() # True
+
+ # Re-enable progress bars globally
+ enable_progress_bars()
+
+ # Progress bar will be shown !
+ for _ in tqdm(range(5)):
+ do_something()
+ ```
+"""
+
+import warnings
+
+from tqdm.auto import tqdm as old_tqdm
+
+from ..config import HF_DATASETS_DISABLE_PROGRESS_BARS
+
+
+# `HF_DATASETS_DISABLE_PROGRESS_BARS` is `Optional[bool]` while `_hf_datasets_progress_bars_disabled`
+# is a `bool`. If `HF_DATASETS_DISABLE_PROGRESS_BARS` is set to True or False, it has priority.
+# If `HF_DATASETS_DISABLE_PROGRESS_BARS` is None, it means the user have not set the
+# environment variable and is free to enable/disable progress bars programmatically.
+# TL;DR: env variable has priority over code.
+#
+# By default, progress bars are enabled.
+_hf_datasets_progress_bars_disabled: bool = HF_DATASETS_DISABLE_PROGRESS_BARS or False
+
+
+def disable_progress_bars() -> None:
+ """
+ Disable globally progress bars used in `datasets` except if `HF_DATASETS_DISABLE_PROGRESS_BAR` environment
+ variable has been set.
+
+ Use [`~utils.enable_progress_bars`] to re-enable them.
+ """
+ if HF_DATASETS_DISABLE_PROGRESS_BARS is False:
+ warnings.warn(
+ "Cannot disable progress bars: environment variable `HF_DATASETS_DISABLE_PROGRESS_BAR=0` is set and has"
+ " priority."
+ )
+ return
+ global _hf_datasets_progress_bars_disabled
+ _hf_datasets_progress_bars_disabled = True
+
+
+def enable_progress_bars() -> None:
+ """
+ Enable globally progress bars used in `datasets` except if `HF_DATASETS_DISABLE_PROGRESS_BAR` environment
+ variable has been set.
+
+ Use [`~utils.disable_progress_bars`] to disable them.
+ """
+ if HF_DATASETS_DISABLE_PROGRESS_BARS is True:
+ warnings.warn(
+ "Cannot enable progress bars: environment variable `HF_DATASETS_DISABLE_PROGRESS_BAR=1` is set and has"
+ " priority."
+ )
+ return
+ global _hf_datasets_progress_bars_disabled
+ _hf_datasets_progress_bars_disabled = False
+
+
+def are_progress_bars_disabled() -> bool:
+ """Return whether progress bars are globally disabled or not.
+
+ Progress bars used in `datasets` can be enable or disabled globally using [`~utils.enable_progress_bars`]
+ and [`~utils.disable_progress_bars`] or by setting `HF_DATASETS_DISABLE_PROGRESS_BAR` as environment variable.
+ """
+ global _hf_datasets_progress_bars_disabled
+ return _hf_datasets_progress_bars_disabled
+
+
+class tqdm(old_tqdm):
+ """
+ Class to override `disable` argument in case progress bars are globally disabled.
+
+ Taken from https://github.com/tqdm/tqdm/issues/619#issuecomment-619639324.
+ """
+
+ def __init__(self, *args, **kwargs):
+ if are_progress_bars_disabled():
+ kwargs["disable"] = True
+ super().__init__(*args, **kwargs)
+
+ def __delattr__(self, attr: str) -> None:
+ """Fix for https://github.com/huggingface/datasets/issues/6066"""
+ try:
+ super().__delattr__(attr)
+ except AttributeError:
+ if attr != "_lock":
+ raise
+
+
+# backward compatibility
+enable_progress_bar = enable_progress_bars
+disable_progress_bar = disable_progress_bars
+
+
+def is_progress_bar_enabled():
+ return not are_progress_bars_disabled()
diff --git a/venv/lib/python3.10/site-packages/datasets/utils/track.py b/venv/lib/python3.10/site-packages/datasets/utils/track.py
new file mode 100644
index 0000000000000000000000000000000000000000..ffd3434af790de63fdd046ad8f93c3067c6e6da8
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/utils/track.py
@@ -0,0 +1,61 @@
+from collections.abc import Iterable, Iterator
+
+
+class tracked_str(str):
+ origins = {}
+
+ def set_origin(self, origin: str):
+ if super().__repr__() not in self.origins:
+ self.origins[super().__repr__()] = origin
+
+ def get_origin(self):
+ return self.origins.get(super().__repr__(), str(self))
+
+ def __repr__(self) -> str:
+ if super().__repr__() not in self.origins or self.origins[super().__repr__()] == self:
+ return super().__repr__()
+ else:
+ return f"{str(self)} (origin={self.origins[super().__repr__()]})"
+
+
+class tracked_list(list):
+ def __init__(self, *args, **kwargs) -> None:
+ super().__init__(*args, **kwargs)
+ self.last_item = None
+
+ def __iter__(self) -> Iterator:
+ for x in super().__iter__():
+ self.last_item = x
+ yield x
+ self.last_item = None
+
+ def __repr__(self) -> str:
+ if self.last_item is None:
+ return super().__repr__()
+ else:
+ return f"{self.__class__.__name__}(current={self.last_item})"
+
+
+class TrackedIterableFromGenerator(Iterable):
+ """Utility class to create an iterable from a generator function, in order to reset the generator when needed."""
+
+ def __init__(self, generator, *args):
+ super().__init__()
+ self.generator = generator
+ self.args = args
+ self.last_item = None
+
+ def __iter__(self):
+ for x in self.generator(*self.args):
+ self.last_item = x
+ yield x
+ self.last_item = None
+
+ def __repr__(self) -> str:
+ if self.last_item is None:
+ return super().__repr__()
+ else:
+ return f"{self.__class__.__name__}(current={self.last_item})"
+
+ def __reduce__(self):
+ return (self.__class__, (self.generator, *self.args))
diff --git a/venv/lib/python3.10/site-packages/datasets/utils/typing.py b/venv/lib/python3.10/site-packages/datasets/utils/typing.py
new file mode 100644
index 0000000000000000000000000000000000000000..4736d74cbb575bd0e12f625c520202a8c74b3ddc
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/utils/typing.py
@@ -0,0 +1,9 @@
+import os
+from typing import TypeVar, Union
+
+
+T = TypeVar("T")
+
+ListLike = Union[list[T], tuple[T, ...]]
+NestedDataStructureLike = Union[T, list[T], dict[str, T]]
+PathLike = Union[str, bytes, os.PathLike]
diff --git a/venv/lib/python3.10/site-packages/datasets/utils/version.py b/venv/lib/python3.10/site-packages/datasets/utils/version.py
new file mode 100644
index 0000000000000000000000000000000000000000..75cf4c39d5f9b916ade8f8d88a0f0ca9e5769217
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/datasets/utils/version.py
@@ -0,0 +1,106 @@
+# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""Version utils."""
+
+import dataclasses
+import re
+from dataclasses import dataclass
+from functools import total_ordering
+from typing import Optional, Union
+
+
+_VERSION_REG = re.compile(r"^(?P\d+)" r"\.(?P\d+)" r"\.(?P\d+)$")
+
+
+@total_ordering
+@dataclass
+class Version:
+ """Dataset version `MAJOR.MINOR.PATCH`.
+
+ Args:
+ version_str (`str`):
+ The dataset version.
+ description (`str`):
+ A description of what is new in this version.
+ major (`str`):
+ minor (`str`):
+ patch (`str`):
+
+ Example:
+
+ ```py
+ >>> VERSION = datasets.Version("1.0.0")
+ ```
+ """
+
+ version_str: str
+ description: Optional[str] = None
+ major: Optional[Union[str, int]] = None
+ minor: Optional[Union[str, int]] = None
+ patch: Optional[Union[str, int]] = None
+
+ def __post_init__(self):
+ self.major, self.minor, self.patch = _str_to_version_tuple(self.version_str)
+
+ def __repr__(self):
+ return f"{self.tuple[0]}.{self.tuple[1]}.{self.tuple[2]}"
+
+ @property
+ def tuple(self):
+ return self.major, self.minor, self.patch
+
+ def _validate_operand(self, other):
+ if isinstance(other, str):
+ return Version(other)
+ elif isinstance(other, Version):
+ return other
+ raise TypeError(f"{other} (type {type(other)}) cannot be compared to version.")
+
+ def __eq__(self, other):
+ try:
+ other = self._validate_operand(other)
+ except (TypeError, ValueError):
+ return False
+ else:
+ return self.tuple == other.tuple
+
+ def __lt__(self, other):
+ other = self._validate_operand(other)
+ return self.tuple < other.tuple
+
+ def __hash__(self):
+ return hash(_version_tuple_to_str(self.tuple))
+
+ @classmethod
+ def from_dict(cls, dic):
+ field_names = {f.name for f in dataclasses.fields(cls)}
+ return cls(**{k: v for k, v in dic.items() if k in field_names})
+
+ def _to_yaml_string(self) -> str:
+ return self.version_str
+
+
+def _str_to_version_tuple(version_str):
+ """Return the tuple (major, minor, patch) version extracted from the str."""
+ res = _VERSION_REG.match(version_str)
+ if not res:
+ raise ValueError(f"Invalid version '{version_str}'. Format should be x.y.z with {{x,y,z}} being digits.")
+ return tuple(int(v) for v in [res.group("major"), res.group("minor"), res.group("patch")])
+
+
+def _version_tuple_to_str(version_tuple):
+ """Return the str version from the version tuple (major, minor, patch)."""
+ return ".".join(str(v) for v in version_tuple)
diff --git a/venv/lib/python3.10/site-packages/exceptiongroup-1.3.0.dist-info/INSTALLER b/venv/lib/python3.10/site-packages/exceptiongroup-1.3.0.dist-info/INSTALLER
new file mode 100644
index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/exceptiongroup-1.3.0.dist-info/INSTALLER
@@ -0,0 +1 @@
+pip
diff --git a/venv/lib/python3.10/site-packages/exceptiongroup-1.3.0.dist-info/METADATA b/venv/lib/python3.10/site-packages/exceptiongroup-1.3.0.dist-info/METADATA
new file mode 100644
index 0000000000000000000000000000000000000000..45df9c805ad4dedd7d04a22e91843820761b9a00
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/exceptiongroup-1.3.0.dist-info/METADATA
@@ -0,0 +1,159 @@
+Metadata-Version: 2.4
+Name: exceptiongroup
+Version: 1.3.0
+Summary: Backport of PEP 654 (exception groups)
+Author-email: Alex Grönholm
+Requires-Python: >=3.7
+Description-Content-Type: text/x-rst
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Typing :: Typed
+License-File: LICENSE
+Requires-Dist: typing-extensions >= 4.6.0; python_version < '3.13'
+Requires-Dist: pytest >= 6 ; extra == "test"
+Project-URL: Changelog, https://github.com/agronholm/exceptiongroup/blob/main/CHANGES.rst
+Project-URL: Issue Tracker, https://github.com/agronholm/exceptiongroup/issues
+Project-URL: Source code, https://github.com/agronholm/exceptiongroup
+Provides-Extra: test
+
+.. image:: https://github.com/agronholm/exceptiongroup/actions/workflows/test.yml/badge.svg
+ :target: https://github.com/agronholm/exceptiongroup/actions/workflows/test.yml
+ :alt: Build Status
+.. image:: https://coveralls.io/repos/github/agronholm/exceptiongroup/badge.svg?branch=main
+ :target: https://coveralls.io/github/agronholm/exceptiongroup?branch=main
+ :alt: Code Coverage
+
+This is a backport of the ``BaseExceptionGroup`` and ``ExceptionGroup`` classes from
+Python 3.11.
+
+It contains the following:
+
+* The ``exceptiongroup.BaseExceptionGroup`` and ``exceptiongroup.ExceptionGroup``
+ classes
+* A utility function (``exceptiongroup.catch()``) for catching exceptions possibly
+ nested in an exception group
+* Patches to the ``TracebackException`` class that properly formats exception groups
+ (installed on import)
+* An exception hook that handles formatting of exception groups through
+ ``TracebackException`` (installed on import)
+* Special versions of some of the functions from the ``traceback`` module, modified to
+ correctly handle exception groups even when monkey patching is disabled, or blocked by
+ another custom exception hook:
+
+ * ``traceback.format_exception()``
+ * ``traceback.format_exception_only()``
+ * ``traceback.print_exception()``
+ * ``traceback.print_exc()``
+* A backported version of ``contextlib.suppress()`` from Python 3.12.1 which also
+ handles suppressing exceptions inside exception groups
+
+If this package is imported on Python 3.11 or later, the built-in implementations of the
+exception group classes are used instead, ``TracebackException`` is not monkey patched
+and the exception hook won't be installed.
+
+See the `standard library documentation`_ for more information on exception groups.
+
+.. _standard library documentation: https://docs.python.org/3/library/exceptions.html
+
+Catching exceptions
+===================
+
+Due to the lack of the ``except*`` syntax introduced by `PEP 654`_ in earlier Python
+versions, you need to use ``exceptiongroup.catch()`` to catch exceptions that are
+potentially nested inside an exception group. This function returns a context manager
+that calls the given handler for any exceptions matching the sole argument.
+
+The argument to ``catch()`` must be a dict (or any ``Mapping``) where each key is either
+an exception class or an iterable of exception classes. Each value must be a callable
+that takes a single positional argument. The handler will be called at most once, with
+an exception group as an argument which will contain all the exceptions that are any
+of the given types, or their subclasses. The exception group may contain nested groups
+containing more matching exceptions.
+
+Thus, the following Python 3.11+ code:
+
+.. code-block:: python
+
+ try:
+ ...
+ except* (ValueError, KeyError) as excgroup:
+ for exc in excgroup.exceptions:
+ print('Caught exception:', type(exc))
+ except* RuntimeError:
+ print('Caught runtime error')
+
+would be written with this backport like this:
+
+.. code-block:: python
+
+ from exceptiongroup import BaseExceptionGroup, catch
+
+ def value_key_err_handler(excgroup: BaseExceptionGroup) -> None:
+ for exc in excgroup.exceptions:
+ print('Caught exception:', type(exc))
+
+ def runtime_err_handler(exc: BaseExceptionGroup) -> None:
+ print('Caught runtime error')
+
+ with catch({
+ (ValueError, KeyError): value_key_err_handler,
+ RuntimeError: runtime_err_handler
+ }):
+ ...
+
+**NOTE**: Just like with ``except*``, you cannot handle ``BaseExceptionGroup`` or
+``ExceptionGroup`` with ``catch()``.
+
+Suppressing exceptions
+======================
+
+This library contains a backport of the ``contextlib.suppress()`` context manager from
+Python 3.12.1. It allows you to selectively ignore certain exceptions, even when they're
+inside exception groups:
+
+.. code-block:: python
+
+ from exceptiongroup import suppress
+
+ with suppress(RuntimeError):
+ raise ExceptionGroup("", [RuntimeError("boo")])
+
+Notes on monkey patching
+========================
+
+To make exception groups render properly when an unhandled exception group is being
+printed out, this package does two things when it is imported on any Python version
+earlier than 3.11:
+
+#. The ``traceback.TracebackException`` class is monkey patched to store extra
+ information about exception groups (in ``__init__()``) and properly format them (in
+ ``format()``)
+#. An exception hook is installed at ``sys.excepthook``, provided that no other hook is
+ already present. This hook causes the exception to be formatted using
+ ``traceback.TracebackException`` rather than the built-in rendered.
+
+If ``sys.exceptionhook`` is found to be set to something else than the default when
+``exceptiongroup`` is imported, no monkeypatching is done at all.
+
+To prevent the exception hook and patches from being installed, set the environment
+variable ``EXCEPTIONGROUP_NO_PATCH`` to ``1``.
+
+Formatting exception groups
+---------------------------
+
+Normally, the monkey patching applied by this library on import will cause exception
+groups to be printed properly in tracebacks. But in cases when the monkey patching is
+blocked by a third party exception hook, or monkey patching is explicitly disabled,
+you can still manually format exceptions using the special versions of the ``traceback``
+functions, like ``format_exception()``, listed at the top of this page. They work just
+like their counterparts in the ``traceback`` module, except that they use a separately
+patched subclass of ``TracebackException`` to perform the rendering.
+
+Particularly in cases where a library installs its own exception hook, it is recommended
+to use these special versions to do the actual formatting of exceptions/tracebacks.
+
+.. _PEP 654: https://www.python.org/dev/peps/pep-0654/
+
diff --git a/venv/lib/python3.10/site-packages/exceptiongroup-1.3.0.dist-info/RECORD b/venv/lib/python3.10/site-packages/exceptiongroup-1.3.0.dist-info/RECORD
new file mode 100644
index 0000000000000000000000000000000000000000..6a90827645d72cf7136140a3101ed4670db15a9c
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/exceptiongroup-1.3.0.dist-info/RECORD
@@ -0,0 +1,19 @@
+exceptiongroup-1.3.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
+exceptiongroup-1.3.0.dist-info/METADATA,sha256=oiAq-llXqOaV0B3dJN316SjtCpthQvuQjmVuVObbaeQ,6725
+exceptiongroup-1.3.0.dist-info/RECORD,,
+exceptiongroup-1.3.0.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+exceptiongroup-1.3.0.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
+exceptiongroup-1.3.0.dist-info/licenses/LICENSE,sha256=blBw12UDHgrUA6HL-Qrm0ZoCKPgC4yC3rP9GCqcu1Hw,3704
+exceptiongroup/__init__.py,sha256=7DHS0hDk-RIs3IQc3SbZVB0-1MhiSCJ9XgvEyEloL7M,1049
+exceptiongroup/__pycache__/__init__.cpython-310.pyc,,
+exceptiongroup/__pycache__/_catch.cpython-310.pyc,,
+exceptiongroup/__pycache__/_exceptions.cpython-310.pyc,,
+exceptiongroup/__pycache__/_formatting.cpython-310.pyc,,
+exceptiongroup/__pycache__/_suppress.cpython-310.pyc,,
+exceptiongroup/__pycache__/_version.cpython-310.pyc,,
+exceptiongroup/_catch.py,sha256=CaJez3E-Jkr-7B7RT3fzusdLWnuyeekooSFn7KyWt9s,4680
+exceptiongroup/_exceptions.py,sha256=wPwPsZ64SXEptuwb4XrTIa1Mc78uqF5vmCrXTdllLn4,11463
+exceptiongroup/_formatting.py,sha256=4UC6bUN3K-FxEX96_ozzgeX05fCyprl6qCVR-efBmQ0,21032
+exceptiongroup/_suppress.py,sha256=LX11PRNpchwfNWwEMY92nYN1F_5qFenQcS8EjIONXKE,1772
+exceptiongroup/_version.py,sha256=qDtcPZdKzxLpd8vVl6fpIFIMkWt2HK_cO9gLDwaHEdk,511
+exceptiongroup/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
diff --git a/venv/lib/python3.10/site-packages/exceptiongroup-1.3.0.dist-info/REQUESTED b/venv/lib/python3.10/site-packages/exceptiongroup-1.3.0.dist-info/REQUESTED
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/venv/lib/python3.10/site-packages/exceptiongroup-1.3.0.dist-info/WHEEL b/venv/lib/python3.10/site-packages/exceptiongroup-1.3.0.dist-info/WHEEL
new file mode 100644
index 0000000000000000000000000000000000000000..d8b9936dad9ab2513fa6979f411560d3b6b57e37
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/exceptiongroup-1.3.0.dist-info/WHEEL
@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: flit 3.12.0
+Root-Is-Purelib: true
+Tag: py3-none-any
diff --git a/venv/lib/python3.10/site-packages/exceptiongroup-1.3.0.dist-info/licenses/LICENSE b/venv/lib/python3.10/site-packages/exceptiongroup-1.3.0.dist-info/licenses/LICENSE
new file mode 100644
index 0000000000000000000000000000000000000000..50d4fa5e68439ce837f6eef437b299c0dd7c8594
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/exceptiongroup-1.3.0.dist-info/licenses/LICENSE
@@ -0,0 +1,73 @@
+The MIT License (MIT)
+
+Copyright (c) 2022 Alex Grönholm
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+
+This project contains code copied from the Python standard library.
+The following is the required license notice for those parts.
+
+PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
+--------------------------------------------
+
+1. This LICENSE AGREEMENT is between the Python Software Foundation
+("PSF"), and the Individual or Organization ("Licensee") accessing and
+otherwise using this software ("Python") in source or binary form and
+its associated documentation.
+
+2. Subject to the terms and conditions of this License Agreement, PSF hereby
+grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce,
+analyze, test, perform and/or display publicly, prepare derivative works,
+distribute, and otherwise use Python alone or in any derivative version,
+provided, however, that PSF's License Agreement and PSF's notice of copyright,
+i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
+2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022 Python Software Foundation;
+All Rights Reserved" are retained in Python alone or in any derivative version
+prepared by Licensee.
+
+3. In the event Licensee prepares a derivative work that is based on
+or incorporates Python or any part thereof, and wants to make
+the derivative work available to others as provided herein, then
+Licensee hereby agrees to include in any such work a brief summary of
+the changes made to Python.
+
+4. PSF is making Python available to Licensee on an "AS IS"
+basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
+IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND
+DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
+FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT
+INFRINGE ANY THIRD PARTY RIGHTS.
+
+5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON
+FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS
+A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON,
+OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
+
+6. This License Agreement will automatically terminate upon a material
+breach of its terms and conditions.
+
+7. Nothing in this License Agreement shall be deemed to create any
+relationship of agency, partnership, or joint venture between PSF and
+Licensee. This License Agreement does not grant permission to use PSF
+trademarks or trade name in a trademark sense to endorse or promote
+products or services of Licensee, or any third party.
+
+8. By copying, installing or otherwise using Python, Licensee
+agrees to be bound by the terms and conditions of this License
+Agreement.
diff --git a/venv/lib/python3.10/site-packages/fontTools/__pycache__/__init__.cpython-310.pyc b/venv/lib/python3.10/site-packages/fontTools/__pycache__/__init__.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..721cc8fb4072d960c1c4b4c88b448dcdf465060b
Binary files /dev/null and b/venv/lib/python3.10/site-packages/fontTools/__pycache__/__init__.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/fontTools/__pycache__/__main__.cpython-310.pyc b/venv/lib/python3.10/site-packages/fontTools/__pycache__/__main__.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..37239127df0ece81f938faa352a9188f3932bdb7
Binary files /dev/null and b/venv/lib/python3.10/site-packages/fontTools/__pycache__/__main__.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/fontTools/__pycache__/afmLib.cpython-310.pyc b/venv/lib/python3.10/site-packages/fontTools/__pycache__/afmLib.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..6073a3912ff376b5808b2254f2d8752de54e7345
Binary files /dev/null and b/venv/lib/python3.10/site-packages/fontTools/__pycache__/afmLib.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/fontTools/__pycache__/annotations.cpython-310.pyc b/venv/lib/python3.10/site-packages/fontTools/__pycache__/annotations.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..a50ebac3bca15c815049d6b90c363e516959196c
Binary files /dev/null and b/venv/lib/python3.10/site-packages/fontTools/__pycache__/annotations.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/fontTools/__pycache__/fontBuilder.cpython-310.pyc b/venv/lib/python3.10/site-packages/fontTools/__pycache__/fontBuilder.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..7a1e2202e1b97cb1cf1f99e4376674ae1b27fa99
Binary files /dev/null and b/venv/lib/python3.10/site-packages/fontTools/__pycache__/fontBuilder.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/fontTools/__pycache__/help.cpython-310.pyc b/venv/lib/python3.10/site-packages/fontTools/__pycache__/help.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..f3a544aa82cbca26fed5256f7cecbb6acc7c4015
Binary files /dev/null and b/venv/lib/python3.10/site-packages/fontTools/__pycache__/help.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/fontTools/__pycache__/tfmLib.cpython-310.pyc b/venv/lib/python3.10/site-packages/fontTools/__pycache__/tfmLib.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..2f15d349b0165e426f6186251447be668fe3ec30
Binary files /dev/null and b/venv/lib/python3.10/site-packages/fontTools/__pycache__/tfmLib.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/fontTools/__pycache__/ttx.cpython-310.pyc b/venv/lib/python3.10/site-packages/fontTools/__pycache__/ttx.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..4033f4bf35973c54d3bfc46d88e9d8da6fe5c8cd
Binary files /dev/null and b/venv/lib/python3.10/site-packages/fontTools/__pycache__/ttx.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/fontTools/__pycache__/unicode.cpython-310.pyc b/venv/lib/python3.10/site-packages/fontTools/__pycache__/unicode.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..44f1080363fee3ab3a13cb24220490be4ca96663
Binary files /dev/null and b/venv/lib/python3.10/site-packages/fontTools/__pycache__/unicode.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/fontTools/cu2qu/__init__.py b/venv/lib/python3.10/site-packages/fontTools/cu2qu/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..4ae6356e44e1fed074b6283bcb4365bf2b770529
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/fontTools/cu2qu/__init__.py
@@ -0,0 +1,15 @@
+# Copyright 2016 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .cu2qu import *
diff --git a/venv/lib/python3.10/site-packages/fontTools/cu2qu/__main__.py b/venv/lib/python3.10/site-packages/fontTools/cu2qu/__main__.py
new file mode 100644
index 0000000000000000000000000000000000000000..5205ffeef9b4d244bdbaa4c15fb0dc11a12b550e
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/fontTools/cu2qu/__main__.py
@@ -0,0 +1,6 @@
+import sys
+from .cli import _main as main
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/venv/lib/python3.10/site-packages/fontTools/cu2qu/__pycache__/__init__.cpython-310.pyc b/venv/lib/python3.10/site-packages/fontTools/cu2qu/__pycache__/__init__.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..6f3bcf4daa27406a0a37ef3ccd8be0a6f8e9d272
Binary files /dev/null and b/venv/lib/python3.10/site-packages/fontTools/cu2qu/__pycache__/__init__.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/fontTools/cu2qu/__pycache__/__main__.cpython-310.pyc b/venv/lib/python3.10/site-packages/fontTools/cu2qu/__pycache__/__main__.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..fa2ad4e42cf15343511f0ebb859a2e67855edabf
Binary files /dev/null and b/venv/lib/python3.10/site-packages/fontTools/cu2qu/__pycache__/__main__.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/fontTools/cu2qu/__pycache__/benchmark.cpython-310.pyc b/venv/lib/python3.10/site-packages/fontTools/cu2qu/__pycache__/benchmark.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..d9b239af4b6caed3f20b6c91d6dc5238371fbda5
Binary files /dev/null and b/venv/lib/python3.10/site-packages/fontTools/cu2qu/__pycache__/benchmark.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/fontTools/cu2qu/__pycache__/cli.cpython-310.pyc b/venv/lib/python3.10/site-packages/fontTools/cu2qu/__pycache__/cli.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..b68b2740b02a45cedf99c7be4f0f764e91ed1b13
Binary files /dev/null and b/venv/lib/python3.10/site-packages/fontTools/cu2qu/__pycache__/cli.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/fontTools/cu2qu/__pycache__/cu2qu.cpython-310.pyc b/venv/lib/python3.10/site-packages/fontTools/cu2qu/__pycache__/cu2qu.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..f42ff47fc2e83bc66388e6a06ad620f7ff359b32
Binary files /dev/null and b/venv/lib/python3.10/site-packages/fontTools/cu2qu/__pycache__/cu2qu.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/fontTools/cu2qu/__pycache__/errors.cpython-310.pyc b/venv/lib/python3.10/site-packages/fontTools/cu2qu/__pycache__/errors.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..d50f2b3ed51255594311f2032129adb76b954107
Binary files /dev/null and b/venv/lib/python3.10/site-packages/fontTools/cu2qu/__pycache__/errors.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/fontTools/cu2qu/__pycache__/ufo.cpython-310.pyc b/venv/lib/python3.10/site-packages/fontTools/cu2qu/__pycache__/ufo.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..2b130aa6a27d2bd2d09a187d517d5acc61be4246
Binary files /dev/null and b/venv/lib/python3.10/site-packages/fontTools/cu2qu/__pycache__/ufo.cpython-310.pyc differ
diff --git a/venv/lib/python3.10/site-packages/fontTools/cu2qu/benchmark.py b/venv/lib/python3.10/site-packages/fontTools/cu2qu/benchmark.py
new file mode 100644
index 0000000000000000000000000000000000000000..007f75d887e312b68a1859546629c6410070770d
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/fontTools/cu2qu/benchmark.py
@@ -0,0 +1,54 @@
+"""Benchmark the cu2qu algorithm performance."""
+
+from .cu2qu import *
+import random
+import timeit
+
+MAX_ERR = 0.05
+
+
+def generate_curve():
+ return [
+ tuple(float(random.randint(0, 2048)) for coord in range(2))
+ for point in range(4)
+ ]
+
+
+def setup_curve_to_quadratic():
+ return generate_curve(), MAX_ERR
+
+
+def setup_curves_to_quadratic():
+ num_curves = 3
+ return ([generate_curve() for curve in range(num_curves)], [MAX_ERR] * num_curves)
+
+
+def run_benchmark(module, function, setup_suffix="", repeat=5, number=1000):
+ setup_func = "setup_" + function
+ if setup_suffix:
+ print("%s with %s:" % (function, setup_suffix), end="")
+ setup_func += "_" + setup_suffix
+ else:
+ print("%s:" % function, end="")
+
+ def wrapper(function, setup_func):
+ function = globals()[function]
+ setup_func = globals()[setup_func]
+
+ def wrapped():
+ return function(*setup_func())
+
+ return wrapped
+
+ results = timeit.repeat(wrapper(function, setup_func), repeat=repeat, number=number)
+ print("\t%5.1fus" % (min(results) * 1000000.0 / number))
+
+
+def main():
+ run_benchmark("cu2qu", "curve_to_quadratic")
+ run_benchmark("cu2qu", "curves_to_quadratic")
+
+
+if __name__ == "__main__":
+ random.seed(1)
+ main()
diff --git a/venv/lib/python3.10/site-packages/fontTools/cu2qu/cli.py b/venv/lib/python3.10/site-packages/fontTools/cu2qu/cli.py
new file mode 100644
index 0000000000000000000000000000000000000000..ddc6450200820766a764c9c9b18b4b3c3c04d0e4
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/fontTools/cu2qu/cli.py
@@ -0,0 +1,198 @@
+import os
+import argparse
+import logging
+import shutil
+import multiprocessing as mp
+from contextlib import closing
+from functools import partial
+
+import fontTools
+from .ufo import font_to_quadratic, fonts_to_quadratic
+
+ufo_module = None
+try:
+ import ufoLib2 as ufo_module
+except ImportError:
+ try:
+ import defcon as ufo_module
+ except ImportError as e:
+ pass
+
+
+logger = logging.getLogger("fontTools.cu2qu")
+
+
+def _cpu_count():
+ try:
+ return mp.cpu_count()
+ except NotImplementedError: # pragma: no cover
+ return 1
+
+
+def open_ufo(path):
+ if hasattr(ufo_module.Font, "open"): # ufoLib2
+ return ufo_module.Font.open(path)
+ return ufo_module.Font(path) # defcon
+
+
+def _font_to_quadratic(input_path, output_path=None, **kwargs):
+ ufo = open_ufo(input_path)
+ logger.info("Converting curves for %s", input_path)
+ if font_to_quadratic(ufo, **kwargs):
+ logger.info("Saving %s", output_path)
+ if output_path:
+ ufo.save(output_path)
+ else:
+ ufo.save() # save in-place
+ elif output_path:
+ _copytree(input_path, output_path)
+
+
+def _samepath(path1, path2):
+ # TODO on python3+, there's os.path.samefile
+ path1 = os.path.normcase(os.path.abspath(os.path.realpath(path1)))
+ path2 = os.path.normcase(os.path.abspath(os.path.realpath(path2)))
+ return path1 == path2
+
+
+def _copytree(input_path, output_path):
+ if _samepath(input_path, output_path):
+ logger.debug("input and output paths are the same file; skipped copy")
+ return
+ if os.path.exists(output_path):
+ shutil.rmtree(output_path)
+ shutil.copytree(input_path, output_path)
+
+
+def _main(args=None):
+ """Convert a UFO font from cubic to quadratic curves"""
+ parser = argparse.ArgumentParser(prog="cu2qu")
+ parser.add_argument("--version", action="version", version=fontTools.__version__)
+ parser.add_argument(
+ "infiles",
+ nargs="+",
+ metavar="INPUT",
+ help="one or more input UFO source file(s).",
+ )
+ parser.add_argument("-v", "--verbose", action="count", default=0)
+ parser.add_argument(
+ "-e",
+ "--conversion-error",
+ type=float,
+ metavar="ERROR",
+ default=None,
+ help="maxiumum approximation error measured in EM (default: 0.001)",
+ )
+ parser.add_argument(
+ "-m",
+ "--mixed",
+ default=False,
+ action="store_true",
+ help="whether to used mixed quadratic and cubic curves",
+ )
+ parser.add_argument(
+ "--keep-direction",
+ dest="reverse_direction",
+ action="store_false",
+ help="do not reverse the contour direction",
+ )
+
+ mode_parser = parser.add_mutually_exclusive_group()
+ mode_parser.add_argument(
+ "-i",
+ "--interpolatable",
+ action="store_true",
+ help="whether curve conversion should keep interpolation compatibility",
+ )
+ mode_parser.add_argument(
+ "-j",
+ "--jobs",
+ type=int,
+ nargs="?",
+ default=1,
+ const=_cpu_count(),
+ metavar="N",
+ help="Convert using N multiple processes (default: %(default)s)",
+ )
+
+ output_parser = parser.add_mutually_exclusive_group()
+ output_parser.add_argument(
+ "-o",
+ "--output-file",
+ default=None,
+ metavar="OUTPUT",
+ help=(
+ "output filename for the converted UFO. By default fonts are "
+ "modified in place. This only works with a single input."
+ ),
+ )
+ output_parser.add_argument(
+ "-d",
+ "--output-dir",
+ default=None,
+ metavar="DIRECTORY",
+ help="output directory where to save converted UFOs",
+ )
+
+ options = parser.parse_args(args)
+
+ if ufo_module is None:
+ parser.error("Either ufoLib2 or defcon are required to run this script.")
+
+ if not options.verbose:
+ level = "WARNING"
+ elif options.verbose == 1:
+ level = "INFO"
+ else:
+ level = "DEBUG"
+ logging.basicConfig(level=level)
+
+ if len(options.infiles) > 1 and options.output_file:
+ parser.error("-o/--output-file can't be used with multile inputs")
+
+ if options.output_dir:
+ output_dir = options.output_dir
+ if not os.path.exists(output_dir):
+ os.mkdir(output_dir)
+ elif not os.path.isdir(output_dir):
+ parser.error("'%s' is not a directory" % output_dir)
+ output_paths = [
+ os.path.join(output_dir, os.path.basename(p)) for p in options.infiles
+ ]
+ elif options.output_file:
+ output_paths = [options.output_file]
+ else:
+ # save in-place
+ output_paths = [None] * len(options.infiles)
+
+ kwargs = dict(
+ dump_stats=options.verbose > 0,
+ max_err_em=options.conversion_error,
+ reverse_direction=options.reverse_direction,
+ all_quadratic=False if options.mixed else True,
+ )
+
+ if options.interpolatable:
+ logger.info("Converting curves compatibly")
+ ufos = [open_ufo(infile) for infile in options.infiles]
+ if fonts_to_quadratic(ufos, **kwargs):
+ for ufo, output_path in zip(ufos, output_paths):
+ logger.info("Saving %s", output_path)
+ if output_path:
+ ufo.save(output_path)
+ else:
+ ufo.save()
+ else:
+ for input_path, output_path in zip(options.infiles, output_paths):
+ if output_path:
+ _copytree(input_path, output_path)
+ else:
+ jobs = min(len(options.infiles), options.jobs) if options.jobs > 1 else 1
+ if jobs > 1:
+ func = partial(_font_to_quadratic, **kwargs)
+ logger.info("Running %d parallel processes", jobs)
+ with closing(mp.Pool(jobs)) as pool:
+ pool.starmap(func, zip(options.infiles, output_paths))
+ else:
+ for input_path, output_path in zip(options.infiles, output_paths):
+ _font_to_quadratic(input_path, output_path, **kwargs)
diff --git a/venv/lib/python3.10/site-packages/fontTools/cu2qu/cu2qu.c b/venv/lib/python3.10/site-packages/fontTools/cu2qu/cu2qu.c
new file mode 100644
index 0000000000000000000000000000000000000000..0b26ad1bbe13b8f4ddd14f0937f4f22d57b1c8a5
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/fontTools/cu2qu/cu2qu.c
@@ -0,0 +1,15773 @@
+/* Generated by Cython 3.1.4 */
+
+/* BEGIN: Cython Metadata
+{
+ "distutils": {
+ "define_macros": [
+ [
+ "CYTHON_TRACE_NOGIL",
+ "1"
+ ]
+ ],
+ "name": "fontTools.cu2qu.cu2qu",
+ "sources": [
+ "Lib/fontTools/cu2qu/cu2qu.py"
+ ]
+ },
+ "module_name": "fontTools.cu2qu.cu2qu"
+}
+END: Cython Metadata */
+
+#ifndef PY_SSIZE_T_CLEAN
+#define PY_SSIZE_T_CLEAN
+#endif /* PY_SSIZE_T_CLEAN */
+/* InitLimitedAPI */
+#if defined(Py_LIMITED_API) && !defined(CYTHON_LIMITED_API)
+ #define CYTHON_LIMITED_API 1
+#endif
+
+#include "Python.h"
+#ifndef Py_PYTHON_H
+ #error Python headers needed to compile C extensions, please install development version of Python.
+#elif PY_VERSION_HEX < 0x03080000
+ #error Cython requires Python 3.8+.
+#else
+#define __PYX_ABI_VERSION "3_1_4"
+#define CYTHON_HEX_VERSION 0x030104F0
+#define CYTHON_FUTURE_DIVISION 1
+/* CModulePreamble */
+#include
+#ifndef offsetof
+ #define offsetof(type, member) ( (size_t) & ((type*)0) -> member )
+#endif
+#if !defined(_WIN32) && !defined(WIN32) && !defined(MS_WINDOWS)
+ #ifndef __stdcall
+ #define __stdcall
+ #endif
+ #ifndef __cdecl
+ #define __cdecl
+ #endif
+ #ifndef __fastcall
+ #define __fastcall
+ #endif
+#endif
+#ifndef DL_IMPORT
+ #define DL_IMPORT(t) t
+#endif
+#ifndef DL_EXPORT
+ #define DL_EXPORT(t) t
+#endif
+#define __PYX_COMMA ,
+#ifndef HAVE_LONG_LONG
+ #define HAVE_LONG_LONG
+#endif
+#ifndef PY_LONG_LONG
+ #define PY_LONG_LONG LONG_LONG
+#endif
+#ifndef Py_HUGE_VAL
+ #define Py_HUGE_VAL HUGE_VAL
+#endif
+#define __PYX_LIMITED_VERSION_HEX PY_VERSION_HEX
+#if defined(GRAALVM_PYTHON)
+ /* For very preliminary testing purposes. Most variables are set the same as PyPy.
+ The existence of this section does not imply that anything works or is even tested */
+ #define CYTHON_COMPILING_IN_PYPY 0
+ #define CYTHON_COMPILING_IN_CPYTHON 0
+ #define CYTHON_COMPILING_IN_LIMITED_API 0
+ #define CYTHON_COMPILING_IN_GRAAL 1
+ #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0
+ #undef CYTHON_USE_TYPE_SLOTS
+ #define CYTHON_USE_TYPE_SLOTS 0
+ #undef CYTHON_USE_TYPE_SPECS
+ #define CYTHON_USE_TYPE_SPECS 0
+ #undef CYTHON_USE_PYTYPE_LOOKUP
+ #define CYTHON_USE_PYTYPE_LOOKUP 0
+ #undef CYTHON_USE_PYLIST_INTERNALS
+ #define CYTHON_USE_PYLIST_INTERNALS 0
+ #undef CYTHON_USE_UNICODE_INTERNALS
+ #define CYTHON_USE_UNICODE_INTERNALS 0
+ #undef CYTHON_USE_UNICODE_WRITER
+ #define CYTHON_USE_UNICODE_WRITER 0
+ #undef CYTHON_USE_PYLONG_INTERNALS
+ #define CYTHON_USE_PYLONG_INTERNALS 0
+ #undef CYTHON_AVOID_BORROWED_REFS
+ #define CYTHON_AVOID_BORROWED_REFS 1
+ #undef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS
+ #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 1
+ #undef CYTHON_ASSUME_SAFE_MACROS
+ #define CYTHON_ASSUME_SAFE_MACROS 0
+ #undef CYTHON_ASSUME_SAFE_SIZE
+ #define CYTHON_ASSUME_SAFE_SIZE 0
+ #undef CYTHON_UNPACK_METHODS
+ #define CYTHON_UNPACK_METHODS 0
+ #undef CYTHON_FAST_THREAD_STATE
+ #define CYTHON_FAST_THREAD_STATE 0
+ #undef CYTHON_FAST_GIL
+ #define CYTHON_FAST_GIL 0
+ #undef CYTHON_METH_FASTCALL
+ #define CYTHON_METH_FASTCALL 0
+ #undef CYTHON_FAST_PYCALL
+ #define CYTHON_FAST_PYCALL 0
+ #ifndef CYTHON_PEP487_INIT_SUBCLASS
+ #define CYTHON_PEP487_INIT_SUBCLASS 1
+ #endif
+ #undef CYTHON_PEP489_MULTI_PHASE_INIT
+ #define CYTHON_PEP489_MULTI_PHASE_INIT 1
+ #undef CYTHON_USE_MODULE_STATE
+ #define CYTHON_USE_MODULE_STATE 0
+ #undef CYTHON_USE_SYS_MONITORING
+ #define CYTHON_USE_SYS_MONITORING 0
+ #undef CYTHON_USE_TP_FINALIZE
+ #define CYTHON_USE_TP_FINALIZE 0
+ #undef CYTHON_USE_AM_SEND
+ #define CYTHON_USE_AM_SEND 0
+ #undef CYTHON_USE_DICT_VERSIONS
+ #define CYTHON_USE_DICT_VERSIONS 0
+ #undef CYTHON_USE_EXC_INFO_STACK
+ #define CYTHON_USE_EXC_INFO_STACK 1
+ #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC
+ #define CYTHON_UPDATE_DESCRIPTOR_DOC 0
+ #endif
+ #undef CYTHON_USE_FREELISTS
+ #define CYTHON_USE_FREELISTS 0
+#elif defined(PYPY_VERSION)
+ #define CYTHON_COMPILING_IN_PYPY 1
+ #define CYTHON_COMPILING_IN_CPYTHON 0
+ #define CYTHON_COMPILING_IN_LIMITED_API 0
+ #define CYTHON_COMPILING_IN_GRAAL 0
+ #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0
+ #undef CYTHON_USE_TYPE_SLOTS
+ #define CYTHON_USE_TYPE_SLOTS 1
+ #ifndef CYTHON_USE_TYPE_SPECS
+ #define CYTHON_USE_TYPE_SPECS 0
+ #endif
+ #undef CYTHON_USE_PYTYPE_LOOKUP
+ #define CYTHON_USE_PYTYPE_LOOKUP 0
+ #undef CYTHON_USE_PYLIST_INTERNALS
+ #define CYTHON_USE_PYLIST_INTERNALS 0
+ #undef CYTHON_USE_UNICODE_INTERNALS
+ #define CYTHON_USE_UNICODE_INTERNALS 0
+ #undef CYTHON_USE_UNICODE_WRITER
+ #define CYTHON_USE_UNICODE_WRITER 0
+ #undef CYTHON_USE_PYLONG_INTERNALS
+ #define CYTHON_USE_PYLONG_INTERNALS 0
+ #undef CYTHON_AVOID_BORROWED_REFS
+ #define CYTHON_AVOID_BORROWED_REFS 1
+ #undef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS
+ #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 1
+ #undef CYTHON_ASSUME_SAFE_MACROS
+ #define CYTHON_ASSUME_SAFE_MACROS 0
+ #ifndef CYTHON_ASSUME_SAFE_SIZE
+ #define CYTHON_ASSUME_SAFE_SIZE 1
+ #endif
+ #undef CYTHON_UNPACK_METHODS
+ #define CYTHON_UNPACK_METHODS 0
+ #undef CYTHON_FAST_THREAD_STATE
+ #define CYTHON_FAST_THREAD_STATE 0
+ #undef CYTHON_FAST_GIL
+ #define CYTHON_FAST_GIL 0
+ #undef CYTHON_METH_FASTCALL
+ #define CYTHON_METH_FASTCALL 0
+ #undef CYTHON_FAST_PYCALL
+ #define CYTHON_FAST_PYCALL 0
+ #ifndef CYTHON_PEP487_INIT_SUBCLASS
+ #define CYTHON_PEP487_INIT_SUBCLASS 1
+ #endif
+ #if PY_VERSION_HEX < 0x03090000
+ #undef CYTHON_PEP489_MULTI_PHASE_INIT
+ #define CYTHON_PEP489_MULTI_PHASE_INIT 0
+ #elif !defined(CYTHON_PEP489_MULTI_PHASE_INIT)
+ #define CYTHON_PEP489_MULTI_PHASE_INIT 1
+ #endif
+ #undef CYTHON_USE_MODULE_STATE
+ #define CYTHON_USE_MODULE_STATE 0
+ #undef CYTHON_USE_SYS_MONITORING
+ #define CYTHON_USE_SYS_MONITORING 0
+ #ifndef CYTHON_USE_TP_FINALIZE
+ #define CYTHON_USE_TP_FINALIZE (PYPY_VERSION_NUM >= 0x07030C00)
+ #endif
+ #undef CYTHON_USE_AM_SEND
+ #define CYTHON_USE_AM_SEND 0
+ #undef CYTHON_USE_DICT_VERSIONS
+ #define CYTHON_USE_DICT_VERSIONS 0
+ #undef CYTHON_USE_EXC_INFO_STACK
+ #define CYTHON_USE_EXC_INFO_STACK 0
+ #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC
+ #define CYTHON_UPDATE_DESCRIPTOR_DOC (PYPY_VERSION_NUM >= 0x07031100)
+ #endif
+ #undef CYTHON_USE_FREELISTS
+ #define CYTHON_USE_FREELISTS 0
+#elif defined(CYTHON_LIMITED_API)
+ #ifdef Py_LIMITED_API
+ #undef __PYX_LIMITED_VERSION_HEX
+ #define __PYX_LIMITED_VERSION_HEX Py_LIMITED_API
+ #endif
+ #define CYTHON_COMPILING_IN_PYPY 0
+ #define CYTHON_COMPILING_IN_CPYTHON 0
+ #define CYTHON_COMPILING_IN_LIMITED_API 1
+ #define CYTHON_COMPILING_IN_GRAAL 0
+ #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0
+ #undef CYTHON_CLINE_IN_TRACEBACK
+ #define CYTHON_CLINE_IN_TRACEBACK 0
+ #undef CYTHON_USE_TYPE_SLOTS
+ #define CYTHON_USE_TYPE_SLOTS 0
+ #undef CYTHON_USE_TYPE_SPECS
+ #define CYTHON_USE_TYPE_SPECS 1
+ #undef CYTHON_USE_PYTYPE_LOOKUP
+ #define CYTHON_USE_PYTYPE_LOOKUP 0
+ #undef CYTHON_USE_PYLIST_INTERNALS
+ #define CYTHON_USE_PYLIST_INTERNALS 0
+ #undef CYTHON_USE_UNICODE_INTERNALS
+ #define CYTHON_USE_UNICODE_INTERNALS 0
+ #ifndef CYTHON_USE_UNICODE_WRITER
+ #define CYTHON_USE_UNICODE_WRITER 0
+ #endif
+ #undef CYTHON_USE_PYLONG_INTERNALS
+ #define CYTHON_USE_PYLONG_INTERNALS 0
+ #ifndef CYTHON_AVOID_BORROWED_REFS
+ #define CYTHON_AVOID_BORROWED_REFS 0
+ #endif
+ #ifndef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS
+ #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 0
+ #endif
+ #undef CYTHON_ASSUME_SAFE_MACROS
+ #define CYTHON_ASSUME_SAFE_MACROS 0
+ #undef CYTHON_ASSUME_SAFE_SIZE
+ #define CYTHON_ASSUME_SAFE_SIZE 0
+ #undef CYTHON_UNPACK_METHODS
+ #define CYTHON_UNPACK_METHODS 0
+ #undef CYTHON_FAST_THREAD_STATE
+ #define CYTHON_FAST_THREAD_STATE 0
+ #undef CYTHON_FAST_GIL
+ #define CYTHON_FAST_GIL 0
+ #undef CYTHON_METH_FASTCALL
+ #define CYTHON_METH_FASTCALL (__PYX_LIMITED_VERSION_HEX >= 0x030C0000)
+ #undef CYTHON_FAST_PYCALL
+ #define CYTHON_FAST_PYCALL 0
+ #ifndef CYTHON_PEP487_INIT_SUBCLASS
+ #define CYTHON_PEP487_INIT_SUBCLASS 1
+ #endif
+ #ifndef CYTHON_PEP489_MULTI_PHASE_INIT
+ #define CYTHON_PEP489_MULTI_PHASE_INIT 1
+ #endif
+ #ifndef CYTHON_USE_MODULE_STATE
+ #define CYTHON_USE_MODULE_STATE 0
+ #endif
+ #undef CYTHON_USE_SYS_MONITORING
+ #define CYTHON_USE_SYS_MONITORING 0
+ #ifndef CYTHON_USE_TP_FINALIZE
+ #define CYTHON_USE_TP_FINALIZE 0
+ #endif
+ #ifndef CYTHON_USE_AM_SEND
+ #define CYTHON_USE_AM_SEND (__PYX_LIMITED_VERSION_HEX >= 0x030A0000)
+ #endif
+ #undef CYTHON_USE_DICT_VERSIONS
+ #define CYTHON_USE_DICT_VERSIONS 0
+ #undef CYTHON_USE_EXC_INFO_STACK
+ #define CYTHON_USE_EXC_INFO_STACK 0
+ #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC
+ #define CYTHON_UPDATE_DESCRIPTOR_DOC 0
+ #endif
+ #undef CYTHON_USE_FREELISTS
+ #define CYTHON_USE_FREELISTS 0
+#else
+ #define CYTHON_COMPILING_IN_PYPY 0
+ #define CYTHON_COMPILING_IN_CPYTHON 1
+ #define CYTHON_COMPILING_IN_LIMITED_API 0
+ #define CYTHON_COMPILING_IN_GRAAL 0
+ #ifdef Py_GIL_DISABLED
+ #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 1
+ #else
+ #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0
+ #endif
+ #if PY_VERSION_HEX < 0x030A0000
+ #undef CYTHON_USE_TYPE_SLOTS
+ #define CYTHON_USE_TYPE_SLOTS 1
+ #elif !defined(CYTHON_USE_TYPE_SLOTS)
+ #define CYTHON_USE_TYPE_SLOTS 1
+ #endif
+ #ifndef CYTHON_USE_TYPE_SPECS
+ #define CYTHON_USE_TYPE_SPECS 0
+ #endif
+ #ifndef CYTHON_USE_PYTYPE_LOOKUP
+ #define CYTHON_USE_PYTYPE_LOOKUP 1
+ #endif
+ #ifndef CYTHON_USE_PYLONG_INTERNALS
+ #define CYTHON_USE_PYLONG_INTERNALS 1
+ #endif
+ #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING
+ #undef CYTHON_USE_PYLIST_INTERNALS
+ #define CYTHON_USE_PYLIST_INTERNALS 0
+ #elif !defined(CYTHON_USE_PYLIST_INTERNALS)
+ #define CYTHON_USE_PYLIST_INTERNALS 1
+ #endif
+ #ifndef CYTHON_USE_UNICODE_INTERNALS
+ #define CYTHON_USE_UNICODE_INTERNALS 1
+ #endif
+ #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING || PY_VERSION_HEX >= 0x030B00A2
+ #undef CYTHON_USE_UNICODE_WRITER
+ #define CYTHON_USE_UNICODE_WRITER 0
+ #elif !defined(CYTHON_USE_UNICODE_WRITER)
+ #define CYTHON_USE_UNICODE_WRITER 1
+ #endif
+ #ifndef CYTHON_AVOID_BORROWED_REFS
+ #define CYTHON_AVOID_BORROWED_REFS 0
+ #endif
+ #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING
+ #undef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS
+ #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 1
+ #elif !defined(CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS)
+ #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 0
+ #endif
+ #ifndef CYTHON_ASSUME_SAFE_MACROS
+ #define CYTHON_ASSUME_SAFE_MACROS 1
+ #endif
+ #ifndef CYTHON_ASSUME_SAFE_SIZE
+ #define CYTHON_ASSUME_SAFE_SIZE 1
+ #endif
+ #ifndef CYTHON_UNPACK_METHODS
+ #define CYTHON_UNPACK_METHODS 1
+ #endif
+ #ifndef CYTHON_FAST_THREAD_STATE
+ #define CYTHON_FAST_THREAD_STATE 1
+ #endif
+ #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING
+ #undef CYTHON_FAST_GIL
+ #define CYTHON_FAST_GIL 0
+ #elif !defined(CYTHON_FAST_GIL)
+ #define CYTHON_FAST_GIL (PY_VERSION_HEX < 0x030C00A6)
+ #endif
+ #ifndef CYTHON_METH_FASTCALL
+ #define CYTHON_METH_FASTCALL 1
+ #endif
+ #ifndef CYTHON_FAST_PYCALL
+ #define CYTHON_FAST_PYCALL 1
+ #endif
+ #ifndef CYTHON_PEP487_INIT_SUBCLASS
+ #define CYTHON_PEP487_INIT_SUBCLASS 1
+ #endif
+ #ifndef CYTHON_PEP489_MULTI_PHASE_INIT
+ #define CYTHON_PEP489_MULTI_PHASE_INIT 1
+ #endif
+ #ifndef CYTHON_USE_MODULE_STATE
+ #define CYTHON_USE_MODULE_STATE 0
+ #endif
+ #ifndef CYTHON_USE_SYS_MONITORING
+ #define CYTHON_USE_SYS_MONITORING (PY_VERSION_HEX >= 0x030d00B1)
+ #endif
+ #ifndef CYTHON_USE_TP_FINALIZE
+ #define CYTHON_USE_TP_FINALIZE 1
+ #endif
+ #ifndef CYTHON_USE_AM_SEND
+ #define CYTHON_USE_AM_SEND 1
+ #endif
+ #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING
+ #undef CYTHON_USE_DICT_VERSIONS
+ #define CYTHON_USE_DICT_VERSIONS 0
+ #elif !defined(CYTHON_USE_DICT_VERSIONS)
+ #define CYTHON_USE_DICT_VERSIONS (PY_VERSION_HEX < 0x030C00A5 && !CYTHON_USE_MODULE_STATE)
+ #endif
+ #ifndef CYTHON_USE_EXC_INFO_STACK
+ #define CYTHON_USE_EXC_INFO_STACK 1
+ #endif
+ #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC
+ #define CYTHON_UPDATE_DESCRIPTOR_DOC 1
+ #endif
+ #ifndef CYTHON_USE_FREELISTS
+ #define CYTHON_USE_FREELISTS (!CYTHON_COMPILING_IN_CPYTHON_FREETHREADING)
+ #endif
+#endif
+#ifndef CYTHON_FAST_PYCCALL
+#define CYTHON_FAST_PYCCALL CYTHON_FAST_PYCALL
+#endif
+#ifndef CYTHON_VECTORCALL
+#if CYTHON_COMPILING_IN_LIMITED_API
+#define CYTHON_VECTORCALL (__PYX_LIMITED_VERSION_HEX >= 0x030C0000)
+#else
+#define CYTHON_VECTORCALL (CYTHON_FAST_PYCCALL && PY_VERSION_HEX >= 0x030800B1)
+#endif
+#endif
+#define CYTHON_BACKPORT_VECTORCALL (CYTHON_METH_FASTCALL && PY_VERSION_HEX < 0x030800B1)
+#if CYTHON_USE_PYLONG_INTERNALS
+ #undef SHIFT
+ #undef BASE
+ #undef MASK
+ #ifdef SIZEOF_VOID_P
+ enum { __pyx_check_sizeof_voidp = 1 / (int)(SIZEOF_VOID_P == sizeof(void*)) };
+ #endif
+#endif
+#ifndef CYTHON_LOCK_AND_GIL_DEADLOCK_AVOIDANCE_TIME
+ #define CYTHON_LOCK_AND_GIL_DEADLOCK_AVOIDANCE_TIME 100
+#endif
+#ifndef __has_attribute
+ #define __has_attribute(x) 0
+#endif
+#ifndef __has_cpp_attribute
+ #define __has_cpp_attribute(x) 0
+#endif
+#ifndef CYTHON_RESTRICT
+ #if defined(__GNUC__)
+ #define CYTHON_RESTRICT __restrict__
+ #elif defined(_MSC_VER) && _MSC_VER >= 1400
+ #define CYTHON_RESTRICT __restrict
+ #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L
+ #define CYTHON_RESTRICT restrict
+ #else
+ #define CYTHON_RESTRICT
+ #endif
+#endif
+#ifndef CYTHON_UNUSED
+ #if defined(__cplusplus)
+ /* for clang __has_cpp_attribute(maybe_unused) is true even before C++17
+ * but leads to warnings with -pedantic, since it is a C++17 feature */
+ #if ((defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) || __cplusplus >= 201703L)
+ #if __has_cpp_attribute(maybe_unused)
+ #define CYTHON_UNUSED [[maybe_unused]]
+ #endif
+ #endif
+ #endif
+#endif
+#ifndef CYTHON_UNUSED
+# if defined(__GNUC__)
+# if !(defined(__cplusplus)) || (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4))
+# define CYTHON_UNUSED __attribute__ ((__unused__))
+# else
+# define CYTHON_UNUSED
+# endif
+# elif defined(__ICC) || (defined(__INTEL_COMPILER) && !defined(_MSC_VER))
+# define CYTHON_UNUSED __attribute__ ((__unused__))
+# else
+# define CYTHON_UNUSED
+# endif
+#endif
+#ifndef CYTHON_UNUSED_VAR
+# if defined(__cplusplus)
+ template void CYTHON_UNUSED_VAR( const T& ) { }
+# else
+# define CYTHON_UNUSED_VAR(x) (void)(x)
+# endif
+#endif
+#ifndef CYTHON_MAYBE_UNUSED_VAR
+ #define CYTHON_MAYBE_UNUSED_VAR(x) CYTHON_UNUSED_VAR(x)
+#endif
+#ifndef CYTHON_NCP_UNUSED
+# if CYTHON_COMPILING_IN_CPYTHON && !CYTHON_COMPILING_IN_CPYTHON_FREETHREADING
+# define CYTHON_NCP_UNUSED
+# else
+# define CYTHON_NCP_UNUSED CYTHON_UNUSED
+# endif
+#endif
+#ifndef CYTHON_USE_CPP_STD_MOVE
+ #if defined(__cplusplus) && (\
+ __cplusplus >= 201103L || (defined(_MSC_VER) && _MSC_VER >= 1600))
+ #define CYTHON_USE_CPP_STD_MOVE 1
+ #else
+ #define CYTHON_USE_CPP_STD_MOVE 0
+ #endif
+#endif
+#define __Pyx_void_to_None(void_result) ((void)(void_result), Py_INCREF(Py_None), Py_None)
+#ifdef _MSC_VER
+ #ifndef _MSC_STDINT_H_
+ #if _MSC_VER < 1300
+ typedef unsigned char uint8_t;
+ typedef unsigned short uint16_t;
+ typedef unsigned int uint32_t;
+ #else
+ typedef unsigned __int8 uint8_t;
+ typedef unsigned __int16 uint16_t;
+ typedef unsigned __int32 uint32_t;
+ #endif
+ #endif
+ #if _MSC_VER < 1300
+ #ifdef _WIN64
+ typedef unsigned long long __pyx_uintptr_t;
+ #else
+ typedef unsigned int __pyx_uintptr_t;
+ #endif
+ #else
+ #ifdef _WIN64
+ typedef unsigned __int64 __pyx_uintptr_t;
+ #else
+ typedef unsigned __int32 __pyx_uintptr_t;
+ #endif
+ #endif
+#else
+ #include
+ typedef uintptr_t __pyx_uintptr_t;
+#endif
+#ifndef CYTHON_FALLTHROUGH
+ #if defined(__cplusplus)
+ /* for clang __has_cpp_attribute(fallthrough) is true even before C++17
+ * but leads to warnings with -pedantic, since it is a C++17 feature */
+ #if ((defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) || __cplusplus >= 201703L)
+ #if __has_cpp_attribute(fallthrough)
+ #define CYTHON_FALLTHROUGH [[fallthrough]]
+ #endif
+ #endif
+ #ifndef CYTHON_FALLTHROUGH
+ #if __has_cpp_attribute(clang::fallthrough)
+ #define CYTHON_FALLTHROUGH [[clang::fallthrough]]
+ #elif __has_cpp_attribute(gnu::fallthrough)
+ #define CYTHON_FALLTHROUGH [[gnu::fallthrough]]
+ #endif
+ #endif
+ #endif
+ #ifndef CYTHON_FALLTHROUGH
+ #if __has_attribute(fallthrough)
+ #define CYTHON_FALLTHROUGH __attribute__((fallthrough))
+ #else
+ #define CYTHON_FALLTHROUGH
+ #endif
+ #endif
+ #if defined(__clang__) && defined(__apple_build_version__)
+ #if __apple_build_version__ < 7000000
+ #undef CYTHON_FALLTHROUGH
+ #define CYTHON_FALLTHROUGH
+ #endif
+ #endif
+#endif
+#ifndef Py_UNREACHABLE
+ #define Py_UNREACHABLE() assert(0); abort()
+#endif
+#ifdef __cplusplus
+ template
+ struct __PYX_IS_UNSIGNED_IMPL {static const bool value = T(0) < T(-1);};
+ #define __PYX_IS_UNSIGNED(type) (__PYX_IS_UNSIGNED_IMPL::value)
+#else
+ #define __PYX_IS_UNSIGNED(type) (((type)-1) > 0)
+#endif
+#if CYTHON_COMPILING_IN_PYPY == 1
+ #define __PYX_NEED_TP_PRINT_SLOT (PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x030A0000)
+#else
+ #define __PYX_NEED_TP_PRINT_SLOT (PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000)
+#endif
+#define __PYX_REINTERPRET_FUNCION(func_pointer, other_pointer) ((func_pointer)(void(*)(void))(other_pointer))
+
+/* CInitCode */
+#ifndef CYTHON_INLINE
+ #if defined(__clang__)
+ #define CYTHON_INLINE __inline__ __attribute__ ((__unused__))
+ #elif defined(__GNUC__)
+ #define CYTHON_INLINE __inline__
+ #elif defined(_MSC_VER)
+ #define CYTHON_INLINE __inline
+ #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L
+ #define CYTHON_INLINE inline
+ #else
+ #define CYTHON_INLINE
+ #endif
+#endif
+
+/* PythonCompatibility */
+#define __PYX_BUILD_PY_SSIZE_T "n"
+#define CYTHON_FORMAT_SSIZE_T "z"
+#define __Pyx_BUILTIN_MODULE_NAME "builtins"
+#define __Pyx_DefaultClassType PyType_Type
+#if CYTHON_COMPILING_IN_LIMITED_API
+ #ifndef CO_OPTIMIZED
+ static int CO_OPTIMIZED;
+ #endif
+ #ifndef CO_NEWLOCALS
+ static int CO_NEWLOCALS;
+ #endif
+ #ifndef CO_VARARGS
+ static int CO_VARARGS;
+ #endif
+ #ifndef CO_VARKEYWORDS
+ static int CO_VARKEYWORDS;
+ #endif
+ #ifndef CO_ASYNC_GENERATOR
+ static int CO_ASYNC_GENERATOR;
+ #endif
+ #ifndef CO_GENERATOR
+ static int CO_GENERATOR;
+ #endif
+ #ifndef CO_COROUTINE
+ static int CO_COROUTINE;
+ #endif
+#else
+ #ifndef CO_COROUTINE
+ #define CO_COROUTINE 0x80
+ #endif
+ #ifndef CO_ASYNC_GENERATOR
+ #define CO_ASYNC_GENERATOR 0x200
+ #endif
+#endif
+static int __Pyx_init_co_variables(void);
+#if PY_VERSION_HEX >= 0x030900A4 || defined(Py_IS_TYPE)
+ #define __Pyx_IS_TYPE(ob, type) Py_IS_TYPE(ob, type)
+#else
+ #define __Pyx_IS_TYPE(ob, type) (((const PyObject*)ob)->ob_type == (type))
+#endif
+#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_Is)
+ #define __Pyx_Py_Is(x, y) Py_Is(x, y)
+#else
+ #define __Pyx_Py_Is(x, y) ((x) == (y))
+#endif
+#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_IsNone)
+ #define __Pyx_Py_IsNone(ob) Py_IsNone(ob)
+#else
+ #define __Pyx_Py_IsNone(ob) __Pyx_Py_Is((ob), Py_None)
+#endif
+#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_IsTrue)
+ #define __Pyx_Py_IsTrue(ob) Py_IsTrue(ob)
+#else
+ #define __Pyx_Py_IsTrue(ob) __Pyx_Py_Is((ob), Py_True)
+#endif
+#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_IsFalse)
+ #define __Pyx_Py_IsFalse(ob) Py_IsFalse(ob)
+#else
+ #define __Pyx_Py_IsFalse(ob) __Pyx_Py_Is((ob), Py_False)
+#endif
+#define __Pyx_NoneAsNull(obj) (__Pyx_Py_IsNone(obj) ? NULL : (obj))
+#if PY_VERSION_HEX >= 0x030900F0 && !CYTHON_COMPILING_IN_PYPY
+ #define __Pyx_PyObject_GC_IsFinalized(o) PyObject_GC_IsFinalized(o)
+#else
+ #define __Pyx_PyObject_GC_IsFinalized(o) _PyGC_FINALIZED(o)
+#endif
+#ifndef Py_TPFLAGS_CHECKTYPES
+ #define Py_TPFLAGS_CHECKTYPES 0
+#endif
+#ifndef Py_TPFLAGS_HAVE_INDEX
+ #define Py_TPFLAGS_HAVE_INDEX 0
+#endif
+#ifndef Py_TPFLAGS_HAVE_NEWBUFFER
+ #define Py_TPFLAGS_HAVE_NEWBUFFER 0
+#endif
+#ifndef Py_TPFLAGS_HAVE_FINALIZE
+ #define Py_TPFLAGS_HAVE_FINALIZE 0
+#endif
+#ifndef Py_TPFLAGS_SEQUENCE
+ #define Py_TPFLAGS_SEQUENCE 0
+#endif
+#ifndef Py_TPFLAGS_MAPPING
+ #define Py_TPFLAGS_MAPPING 0
+#endif
+#ifndef METH_STACKLESS
+ #define METH_STACKLESS 0
+#endif
+#ifndef METH_FASTCALL
+ #ifndef METH_FASTCALL
+ #define METH_FASTCALL 0x80
+ #endif
+ typedef PyObject *(*__Pyx_PyCFunctionFast) (PyObject *self, PyObject *const *args, Py_ssize_t nargs);
+ typedef PyObject *(*__Pyx_PyCFunctionFastWithKeywords) (PyObject *self, PyObject *const *args,
+ Py_ssize_t nargs, PyObject *kwnames);
+#else
+ #if PY_VERSION_HEX >= 0x030d00A4
+ # define __Pyx_PyCFunctionFast PyCFunctionFast
+ # define __Pyx_PyCFunctionFastWithKeywords PyCFunctionFastWithKeywords
+ #else
+ # define __Pyx_PyCFunctionFast _PyCFunctionFast
+ # define __Pyx_PyCFunctionFastWithKeywords _PyCFunctionFastWithKeywords
+ #endif
+#endif
+#if CYTHON_METH_FASTCALL
+ #define __Pyx_METH_FASTCALL METH_FASTCALL
+ #define __Pyx_PyCFunction_FastCall __Pyx_PyCFunctionFast
+ #define __Pyx_PyCFunction_FastCallWithKeywords __Pyx_PyCFunctionFastWithKeywords
+#else
+ #define __Pyx_METH_FASTCALL METH_VARARGS
+ #define __Pyx_PyCFunction_FastCall PyCFunction
+ #define __Pyx_PyCFunction_FastCallWithKeywords PyCFunctionWithKeywords
+#endif
+#if CYTHON_VECTORCALL
+ #define __pyx_vectorcallfunc vectorcallfunc
+ #define __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET PY_VECTORCALL_ARGUMENTS_OFFSET
+ #define __Pyx_PyVectorcall_NARGS(n) PyVectorcall_NARGS((size_t)(n))
+#elif CYTHON_BACKPORT_VECTORCALL
+ typedef PyObject *(*__pyx_vectorcallfunc)(PyObject *callable, PyObject *const *args,
+ size_t nargsf, PyObject *kwnames);
+ #define __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET ((size_t)1 << (8 * sizeof(size_t) - 1))
+ #define __Pyx_PyVectorcall_NARGS(n) ((Py_ssize_t)(((size_t)(n)) & ~__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET))
+#else
+ #define __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET 0
+ #define __Pyx_PyVectorcall_NARGS(n) ((Py_ssize_t)(n))
+#endif
+#if PY_VERSION_HEX >= 0x030900B1
+#define __Pyx_PyCFunction_CheckExact(func) PyCFunction_CheckExact(func)
+#else
+#define __Pyx_PyCFunction_CheckExact(func) PyCFunction_Check(func)
+#endif
+#define __Pyx_CyOrPyCFunction_Check(func) PyCFunction_Check(func)
+#if CYTHON_COMPILING_IN_CPYTHON
+#define __Pyx_CyOrPyCFunction_GET_FUNCTION(func) (((PyCFunctionObject*)(func))->m_ml->ml_meth)
+#elif !CYTHON_COMPILING_IN_LIMITED_API
+#define __Pyx_CyOrPyCFunction_GET_FUNCTION(func) PyCFunction_GET_FUNCTION(func)
+#endif
+#if CYTHON_COMPILING_IN_CPYTHON
+#define __Pyx_CyOrPyCFunction_GET_FLAGS(func) (((PyCFunctionObject*)(func))->m_ml->ml_flags)
+static CYTHON_INLINE PyObject* __Pyx_CyOrPyCFunction_GET_SELF(PyObject *func) {
+ return (__Pyx_CyOrPyCFunction_GET_FLAGS(func) & METH_STATIC) ? NULL : ((PyCFunctionObject*)func)->m_self;
+}
+#endif
+static CYTHON_INLINE int __Pyx__IsSameCFunction(PyObject *func, void (*cfunc)(void)) {
+#if CYTHON_COMPILING_IN_LIMITED_API
+ return PyCFunction_Check(func) && PyCFunction_GetFunction(func) == (PyCFunction) cfunc;
+#else
+ return PyCFunction_Check(func) && PyCFunction_GET_FUNCTION(func) == (PyCFunction) cfunc;
+#endif
+}
+#define __Pyx_IsSameCFunction(func, cfunc) __Pyx__IsSameCFunction(func, cfunc)
+#if __PYX_LIMITED_VERSION_HEX < 0x03090000
+ #define __Pyx_PyType_FromModuleAndSpec(m, s, b) ((void)m, PyType_FromSpecWithBases(s, b))
+ typedef PyObject *(*__Pyx_PyCMethod)(PyObject *, PyTypeObject *, PyObject *const *, size_t, PyObject *);
+#else
+ #define __Pyx_PyType_FromModuleAndSpec(m, s, b) PyType_FromModuleAndSpec(m, s, b)
+ #define __Pyx_PyCMethod PyCMethod
+#endif
+#ifndef METH_METHOD
+ #define METH_METHOD 0x200
+#endif
+#if CYTHON_COMPILING_IN_PYPY && !defined(PyObject_Malloc)
+ #define PyObject_Malloc(s) PyMem_Malloc(s)
+ #define PyObject_Free(p) PyMem_Free(p)
+ #define PyObject_Realloc(p) PyMem_Realloc(p)
+#endif
+#if CYTHON_COMPILING_IN_LIMITED_API
+ #define __Pyx_PyFrame_SetLineNumber(frame, lineno)
+#elif CYTHON_COMPILING_IN_GRAAL
+ #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0)
+ #define __Pyx_PyFrame_SetLineNumber(frame, lineno) _PyFrame_SetLineNumber((frame), (lineno))
+#else
+ #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0)
+ #define __Pyx_PyFrame_SetLineNumber(frame, lineno) (frame)->f_lineno = (lineno)
+#endif
+#if CYTHON_COMPILING_IN_LIMITED_API
+ #define __Pyx_PyThreadState_Current PyThreadState_Get()
+#elif !CYTHON_FAST_THREAD_STATE
+ #define __Pyx_PyThreadState_Current PyThreadState_GET()
+#elif PY_VERSION_HEX >= 0x030d00A1
+ #define __Pyx_PyThreadState_Current PyThreadState_GetUnchecked()
+#else
+ #define __Pyx_PyThreadState_Current _PyThreadState_UncheckedGet()
+#endif
+#if CYTHON_USE_MODULE_STATE
+static CYTHON_INLINE void *__Pyx__PyModule_GetState(PyObject *op)
+{
+ void *result;
+ result = PyModule_GetState(op);
+ if (!result)
+ Py_FatalError("Couldn't find the module state");
+ return result;
+}
+#define __Pyx_PyModule_GetState(o) (__pyx_mstatetype *)__Pyx__PyModule_GetState(o)
+#else
+#define __Pyx_PyModule_GetState(op) ((void)op,__pyx_mstate_global)
+#endif
+#define __Pyx_PyObject_GetSlot(obj, name, func_ctype) __Pyx_PyType_GetSlot(Py_TYPE((PyObject *) obj), name, func_ctype)
+#define __Pyx_PyObject_TryGetSlot(obj, name, func_ctype) __Pyx_PyType_TryGetSlot(Py_TYPE(obj), name, func_ctype)
+#define __Pyx_PyObject_GetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_GetSubSlot(Py_TYPE(obj), sub, name, func_ctype)
+#define __Pyx_PyObject_TryGetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_TryGetSubSlot(Py_TYPE(obj), sub, name, func_ctype)
+#if CYTHON_USE_TYPE_SLOTS
+ #define __Pyx_PyType_GetSlot(type, name, func_ctype) ((type)->name)
+ #define __Pyx_PyType_TryGetSlot(type, name, func_ctype) __Pyx_PyType_GetSlot(type, name, func_ctype)
+ #define __Pyx_PyType_GetSubSlot(type, sub, name, func_ctype) (((type)->sub) ? ((type)->sub->name) : NULL)
+ #define __Pyx_PyType_TryGetSubSlot(type, sub, name, func_ctype) __Pyx_PyType_GetSubSlot(type, sub, name, func_ctype)
+#else
+ #define __Pyx_PyType_GetSlot(type, name, func_ctype) ((func_ctype) PyType_GetSlot((type), Py_##name))
+ #define __Pyx_PyType_TryGetSlot(type, name, func_ctype)\
+ ((__PYX_LIMITED_VERSION_HEX >= 0x030A0000 ||\
+ (PyType_GetFlags(type) & Py_TPFLAGS_HEAPTYPE) || __Pyx_get_runtime_version() >= 0x030A0000) ?\
+ __Pyx_PyType_GetSlot(type, name, func_ctype) : NULL)
+ #define __Pyx_PyType_GetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_GetSlot(obj, name, func_ctype)
+ #define __Pyx_PyType_TryGetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_TryGetSlot(obj, name, func_ctype)
+#endif
+#if CYTHON_COMPILING_IN_CPYTHON || defined(_PyDict_NewPresized)
+#define __Pyx_PyDict_NewPresized(n) ((n <= 8) ? PyDict_New() : _PyDict_NewPresized(n))
+#else
+#define __Pyx_PyDict_NewPresized(n) PyDict_New()
+#endif
+#define __Pyx_PyNumber_Divide(x,y) PyNumber_TrueDivide(x,y)
+#define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceTrueDivide(x,y)
+#if CYTHON_COMPILING_IN_CPYTHON && CYTHON_USE_UNICODE_INTERNALS
+#define __Pyx_PyDict_GetItemStrWithError(dict, name) _PyDict_GetItem_KnownHash(dict, name, ((PyASCIIObject *) name)->hash)
+static CYTHON_INLINE PyObject * __Pyx_PyDict_GetItemStr(PyObject *dict, PyObject *name) {
+ PyObject *res = __Pyx_PyDict_GetItemStrWithError(dict, name);
+ if (res == NULL) PyErr_Clear();
+ return res;
+}
+#elif !CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07020000
+#define __Pyx_PyDict_GetItemStrWithError PyDict_GetItemWithError
+#define __Pyx_PyDict_GetItemStr PyDict_GetItem
+#else
+static CYTHON_INLINE PyObject * __Pyx_PyDict_GetItemStrWithError(PyObject *dict, PyObject *name) {
+#if CYTHON_COMPILING_IN_PYPY
+ return PyDict_GetItem(dict, name);
+#else
+ PyDictEntry *ep;
+ PyDictObject *mp = (PyDictObject*) dict;
+ long hash = ((PyStringObject *) name)->ob_shash;
+ assert(hash != -1);
+ ep = (mp->ma_lookup)(mp, name, hash);
+ if (ep == NULL) {
+ return NULL;
+ }
+ return ep->me_value;
+#endif
+}
+#define __Pyx_PyDict_GetItemStr PyDict_GetItem
+#endif
+#if CYTHON_USE_TYPE_SLOTS
+ #define __Pyx_PyType_GetFlags(tp) (((PyTypeObject *)tp)->tp_flags)
+ #define __Pyx_PyType_HasFeature(type, feature) ((__Pyx_PyType_GetFlags(type) & (feature)) != 0)
+#else
+ #define __Pyx_PyType_GetFlags(tp) (PyType_GetFlags((PyTypeObject *)tp))
+ #define __Pyx_PyType_HasFeature(type, feature) PyType_HasFeature(type, feature)
+#endif
+#define __Pyx_PyObject_GetIterNextFunc(iterator) __Pyx_PyObject_GetSlot(iterator, tp_iternext, iternextfunc)
+#if CYTHON_USE_TYPE_SPECS && PY_VERSION_HEX >= 0x03080000
+#define __Pyx_PyHeapTypeObject_GC_Del(obj) {\
+ PyTypeObject *type = Py_TYPE((PyObject*)obj);\
+ assert(__Pyx_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE));\
+ PyObject_GC_Del(obj);\
+ Py_DECREF(type);\
+}
+#else
+#define __Pyx_PyHeapTypeObject_GC_Del(obj) PyObject_GC_Del(obj)
+#endif
+#if CYTHON_COMPILING_IN_LIMITED_API
+ #define __Pyx_PyUnicode_READY(op) (0)
+ #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_ReadChar(u, i)
+ #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) ((void)u, 1114111U)
+ #define __Pyx_PyUnicode_KIND(u) ((void)u, (0))
+ #define __Pyx_PyUnicode_DATA(u) ((void*)u)
+ #define __Pyx_PyUnicode_READ(k, d, i) ((void)k, PyUnicode_ReadChar((PyObject*)(d), i))
+ #define __Pyx_PyUnicode_IS_TRUE(u) (0 != PyUnicode_GetLength(u))
+#else
+ #if PY_VERSION_HEX >= 0x030C0000
+ #define __Pyx_PyUnicode_READY(op) (0)
+ #else
+ #define __Pyx_PyUnicode_READY(op) (likely(PyUnicode_IS_READY(op)) ?\
+ 0 : _PyUnicode_Ready((PyObject *)(op)))
+ #endif
+ #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_READ_CHAR(u, i)
+ #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) PyUnicode_MAX_CHAR_VALUE(u)
+ #define __Pyx_PyUnicode_KIND(u) ((int)PyUnicode_KIND(u))
+ #define __Pyx_PyUnicode_DATA(u) PyUnicode_DATA(u)
+ #define __Pyx_PyUnicode_READ(k, d, i) PyUnicode_READ(k, d, i)
+ #define __Pyx_PyUnicode_WRITE(k, d, i, ch) PyUnicode_WRITE(k, d, i, (Py_UCS4) ch)
+ #if PY_VERSION_HEX >= 0x030C0000
+ #define __Pyx_PyUnicode_IS_TRUE(u) (0 != PyUnicode_GET_LENGTH(u))
+ #else
+ #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x03090000
+ #define __Pyx_PyUnicode_IS_TRUE(u) (0 != (likely(PyUnicode_IS_READY(u)) ? PyUnicode_GET_LENGTH(u) : ((PyCompactUnicodeObject *)(u))->wstr_length))
+ #else
+ #define __Pyx_PyUnicode_IS_TRUE(u) (0 != (likely(PyUnicode_IS_READY(u)) ? PyUnicode_GET_LENGTH(u) : PyUnicode_GET_SIZE(u)))
+ #endif
+ #endif
+#endif
+#if CYTHON_COMPILING_IN_PYPY
+ #define __Pyx_PyUnicode_Concat(a, b) PyNumber_Add(a, b)
+ #define __Pyx_PyUnicode_ConcatSafe(a, b) PyNumber_Add(a, b)
+#else
+ #define __Pyx_PyUnicode_Concat(a, b) PyUnicode_Concat(a, b)
+ #define __Pyx_PyUnicode_ConcatSafe(a, b) ((unlikely((a) == Py_None) || unlikely((b) == Py_None)) ?\
+ PyNumber_Add(a, b) : __Pyx_PyUnicode_Concat(a, b))
+#endif
+#if CYTHON_COMPILING_IN_PYPY
+ #if !defined(PyUnicode_DecodeUnicodeEscape)
+ #define PyUnicode_DecodeUnicodeEscape(s, size, errors) PyUnicode_Decode(s, size, "unicode_escape", errors)
+ #endif
+ #if !defined(PyUnicode_Contains)
+ #define PyUnicode_Contains(u, s) PySequence_Contains(u, s)
+ #endif
+ #if !defined(PyByteArray_Check)
+ #define PyByteArray_Check(obj) PyObject_TypeCheck(obj, &PyByteArray_Type)
+ #endif
+ #if !defined(PyObject_Format)
+ #define PyObject_Format(obj, fmt) PyObject_CallMethod(obj, "__format__", "O", fmt)
+ #endif
+#endif
+#define __Pyx_PyUnicode_FormatSafe(a, b) ((unlikely((a) == Py_None || (PyUnicode_Check(b) && !PyUnicode_CheckExact(b)))) ? PyNumber_Remainder(a, b) : PyUnicode_Format(a, b))
+#if CYTHON_COMPILING_IN_CPYTHON
+ #define __Pyx_PySequence_ListKeepNew(obj)\
+ (likely(PyList_CheckExact(obj) && Py_REFCNT(obj) == 1) ? __Pyx_NewRef(obj) : PySequence_List(obj))
+#else
+ #define __Pyx_PySequence_ListKeepNew(obj) PySequence_List(obj)
+#endif
+#ifndef PySet_CheckExact
+ #define PySet_CheckExact(obj) __Pyx_IS_TYPE(obj, &PySet_Type)
+#endif
+#if PY_VERSION_HEX >= 0x030900A4
+ #define __Pyx_SET_REFCNT(obj, refcnt) Py_SET_REFCNT(obj, refcnt)
+ #define __Pyx_SET_SIZE(obj, size) Py_SET_SIZE(obj, size)
+#else
+ #define __Pyx_SET_REFCNT(obj, refcnt) Py_REFCNT(obj) = (refcnt)
+ #define __Pyx_SET_SIZE(obj, size) Py_SIZE(obj) = (size)
+#endif
+#if CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS
+ #if __PYX_LIMITED_VERSION_HEX >= 0x030d0000
+ #define __Pyx_PyList_GetItemRef(o, i) PyList_GetItemRef(o, i)
+ #elif CYTHON_COMPILING_IN_LIMITED_API || !CYTHON_ASSUME_SAFE_MACROS
+ #define __Pyx_PyList_GetItemRef(o, i) (likely((i) >= 0) ? PySequence_GetItem(o, i) : (PyErr_SetString(PyExc_IndexError, "list index out of range"), (PyObject*)NULL))
+ #else
+ #define __Pyx_PyList_GetItemRef(o, i) PySequence_ITEM(o, i)
+ #endif
+#elif CYTHON_COMPILING_IN_LIMITED_API || !CYTHON_ASSUME_SAFE_MACROS
+ #if __PYX_LIMITED_VERSION_HEX >= 0x030d0000
+ #define __Pyx_PyList_GetItemRef(o, i) PyList_GetItemRef(o, i)
+ #else
+ #define __Pyx_PyList_GetItemRef(o, i) __Pyx_XNewRef(PyList_GetItem(o, i))
+ #endif
+#else
+ #define __Pyx_PyList_GetItemRef(o, i) __Pyx_NewRef(PyList_GET_ITEM(o, i))
+#endif
+#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000
+#define __Pyx_PyDict_GetItemRef(dict, key, result) PyDict_GetItemRef(dict, key, result)
+#elif CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS
+static CYTHON_INLINE int __Pyx_PyDict_GetItemRef(PyObject *dict, PyObject *key, PyObject **result) {
+ *result = PyObject_GetItem(dict, key);
+ if (*result == NULL) {
+ if (PyErr_ExceptionMatches(PyExc_KeyError)) {
+ PyErr_Clear();
+ return 0;
+ }
+ return -1;
+ }
+ return 1;
+}
+#else
+static CYTHON_INLINE int __Pyx_PyDict_GetItemRef(PyObject *dict, PyObject *key, PyObject **result) {
+ *result = PyDict_GetItemWithError(dict, key);
+ if (*result == NULL) {
+ return PyErr_Occurred() ? -1 : 0;
+ }
+ Py_INCREF(*result);
+ return 1;
+}
+#endif
+#if defined(CYTHON_DEBUG_VISIT_CONST) && CYTHON_DEBUG_VISIT_CONST
+ #define __Pyx_VISIT_CONST(obj) Py_VISIT(obj)
+#else
+ #define __Pyx_VISIT_CONST(obj)
+#endif
+#if CYTHON_ASSUME_SAFE_MACROS
+ #define __Pyx_PySequence_ITEM(o, i) PySequence_ITEM(o, i)
+ #define __Pyx_PySequence_SIZE(seq) Py_SIZE(seq)
+ #define __Pyx_PyTuple_SET_ITEM(o, i, v) (PyTuple_SET_ITEM(o, i, v), (0))
+ #define __Pyx_PyTuple_GET_ITEM(o, i) PyTuple_GET_ITEM(o, i)
+ #define __Pyx_PyList_SET_ITEM(o, i, v) (PyList_SET_ITEM(o, i, v), (0))
+ #define __Pyx_PyList_GET_ITEM(o, i) PyList_GET_ITEM(o, i)
+#else
+ #define __Pyx_PySequence_ITEM(o, i) PySequence_GetItem(o, i)
+ #define __Pyx_PySequence_SIZE(seq) PySequence_Size(seq)
+ #define __Pyx_PyTuple_SET_ITEM(o, i, v) PyTuple_SetItem(o, i, v)
+ #define __Pyx_PyTuple_GET_ITEM(o, i) PyTuple_GetItem(o, i)
+ #define __Pyx_PyList_SET_ITEM(o, i, v) PyList_SetItem(o, i, v)
+ #define __Pyx_PyList_GET_ITEM(o, i) PyList_GetItem(o, i)
+#endif
+#if CYTHON_ASSUME_SAFE_SIZE
+ #define __Pyx_PyTuple_GET_SIZE(o) PyTuple_GET_SIZE(o)
+ #define __Pyx_PyList_GET_SIZE(o) PyList_GET_SIZE(o)
+ #define __Pyx_PySet_GET_SIZE(o) PySet_GET_SIZE(o)
+ #define __Pyx_PyBytes_GET_SIZE(o) PyBytes_GET_SIZE(o)
+ #define __Pyx_PyByteArray_GET_SIZE(o) PyByteArray_GET_SIZE(o)
+ #define __Pyx_PyUnicode_GET_LENGTH(o) PyUnicode_GET_LENGTH(o)
+#else
+ #define __Pyx_PyTuple_GET_SIZE(o) PyTuple_Size(o)
+ #define __Pyx_PyList_GET_SIZE(o) PyList_Size(o)
+ #define __Pyx_PySet_GET_SIZE(o) PySet_Size(o)
+ #define __Pyx_PyBytes_GET_SIZE(o) PyBytes_Size(o)
+ #define __Pyx_PyByteArray_GET_SIZE(o) PyByteArray_Size(o)
+ #define __Pyx_PyUnicode_GET_LENGTH(o) PyUnicode_GetLength(o)
+#endif
+#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000
+ #define __Pyx_PyImport_AddModuleRef(name) PyImport_AddModuleRef(name)
+#else
+ static CYTHON_INLINE PyObject *__Pyx_PyImport_AddModuleRef(const char *name) {
+ PyObject *module = PyImport_AddModule(name);
+ Py_XINCREF(module);
+ return module;
+ }
+#endif
+#if CYTHON_COMPILING_IN_PYPY && !defined(PyUnicode_InternFromString)
+ #define PyUnicode_InternFromString(s) PyUnicode_FromString(s)
+#endif
+#define __Pyx_PyLong_FromHash_t PyLong_FromSsize_t
+#define __Pyx_PyLong_AsHash_t __Pyx_PyIndex_AsSsize_t
+#if __PYX_LIMITED_VERSION_HEX >= 0x030A0000
+ #define __Pyx_PySendResult PySendResult
+#else
+ typedef enum {
+ PYGEN_RETURN = 0,
+ PYGEN_ERROR = -1,
+ PYGEN_NEXT = 1,
+ } __Pyx_PySendResult;
+#endif
+#if CYTHON_COMPILING_IN_LIMITED_API || PY_VERSION_HEX < 0x030A00A3
+ typedef __Pyx_PySendResult (*__Pyx_pyiter_sendfunc)(PyObject *iter, PyObject *value, PyObject **result);
+#else
+ #define __Pyx_pyiter_sendfunc sendfunc
+#endif
+#if !CYTHON_USE_AM_SEND
+#define __PYX_HAS_PY_AM_SEND 0
+#elif __PYX_LIMITED_VERSION_HEX >= 0x030A0000
+#define __PYX_HAS_PY_AM_SEND 1
+#else
+#define __PYX_HAS_PY_AM_SEND 2 // our own backported implementation
+#endif
+#if __PYX_HAS_PY_AM_SEND < 2
+ #define __Pyx_PyAsyncMethodsStruct PyAsyncMethods
+#else
+ typedef struct {
+ unaryfunc am_await;
+ unaryfunc am_aiter;
+ unaryfunc am_anext;
+ __Pyx_pyiter_sendfunc am_send;
+ } __Pyx_PyAsyncMethodsStruct;
+ #define __Pyx_SlotTpAsAsync(s) ((PyAsyncMethods*)(s))
+#endif
+#if CYTHON_USE_AM_SEND && PY_VERSION_HEX < 0x030A00F0
+ #define __Pyx_TPFLAGS_HAVE_AM_SEND (1UL << 21)
+#else
+ #define __Pyx_TPFLAGS_HAVE_AM_SEND (0)
+#endif
+#if PY_VERSION_HEX >= 0x03090000
+#define __Pyx_PyInterpreterState_Get() PyInterpreterState_Get()
+#else
+#define __Pyx_PyInterpreterState_Get() PyThreadState_Get()->interp
+#endif
+#if CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x030A0000
+#ifdef __cplusplus
+extern "C"
+#endif
+PyAPI_FUNC(void *) PyMem_Calloc(size_t nelem, size_t elsize);
+#endif
+#if CYTHON_COMPILING_IN_LIMITED_API
+static int __Pyx_init_co_variable(PyObject *inspect, const char* name, int *write_to) {
+ int value;
+ PyObject *py_value = PyObject_GetAttrString(inspect, name);
+ if (!py_value) return 0;
+ value = (int) PyLong_AsLong(py_value);
+ Py_DECREF(py_value);
+ *write_to = value;
+ return value != -1 || !PyErr_Occurred();
+}
+static int __Pyx_init_co_variables(void) {
+ PyObject *inspect;
+ int result;
+ inspect = PyImport_ImportModule("inspect");
+ result =
+#if !defined(CO_OPTIMIZED)
+ __Pyx_init_co_variable(inspect, "CO_OPTIMIZED", &CO_OPTIMIZED) &&
+#endif
+#if !defined(CO_NEWLOCALS)
+ __Pyx_init_co_variable(inspect, "CO_NEWLOCALS", &CO_NEWLOCALS) &&
+#endif
+#if !defined(CO_VARARGS)
+ __Pyx_init_co_variable(inspect, "CO_VARARGS", &CO_VARARGS) &&
+#endif
+#if !defined(CO_VARKEYWORDS)
+ __Pyx_init_co_variable(inspect, "CO_VARKEYWORDS", &CO_VARKEYWORDS) &&
+#endif
+#if !defined(CO_ASYNC_GENERATOR)
+ __Pyx_init_co_variable(inspect, "CO_ASYNC_GENERATOR", &CO_ASYNC_GENERATOR) &&
+#endif
+#if !defined(CO_GENERATOR)
+ __Pyx_init_co_variable(inspect, "CO_GENERATOR", &CO_GENERATOR) &&
+#endif
+#if !defined(CO_COROUTINE)
+ __Pyx_init_co_variable(inspect, "CO_COROUTINE", &CO_COROUTINE) &&
+#endif
+ 1;
+ Py_DECREF(inspect);
+ return result ? 0 : -1;
+}
+#else
+static int __Pyx_init_co_variables(void) {
+ return 0; // It's a limited API-only feature
+}
+#endif
+
+/* MathInitCode */
+#if defined(_WIN32) || defined(WIN32) || defined(MS_WINDOWS)
+ #ifndef _USE_MATH_DEFINES
+ #define _USE_MATH_DEFINES
+ #endif
+#endif
+#include
+#ifdef NAN
+#define __PYX_NAN() ((float) NAN)
+#else
+static CYTHON_INLINE float __PYX_NAN() {
+ float value;
+ memset(&value, 0xFF, sizeof(value));
+ return value;
+}
+#endif
+#if defined(__CYGWIN__) && defined(_LDBL_EQ_DBL)
+#define __Pyx_truncl trunc
+#else
+#define __Pyx_truncl truncl
+#endif
+
+#ifndef CYTHON_CLINE_IN_TRACEBACK_RUNTIME
+#define CYTHON_CLINE_IN_TRACEBACK_RUNTIME 0
+#endif
+#ifndef CYTHON_CLINE_IN_TRACEBACK
+#define CYTHON_CLINE_IN_TRACEBACK CYTHON_CLINE_IN_TRACEBACK_RUNTIME
+#endif
+#if CYTHON_CLINE_IN_TRACEBACK
+#define __PYX_MARK_ERR_POS(f_index, lineno) { __pyx_filename = __pyx_f[f_index]; (void) __pyx_filename; __pyx_lineno = lineno; (void) __pyx_lineno; __pyx_clineno = __LINE__; (void) __pyx_clineno; }
+#else
+#define __PYX_MARK_ERR_POS(f_index, lineno) { __pyx_filename = __pyx_f[f_index]; (void) __pyx_filename; __pyx_lineno = lineno; (void) __pyx_lineno; (void) __pyx_clineno; }
+#endif
+#define __PYX_ERR(f_index, lineno, Ln_error) \
+ { __PYX_MARK_ERR_POS(f_index, lineno) goto Ln_error; }
+
+#ifdef CYTHON_EXTERN_C
+ #undef __PYX_EXTERN_C
+ #define __PYX_EXTERN_C CYTHON_EXTERN_C
+#elif defined(__PYX_EXTERN_C)
+ #ifdef _MSC_VER
+ #pragma message ("Please do not define the '__PYX_EXTERN_C' macro externally. Use 'CYTHON_EXTERN_C' instead.")
+ #else
+ #warning Please do not define the '__PYX_EXTERN_C' macro externally. Use 'CYTHON_EXTERN_C' instead.
+ #endif
+#else
+ #ifdef __cplusplus
+ #define __PYX_EXTERN_C extern "C"
+ #else
+ #define __PYX_EXTERN_C extern
+ #endif
+#endif
+
+#define __PYX_HAVE__fontTools__cu2qu__cu2qu
+#define __PYX_HAVE_API__fontTools__cu2qu__cu2qu
+/* Early includes */
+#ifdef _OPENMP
+#include
+#endif /* _OPENMP */
+
+#if defined(PYREX_WITHOUT_ASSERTIONS) && !defined(CYTHON_WITHOUT_ASSERTIONS)
+#define CYTHON_WITHOUT_ASSERTIONS
+#endif
+
+#define __PYX_DEFAULT_STRING_ENCODING_IS_ASCII 0
+#define __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 0
+#define __PYX_DEFAULT_STRING_ENCODING ""
+#define __Pyx_PyObject_FromString __Pyx_PyBytes_FromString
+#define __Pyx_PyObject_FromStringAndSize __Pyx_PyBytes_FromStringAndSize
+#define __Pyx_uchar_cast(c) ((unsigned char)c)
+#define __Pyx_long_cast(x) ((long)x)
+#define __Pyx_fits_Py_ssize_t(v, type, is_signed) (\
+ (sizeof(type) < sizeof(Py_ssize_t)) ||\
+ (sizeof(type) > sizeof(Py_ssize_t) &&\
+ likely(v < (type)PY_SSIZE_T_MAX ||\
+ v == (type)PY_SSIZE_T_MAX) &&\
+ (!is_signed || likely(v > (type)PY_SSIZE_T_MIN ||\
+ v == (type)PY_SSIZE_T_MIN))) ||\
+ (sizeof(type) == sizeof(Py_ssize_t) &&\
+ (is_signed || likely(v < (type)PY_SSIZE_T_MAX ||\
+ v == (type)PY_SSIZE_T_MAX))) )
+static CYTHON_INLINE int __Pyx_is_valid_index(Py_ssize_t i, Py_ssize_t limit) {
+ return (size_t) i < (size_t) limit;
+}
+#if defined (__cplusplus) && __cplusplus >= 201103L
+ #include
+ #define __Pyx_sst_abs(value) std::abs(value)
+#elif SIZEOF_INT >= SIZEOF_SIZE_T
+ #define __Pyx_sst_abs(value) abs(value)
+#elif SIZEOF_LONG >= SIZEOF_SIZE_T
+ #define __Pyx_sst_abs(value) labs(value)
+#elif defined (_MSC_VER)
+ #define __Pyx_sst_abs(value) ((Py_ssize_t)_abs64(value))
+#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L
+ #define __Pyx_sst_abs(value) llabs(value)
+#elif defined (__GNUC__)
+ #define __Pyx_sst_abs(value) __builtin_llabs(value)
+#else
+ #define __Pyx_sst_abs(value) ((value<0) ? -value : value)
+#endif
+static CYTHON_INLINE Py_ssize_t __Pyx_ssize_strlen(const char *s);
+static CYTHON_INLINE const char* __Pyx_PyObject_AsString(PyObject*);
+static CYTHON_INLINE const char* __Pyx_PyObject_AsStringAndSize(PyObject*, Py_ssize_t* length);
+static CYTHON_INLINE PyObject* __Pyx_PyByteArray_FromString(const char*);
+#define __Pyx_PyByteArray_FromStringAndSize(s, l) PyByteArray_FromStringAndSize((const char*)s, l)
+#define __Pyx_PyBytes_FromString PyBytes_FromString
+#define __Pyx_PyBytes_FromStringAndSize PyBytes_FromStringAndSize
+static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char*);
+#if CYTHON_ASSUME_SAFE_MACROS
+ #define __Pyx_PyBytes_AsWritableString(s) ((char*) PyBytes_AS_STRING(s))
+ #define __Pyx_PyBytes_AsWritableSString(s) ((signed char*) PyBytes_AS_STRING(s))
+ #define __Pyx_PyBytes_AsWritableUString(s) ((unsigned char*) PyBytes_AS_STRING(s))
+ #define __Pyx_PyBytes_AsString(s) ((const char*) PyBytes_AS_STRING(s))
+ #define __Pyx_PyBytes_AsSString(s) ((const signed char*) PyBytes_AS_STRING(s))
+ #define __Pyx_PyBytes_AsUString(s) ((const unsigned char*) PyBytes_AS_STRING(s))
+ #define __Pyx_PyByteArray_AsString(s) PyByteArray_AS_STRING(s)
+#else
+ #define __Pyx_PyBytes_AsWritableString(s) ((char*) PyBytes_AsString(s))
+ #define __Pyx_PyBytes_AsWritableSString(s) ((signed char*) PyBytes_AsString(s))
+ #define __Pyx_PyBytes_AsWritableUString(s) ((unsigned char*) PyBytes_AsString(s))
+ #define __Pyx_PyBytes_AsString(s) ((const char*) PyBytes_AsString(s))
+ #define __Pyx_PyBytes_AsSString(s) ((const signed char*) PyBytes_AsString(s))
+ #define __Pyx_PyBytes_AsUString(s) ((const unsigned char*) PyBytes_AsString(s))
+ #define __Pyx_PyByteArray_AsString(s) PyByteArray_AsString(s)
+#endif
+#define __Pyx_PyObject_AsWritableString(s) ((char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s))
+#define __Pyx_PyObject_AsWritableSString(s) ((signed char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s))
+#define __Pyx_PyObject_AsWritableUString(s) ((unsigned char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s))
+#define __Pyx_PyObject_AsSString(s) ((const signed char*) __Pyx_PyObject_AsString(s))
+#define __Pyx_PyObject_AsUString(s) ((const unsigned char*) __Pyx_PyObject_AsString(s))
+#define __Pyx_PyObject_FromCString(s) __Pyx_PyObject_FromString((const char*)s)
+#define __Pyx_PyBytes_FromCString(s) __Pyx_PyBytes_FromString((const char*)s)
+#define __Pyx_PyByteArray_FromCString(s) __Pyx_PyByteArray_FromString((const char*)s)
+#define __Pyx_PyUnicode_FromCString(s) __Pyx_PyUnicode_FromString((const char*)s)
+#define __Pyx_PyUnicode_FromOrdinal(o) PyUnicode_FromOrdinal((int)o)
+#define __Pyx_PyUnicode_AsUnicode PyUnicode_AsUnicode
+static CYTHON_INLINE PyObject *__Pyx_NewRef(PyObject *obj) {
+#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030a0000 || defined(Py_NewRef)
+ return Py_NewRef(obj);
+#else
+ Py_INCREF(obj);
+ return obj;
+#endif
+}
+static CYTHON_INLINE PyObject *__Pyx_XNewRef(PyObject *obj) {
+#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030a0000 || defined(Py_XNewRef)
+ return Py_XNewRef(obj);
+#else
+ Py_XINCREF(obj);
+ return obj;
+#endif
+}
+static CYTHON_INLINE PyObject *__Pyx_Owned_Py_None(int b);
+static CYTHON_INLINE PyObject * __Pyx_PyBool_FromLong(long b);
+static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject*);
+static CYTHON_INLINE int __Pyx_PyObject_IsTrueAndDecref(PyObject*);
+static CYTHON_INLINE PyObject* __Pyx_PyNumber_Long(PyObject* x);
+#define __Pyx_PySequence_Tuple(obj)\
+ (likely(PyTuple_CheckExact(obj)) ? __Pyx_NewRef(obj) : PySequence_Tuple(obj))
+static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject*);
+static CYTHON_INLINE PyObject * __Pyx_PyLong_FromSize_t(size_t);
+static CYTHON_INLINE Py_hash_t __Pyx_PyIndex_AsHash_t(PyObject*);
+#if CYTHON_ASSUME_SAFE_MACROS
+#define __Pyx_PyFloat_AsDouble(x) (PyFloat_CheckExact(x) ? PyFloat_AS_DOUBLE(x) : PyFloat_AsDouble(x))
+#define __Pyx_PyFloat_AS_DOUBLE(x) PyFloat_AS_DOUBLE(x)
+#else
+#define __Pyx_PyFloat_AsDouble(x) PyFloat_AsDouble(x)
+#define __Pyx_PyFloat_AS_DOUBLE(x) PyFloat_AsDouble(x)
+#endif
+#define __Pyx_PyFloat_AsFloat(x) ((float) __Pyx_PyFloat_AsDouble(x))
+#define __Pyx_PyNumber_Int(x) (PyLong_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Long(x))
+#if CYTHON_USE_PYLONG_INTERNALS
+ #if PY_VERSION_HEX >= 0x030C00A7
+ #ifndef _PyLong_SIGN_MASK
+ #define _PyLong_SIGN_MASK 3
+ #endif
+ #ifndef _PyLong_NON_SIZE_BITS
+ #define _PyLong_NON_SIZE_BITS 3
+ #endif
+ #define __Pyx_PyLong_Sign(x) (((PyLongObject*)x)->long_value.lv_tag & _PyLong_SIGN_MASK)
+ #define __Pyx_PyLong_IsNeg(x) ((__Pyx_PyLong_Sign(x) & 2) != 0)
+ #define __Pyx_PyLong_IsNonNeg(x) (!__Pyx_PyLong_IsNeg(x))
+ #define __Pyx_PyLong_IsZero(x) (__Pyx_PyLong_Sign(x) & 1)
+ #define __Pyx_PyLong_IsPos(x) (__Pyx_PyLong_Sign(x) == 0)
+ #define __Pyx_PyLong_CompactValueUnsigned(x) (__Pyx_PyLong_Digits(x)[0])
+ #define __Pyx_PyLong_DigitCount(x) ((Py_ssize_t) (((PyLongObject*)x)->long_value.lv_tag >> _PyLong_NON_SIZE_BITS))
+ #define __Pyx_PyLong_SignedDigitCount(x)\
+ ((1 - (Py_ssize_t) __Pyx_PyLong_Sign(x)) * __Pyx_PyLong_DigitCount(x))
+ #if defined(PyUnstable_Long_IsCompact) && defined(PyUnstable_Long_CompactValue)
+ #define __Pyx_PyLong_IsCompact(x) PyUnstable_Long_IsCompact((PyLongObject*) x)
+ #define __Pyx_PyLong_CompactValue(x) PyUnstable_Long_CompactValue((PyLongObject*) x)
+ #else
+ #define __Pyx_PyLong_IsCompact(x) (((PyLongObject*)x)->long_value.lv_tag < (2 << _PyLong_NON_SIZE_BITS))
+ #define __Pyx_PyLong_CompactValue(x) ((1 - (Py_ssize_t) __Pyx_PyLong_Sign(x)) * (Py_ssize_t) __Pyx_PyLong_Digits(x)[0])
+ #endif
+ typedef Py_ssize_t __Pyx_compact_pylong;
+ typedef size_t __Pyx_compact_upylong;
+ #else
+ #define __Pyx_PyLong_IsNeg(x) (Py_SIZE(x) < 0)
+ #define __Pyx_PyLong_IsNonNeg(x) (Py_SIZE(x) >= 0)
+ #define __Pyx_PyLong_IsZero(x) (Py_SIZE(x) == 0)
+ #define __Pyx_PyLong_IsPos(x) (Py_SIZE(x) > 0)
+ #define __Pyx_PyLong_CompactValueUnsigned(x) ((Py_SIZE(x) == 0) ? 0 : __Pyx_PyLong_Digits(x)[0])
+ #define __Pyx_PyLong_DigitCount(x) __Pyx_sst_abs(Py_SIZE(x))
+ #define __Pyx_PyLong_SignedDigitCount(x) Py_SIZE(x)
+ #define __Pyx_PyLong_IsCompact(x) (Py_SIZE(x) == 0 || Py_SIZE(x) == 1 || Py_SIZE(x) == -1)
+ #define __Pyx_PyLong_CompactValue(x)\
+ ((Py_SIZE(x) == 0) ? (sdigit) 0 : ((Py_SIZE(x) < 0) ? -(sdigit)__Pyx_PyLong_Digits(x)[0] : (sdigit)__Pyx_PyLong_Digits(x)[0]))
+ typedef sdigit __Pyx_compact_pylong;
+ typedef digit __Pyx_compact_upylong;
+ #endif
+ #if PY_VERSION_HEX >= 0x030C00A5
+ #define __Pyx_PyLong_Digits(x) (((PyLongObject*)x)->long_value.ob_digit)
+ #else
+ #define __Pyx_PyLong_Digits(x) (((PyLongObject*)x)->ob_digit)
+ #endif
+#endif
+#if __PYX_DEFAULT_STRING_ENCODING_IS_UTF8
+ #define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_DecodeUTF8(c_str, size, NULL)
+#elif __PYX_DEFAULT_STRING_ENCODING_IS_ASCII
+ #define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_DecodeASCII(c_str, size, NULL)
+#else
+ #define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_Decode(c_str, size, __PYX_DEFAULT_STRING_ENCODING, NULL)
+#endif
+
+
+/* Test for GCC > 2.95 */
+#if defined(__GNUC__) && (__GNUC__ > 2 || (__GNUC__ == 2 && (__GNUC_MINOR__ > 95)))
+ #define likely(x) __builtin_expect(!!(x), 1)
+ #define unlikely(x) __builtin_expect(!!(x), 0)
+#else /* !__GNUC__ or GCC < 2.95 */
+ #define likely(x) (x)
+ #define unlikely(x) (x)
+#endif /* __GNUC__ */
+/* PretendToInitialize */
+#ifdef __cplusplus
+#if __cplusplus > 201103L
+#include
+#endif
+template
+static void __Pyx_pretend_to_initialize(T* ptr) {
+#if __cplusplus > 201103L
+ if ((std::is_trivially_default_constructible::value))
+#endif
+ *ptr = T();
+ (void)ptr;
+}
+#else
+static CYTHON_INLINE void __Pyx_pretend_to_initialize(void* ptr) { (void)ptr; }
+#endif
+
+
+#if !CYTHON_USE_MODULE_STATE
+static PyObject *__pyx_m = NULL;
+#endif
+static int __pyx_lineno;
+static int __pyx_clineno = 0;
+static const char * const __pyx_cfilenm = __FILE__;
+static const char *__pyx_filename;
+
+/* Header.proto */
+#if !defined(CYTHON_CCOMPLEX)
+ #if defined(__cplusplus)
+ #define CYTHON_CCOMPLEX 1
+ #elif (defined(_Complex_I) && !defined(_MSC_VER)) || ((defined (__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) && !defined(__STDC_NO_COMPLEX__) && !defined(_MSC_VER))
+ #define CYTHON_CCOMPLEX 1
+ #else
+ #define CYTHON_CCOMPLEX 0
+ #endif
+#endif
+#if CYTHON_CCOMPLEX
+ #ifdef __cplusplus
+ #include
+ #else
+ #include
+ #endif
+#endif
+#if CYTHON_CCOMPLEX && !defined(__cplusplus) && defined(__sun__) && defined(__GNUC__)
+ #undef _Complex_I
+ #define _Complex_I 1.0fj
+#endif
+
+/* #### Code section: filename_table ### */
+
+static const char* const __pyx_f[] = {
+ "Lib/fontTools/cu2qu/cu2qu.py",
+};
+/* #### Code section: utility_code_proto_before_types ### */
+/* Atomics.proto */
+#include
+#ifndef CYTHON_ATOMICS
+ #define CYTHON_ATOMICS 1
+#endif
+#define __PYX_CYTHON_ATOMICS_ENABLED() CYTHON_ATOMICS
+#define __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING() CYTHON_COMPILING_IN_CPYTHON_FREETHREADING
+#define __pyx_atomic_int_type int
+#define __pyx_nonatomic_int_type int
+#if CYTHON_ATOMICS && (defined(__STDC_VERSION__) &&\
+ (__STDC_VERSION__ >= 201112L) &&\
+ !defined(__STDC_NO_ATOMICS__))
+ #include
+#elif CYTHON_ATOMICS && (defined(__cplusplus) && (\
+ (__cplusplus >= 201103L) ||\
+ (defined(_MSC_VER) && _MSC_VER >= 1700)))
+ #include
+#endif
+#if CYTHON_ATOMICS && (defined(__STDC_VERSION__) &&\
+ (__STDC_VERSION__ >= 201112L) &&\
+ !defined(__STDC_NO_ATOMICS__) &&\
+ ATOMIC_INT_LOCK_FREE == 2)
+ #undef __pyx_atomic_int_type
+ #define __pyx_atomic_int_type atomic_int
+ #define __pyx_atomic_ptr_type atomic_uintptr_t
+ #define __pyx_nonatomic_ptr_type uintptr_t
+ #define __pyx_atomic_incr_relaxed(value) atomic_fetch_add_explicit(value, 1, memory_order_relaxed)
+ #define __pyx_atomic_incr_acq_rel(value) atomic_fetch_add_explicit(value, 1, memory_order_acq_rel)
+ #define __pyx_atomic_decr_acq_rel(value) atomic_fetch_sub_explicit(value, 1, memory_order_acq_rel)
+ #define __pyx_atomic_sub(value, arg) atomic_fetch_sub(value, arg)
+ #define __pyx_atomic_int_cmp_exchange(value, expected, desired) atomic_compare_exchange_strong(value, expected, desired)
+ #define __pyx_atomic_load(value) atomic_load(value)
+ #define __pyx_atomic_store(value, new_value) atomic_store(value, new_value)
+ #define __pyx_atomic_pointer_load_relaxed(value) atomic_load_explicit(value, memory_order_relaxed)
+ #define __pyx_atomic_pointer_load_acquire(value) atomic_load_explicit(value, memory_order_acquire)
+ #define __pyx_atomic_pointer_exchange(value, new_value) atomic_exchange(value, (__pyx_nonatomic_ptr_type)new_value)
+ #if defined(__PYX_DEBUG_ATOMICS) && defined(_MSC_VER)
+ #pragma message ("Using standard C atomics")
+ #elif defined(__PYX_DEBUG_ATOMICS)
+ #warning "Using standard C atomics"
+ #endif
+#elif CYTHON_ATOMICS && (defined(__cplusplus) && (\
+ (__cplusplus >= 201103L) ||\
+\
+ (defined(_MSC_VER) && _MSC_VER >= 1700)) &&\
+ ATOMIC_INT_LOCK_FREE == 2)
+ #undef __pyx_atomic_int_type
+ #define __pyx_atomic_int_type std::atomic_int
+ #define __pyx_atomic_ptr_type std::atomic_uintptr_t
+ #define __pyx_nonatomic_ptr_type uintptr_t
+ #define __pyx_atomic_incr_relaxed(value) std::atomic_fetch_add_explicit(value, 1, std::memory_order_relaxed)
+ #define __pyx_atomic_incr_acq_rel(value) std::atomic_fetch_add_explicit(value, 1, std::memory_order_acq_rel)
+ #define __pyx_atomic_decr_acq_rel(value) std::atomic_fetch_sub_explicit(value, 1, std::memory_order_acq_rel)
+ #define __pyx_atomic_sub(value, arg) std::atomic_fetch_sub(value, arg)
+ #define __pyx_atomic_int_cmp_exchange(value, expected, desired) std::atomic_compare_exchange_strong(value, expected, desired)
+ #define __pyx_atomic_load(value) std::atomic_load(value)
+ #define __pyx_atomic_store(value, new_value) std::atomic_store(value, new_value)
+ #define __pyx_atomic_pointer_load_relaxed(value) std::atomic_load_explicit(value, std::memory_order_relaxed)
+ #define __pyx_atomic_pointer_load_acquire(value) std::atomic_load_explicit(value, std::memory_order_acquire)
+ #define __pyx_atomic_pointer_exchange(value, new_value) std::atomic_exchange(value, (__pyx_nonatomic_ptr_type)new_value)
+ #if defined(__PYX_DEBUG_ATOMICS) && defined(_MSC_VER)
+ #pragma message ("Using standard C++ atomics")
+ #elif defined(__PYX_DEBUG_ATOMICS)
+ #warning "Using standard C++ atomics"
+ #endif
+#elif CYTHON_ATOMICS && (__GNUC__ >= 5 || (__GNUC__ == 4 &&\
+ (__GNUC_MINOR__ > 1 ||\
+ (__GNUC_MINOR__ == 1 && __GNUC_PATCHLEVEL__ >= 2))))
+ #define __pyx_atomic_ptr_type void*
+ #define __pyx_atomic_incr_relaxed(value) __sync_fetch_and_add(value, 1)
+ #define __pyx_atomic_incr_acq_rel(value) __sync_fetch_and_add(value, 1)
+ #define __pyx_atomic_decr_acq_rel(value) __sync_fetch_and_sub(value, 1)
+ #define __pyx_atomic_sub(value, arg) __sync_fetch_and_sub(value, arg)
+ static CYTHON_INLINE int __pyx_atomic_int_cmp_exchange(__pyx_atomic_int_type* value, __pyx_nonatomic_int_type* expected, __pyx_nonatomic_int_type desired) {
+ __pyx_nonatomic_int_type old = __sync_val_compare_and_swap(value, *expected, desired);
+ int result = old == *expected;
+ *expected = old;
+ return result;
+ }
+ #define __pyx_atomic_load(value) __sync_fetch_and_add(value, 0)
+ #define __pyx_atomic_store(value, new_value) __sync_lock_test_and_set(value, new_value)
+ #define __pyx_atomic_pointer_load_relaxed(value) __sync_fetch_and_add(value, 0)
+ #define __pyx_atomic_pointer_load_acquire(value) __sync_fetch_and_add(value, 0)
+ #define __pyx_atomic_pointer_exchange(value, new_value) __sync_lock_test_and_set(value, (__pyx_atomic_ptr_type)new_value)
+ #ifdef __PYX_DEBUG_ATOMICS
+ #warning "Using GNU atomics"
+ #endif
+#elif CYTHON_ATOMICS && defined(_MSC_VER)
+ #include
+ #undef __pyx_atomic_int_type
+ #define __pyx_atomic_int_type long
+ #define __pyx_atomic_ptr_type void*
+ #undef __pyx_nonatomic_int_type
+ #define __pyx_nonatomic_int_type long
+ #pragma intrinsic (_InterlockedExchangeAdd, _InterlockedExchange, _InterlockedCompareExchange, _InterlockedCompareExchangePointer, _InterlockedExchangePointer)
+ #define __pyx_atomic_incr_relaxed(value) _InterlockedExchangeAdd(value, 1)
+ #define __pyx_atomic_incr_acq_rel(value) _InterlockedExchangeAdd(value, 1)
+ #define __pyx_atomic_decr_acq_rel(value) _InterlockedExchangeAdd(value, -1)
+ #define __pyx_atomic_sub(value, arg) _InterlockedExchangeAdd(value, -arg)
+ static CYTHON_INLINE int __pyx_atomic_int_cmp_exchange(__pyx_atomic_int_type* value, __pyx_nonatomic_int_type* expected, __pyx_nonatomic_int_type desired) {
+ __pyx_nonatomic_int_type old = _InterlockedCompareExchange(value, desired, *expected);
+ int result = old == *expected;
+ *expected = old;
+ return result;
+ }
+ #define __pyx_atomic_load(value) _InterlockedExchangeAdd(value, 0)
+ #define __pyx_atomic_store(value, new_value) _InterlockedExchange(value, new_value)
+ #define __pyx_atomic_pointer_load_relaxed(value) *(void * volatile *)value
+ #define __pyx_atomic_pointer_load_acquire(value) _InterlockedCompareExchangePointer(value, 0, 0)
+ #define __pyx_atomic_pointer_exchange(value, new_value) _InterlockedExchangePointer(value, (__pyx_atomic_ptr_type)new_value)
+ #ifdef __PYX_DEBUG_ATOMICS
+ #pragma message ("Using MSVC atomics")
+ #endif
+#else
+ #undef CYTHON_ATOMICS
+ #define CYTHON_ATOMICS 0
+ #ifdef __PYX_DEBUG_ATOMICS
+ #warning "Not using atomics"
+ #endif
+#endif
+#if CYTHON_ATOMICS
+ #define __pyx_add_acquisition_count(memview)\
+ __pyx_atomic_incr_relaxed(__pyx_get_slice_count_pointer(memview))
+ #define __pyx_sub_acquisition_count(memview)\
+ __pyx_atomic_decr_acq_rel(__pyx_get_slice_count_pointer(memview))
+#else
+ #define __pyx_add_acquisition_count(memview)\
+ __pyx_add_acquisition_count_locked(__pyx_get_slice_count_pointer(memview), memview->lock)
+ #define __pyx_sub_acquisition_count(memview)\
+ __pyx_sub_acquisition_count_locked(__pyx_get_slice_count_pointer(memview), memview->lock)
+#endif
+
+/* IncludeStructmemberH.proto */
+#include
+
+/* CriticalSections.proto */
+#if !CYTHON_COMPILING_IN_CPYTHON_FREETHREADING
+#define __Pyx_PyCriticalSection void*
+#define __Pyx_PyCriticalSection2 void*
+#define __Pyx_PyCriticalSection_Begin1(cs, arg) (void)cs
+#define __Pyx_PyCriticalSection_Begin2(cs, arg1, arg2) (void)cs
+#define __Pyx_PyCriticalSection_End1(cs)
+#define __Pyx_PyCriticalSection_End2(cs)
+#else
+#define __Pyx_PyCriticalSection PyCriticalSection
+#define __Pyx_PyCriticalSection2 PyCriticalSection2
+#define __Pyx_PyCriticalSection_Begin1 PyCriticalSection_Begin
+#define __Pyx_PyCriticalSection_Begin2 PyCriticalSection2_Begin
+#define __Pyx_PyCriticalSection_End1 PyCriticalSection_End
+#define __Pyx_PyCriticalSection_End2 PyCriticalSection2_End
+#endif
+#if PY_VERSION_HEX < 0x030d0000 || CYTHON_COMPILING_IN_LIMITED_API
+#define __Pyx_BEGIN_CRITICAL_SECTION(o) {
+#define __Pyx_END_CRITICAL_SECTION() }
+#else
+#define __Pyx_BEGIN_CRITICAL_SECTION Py_BEGIN_CRITICAL_SECTION
+#define __Pyx_END_CRITICAL_SECTION Py_END_CRITICAL_SECTION
+#endif
+
+/* #### Code section: numeric_typedefs ### */
+/* #### Code section: complex_type_declarations ### */
+/* Declarations.proto */
+#if CYTHON_CCOMPLEX && (1) && (!0 || __cplusplus)
+ #ifdef __cplusplus
+ typedef ::std::complex< double > __pyx_t_double_complex;
+ #else
+ typedef double _Complex __pyx_t_double_complex;
+ #endif
+#else
+ typedef struct { double real, imag; } __pyx_t_double_complex;
+#endif
+static CYTHON_INLINE __pyx_t_double_complex __pyx_t_double_complex_from_parts(double, double);
+
+/* #### Code section: type_declarations ### */
+
+/*--- Type declarations ---*/
+struct __pyx_obj_9fontTools_5cu2qu_5cu2qu___pyx_scope_struct___split_cubic_into_n_gen;
+
+/* "fontTools/cu2qu/cu2qu.py":150
+ *
+ *
+ * @cython.locals( # <<<<<<<<<<<<<<
+ * p0=cython.complex,
+ * p1=cython.complex,
+*/
+struct __pyx_obj_9fontTools_5cu2qu_5cu2qu___pyx_scope_struct___split_cubic_into_n_gen {
+ PyObject_HEAD
+ __pyx_t_double_complex __pyx_v_a;
+ __pyx_t_double_complex __pyx_v_a1;
+ __pyx_t_double_complex __pyx_v_b;
+ __pyx_t_double_complex __pyx_v_b1;
+ __pyx_t_double_complex __pyx_v_c;
+ __pyx_t_double_complex __pyx_v_c1;
+ __pyx_t_double_complex __pyx_v_d;
+ __pyx_t_double_complex __pyx_v_d1;
+ double __pyx_v_delta_2;
+ double __pyx_v_delta_3;
+ double __pyx_v_dt;
+ int __pyx_v_i;
+ int __pyx_v_n;
+ __pyx_t_double_complex __pyx_v_p0;
+ __pyx_t_double_complex __pyx_v_p1;
+ __pyx_t_double_complex __pyx_v_p2;
+ __pyx_t_double_complex __pyx_v_p3;
+ double __pyx_v_t1;
+ double __pyx_v_t1_2;
+ int __pyx_t_0;
+ int __pyx_t_1;
+ int __pyx_t_2;
+};
+
+/* #### Code section: utility_code_proto ### */
+
+/* --- Runtime support code (head) --- */
+/* Refnanny.proto */
+#ifndef CYTHON_REFNANNY
+ #define CYTHON_REFNANNY 0
+#endif
+#if CYTHON_REFNANNY
+ typedef struct {
+ void (*INCREF)(void*, PyObject*, Py_ssize_t);
+ void (*DECREF)(void*, PyObject*, Py_ssize_t);
+ void (*GOTREF)(void*, PyObject*, Py_ssize_t);
+ void (*GIVEREF)(void*, PyObject*, Py_ssize_t);
+ void* (*SetupContext)(const char*, Py_ssize_t, const char*);
+ void (*FinishContext)(void**);
+ } __Pyx_RefNannyAPIStruct;
+ static __Pyx_RefNannyAPIStruct *__Pyx_RefNanny = NULL;
+ static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname);
+ #define __Pyx_RefNannyDeclarations void *__pyx_refnanny = NULL;
+ #define __Pyx_RefNannySetupContext(name, acquire_gil)\
+ if (acquire_gil) {\
+ PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\
+ __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), (__LINE__), (__FILE__));\
+ PyGILState_Release(__pyx_gilstate_save);\
+ } else {\
+ __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), (__LINE__), (__FILE__));\
+ }
+ #define __Pyx_RefNannyFinishContextNogil() {\
+ PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\
+ __Pyx_RefNannyFinishContext();\
+ PyGILState_Release(__pyx_gilstate_save);\
+ }
+ #define __Pyx_RefNannyFinishContextNogil() {\
+ PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\
+ __Pyx_RefNannyFinishContext();\
+ PyGILState_Release(__pyx_gilstate_save);\
+ }
+ #define __Pyx_RefNannyFinishContext()\
+ __Pyx_RefNanny->FinishContext(&__pyx_refnanny)
+ #define __Pyx_INCREF(r) __Pyx_RefNanny->INCREF(__pyx_refnanny, (PyObject *)(r), (__LINE__))
+ #define __Pyx_DECREF(r) __Pyx_RefNanny->DECREF(__pyx_refnanny, (PyObject *)(r), (__LINE__))
+ #define __Pyx_GOTREF(r) __Pyx_RefNanny->GOTREF(__pyx_refnanny, (PyObject *)(r), (__LINE__))
+ #define __Pyx_GIVEREF(r) __Pyx_RefNanny->GIVEREF(__pyx_refnanny, (PyObject *)(r), (__LINE__))
+ #define __Pyx_XINCREF(r) do { if((r) == NULL); else {__Pyx_INCREF(r); }} while(0)
+ #define __Pyx_XDECREF(r) do { if((r) == NULL); else {__Pyx_DECREF(r); }} while(0)
+ #define __Pyx_XGOTREF(r) do { if((r) == NULL); else {__Pyx_GOTREF(r); }} while(0)
+ #define __Pyx_XGIVEREF(r) do { if((r) == NULL); else {__Pyx_GIVEREF(r);}} while(0)
+#else
+ #define __Pyx_RefNannyDeclarations
+ #define __Pyx_RefNannySetupContext(name, acquire_gil)
+ #define __Pyx_RefNannyFinishContextNogil()
+ #define __Pyx_RefNannyFinishContext()
+ #define __Pyx_INCREF(r) Py_INCREF(r)
+ #define __Pyx_DECREF(r) Py_DECREF(r)
+ #define __Pyx_GOTREF(r)
+ #define __Pyx_GIVEREF(r)
+ #define __Pyx_XINCREF(r) Py_XINCREF(r)
+ #define __Pyx_XDECREF(r) Py_XDECREF(r)
+ #define __Pyx_XGOTREF(r)
+ #define __Pyx_XGIVEREF(r)
+#endif
+#define __Pyx_Py_XDECREF_SET(r, v) do {\
+ PyObject *tmp = (PyObject *) r;\
+ r = v; Py_XDECREF(tmp);\
+ } while (0)
+#define __Pyx_XDECREF_SET(r, v) do {\
+ PyObject *tmp = (PyObject *) r;\
+ r = v; __Pyx_XDECREF(tmp);\
+ } while (0)
+#define __Pyx_DECREF_SET(r, v) do {\
+ PyObject *tmp = (PyObject *) r;\
+ r = v; __Pyx_DECREF(tmp);\
+ } while (0)
+#define __Pyx_CLEAR(r) do { PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);} while(0)
+#define __Pyx_XCLEAR(r) do { if((r) != NULL) {PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);}} while(0)
+
+/* PyErrExceptionMatches.proto */
+#if CYTHON_FAST_THREAD_STATE
+#define __Pyx_PyErr_ExceptionMatches(err) __Pyx_PyErr_ExceptionMatchesInState(__pyx_tstate, err)
+static CYTHON_INLINE int __Pyx_PyErr_ExceptionMatchesInState(PyThreadState* tstate, PyObject* err);
+#else
+#define __Pyx_PyErr_ExceptionMatches(err) PyErr_ExceptionMatches(err)
+#endif
+
+/* PyThreadStateGet.proto */
+#if CYTHON_FAST_THREAD_STATE
+#define __Pyx_PyThreadState_declare PyThreadState *__pyx_tstate;
+#define __Pyx_PyThreadState_assign __pyx_tstate = __Pyx_PyThreadState_Current;
+#if PY_VERSION_HEX >= 0x030C00A6
+#define __Pyx_PyErr_Occurred() (__pyx_tstate->current_exception != NULL)
+#define __Pyx_PyErr_CurrentExceptionType() (__pyx_tstate->current_exception ? (PyObject*) Py_TYPE(__pyx_tstate->current_exception) : (PyObject*) NULL)
+#else
+#define __Pyx_PyErr_Occurred() (__pyx_tstate->curexc_type != NULL)
+#define __Pyx_PyErr_CurrentExceptionType() (__pyx_tstate->curexc_type)
+#endif
+#else
+#define __Pyx_PyThreadState_declare
+#define __Pyx_PyThreadState_assign
+#define __Pyx_PyErr_Occurred() (PyErr_Occurred() != NULL)
+#define __Pyx_PyErr_CurrentExceptionType() PyErr_Occurred()
+#endif
+
+/* PyErrFetchRestore.proto */
+#if CYTHON_FAST_THREAD_STATE
+#define __Pyx_PyErr_Clear() __Pyx_ErrRestore(NULL, NULL, NULL)
+#define __Pyx_ErrRestoreWithState(type, value, tb) __Pyx_ErrRestoreInState(PyThreadState_GET(), type, value, tb)
+#define __Pyx_ErrFetchWithState(type, value, tb) __Pyx_ErrFetchInState(PyThreadState_GET(), type, value, tb)
+#define __Pyx_ErrRestore(type, value, tb) __Pyx_ErrRestoreInState(__pyx_tstate, type, value, tb)
+#define __Pyx_ErrFetch(type, value, tb) __Pyx_ErrFetchInState(__pyx_tstate, type, value, tb)
+static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb);
+static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb);
+#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A6
+#define __Pyx_PyErr_SetNone(exc) (Py_INCREF(exc), __Pyx_ErrRestore((exc), NULL, NULL))
+#else
+#define __Pyx_PyErr_SetNone(exc) PyErr_SetNone(exc)
+#endif
+#else
+#define __Pyx_PyErr_Clear() PyErr_Clear()
+#define __Pyx_PyErr_SetNone(exc) PyErr_SetNone(exc)
+#define __Pyx_ErrRestoreWithState(type, value, tb) PyErr_Restore(type, value, tb)
+#define __Pyx_ErrFetchWithState(type, value, tb) PyErr_Fetch(type, value, tb)
+#define __Pyx_ErrRestoreInState(tstate, type, value, tb) PyErr_Restore(type, value, tb)
+#define __Pyx_ErrFetchInState(tstate, type, value, tb) PyErr_Fetch(type, value, tb)
+#define __Pyx_ErrRestore(type, value, tb) PyErr_Restore(type, value, tb)
+#define __Pyx_ErrFetch(type, value, tb) PyErr_Fetch(type, value, tb)
+#endif
+
+/* PyObjectGetAttrStr.proto */
+#if CYTHON_USE_TYPE_SLOTS
+static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject* attr_name);
+#else
+#define __Pyx_PyObject_GetAttrStr(o,n) PyObject_GetAttr(o,n)
+#endif
+
+/* PyObjectGetAttrStrNoError.proto */
+static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStrNoError(PyObject* obj, PyObject* attr_name);
+
+/* GetBuiltinName.proto */
+static PyObject *__Pyx_GetBuiltinName(PyObject *name);
+
+/* IncludeStdlibH.proto */
+#include
+
+/* PyFunctionFastCall.proto */
+#if CYTHON_FAST_PYCALL
+#if !CYTHON_VECTORCALL
+#define __Pyx_PyFunction_FastCall(func, args, nargs)\
+ __Pyx_PyFunction_FastCallDict((func), (args), (nargs), NULL)
+static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject *const *args, Py_ssize_t nargs, PyObject *kwargs);
+#endif
+#define __Pyx_BUILD_ASSERT_EXPR(cond)\
+ (sizeof(char [1 - 2*!(cond)]) - 1)
+#ifndef Py_MEMBER_SIZE
+#define Py_MEMBER_SIZE(type, member) sizeof(((type *)0)->member)
+#endif
+#if !CYTHON_VECTORCALL
+#if PY_VERSION_HEX >= 0x03080000
+ #include "frameobject.h"
+ #define __Pxy_PyFrame_Initialize_Offsets()
+ #define __Pyx_PyFrame_GetLocalsplus(frame) ((frame)->f_localsplus)
+#else
+ static size_t __pyx_pyframe_localsplus_offset = 0;
+ #include "frameobject.h"
+ #define __Pxy_PyFrame_Initialize_Offsets()\
+ ((void)__Pyx_BUILD_ASSERT_EXPR(sizeof(PyFrameObject) == offsetof(PyFrameObject, f_localsplus) + Py_MEMBER_SIZE(PyFrameObject, f_localsplus)),\
+ (void)(__pyx_pyframe_localsplus_offset = ((size_t)PyFrame_Type.tp_basicsize) - Py_MEMBER_SIZE(PyFrameObject, f_localsplus)))
+ #define __Pyx_PyFrame_GetLocalsplus(frame)\
+ (assert(__pyx_pyframe_localsplus_offset), (PyObject **)(((char *)(frame)) + __pyx_pyframe_localsplus_offset))
+#endif
+#endif
+#endif
+
+/* PyObjectCall.proto */
+#if CYTHON_COMPILING_IN_CPYTHON
+static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw);
+#else
+#define __Pyx_PyObject_Call(func, arg, kw) PyObject_Call(func, arg, kw)
+#endif
+
+/* PyObjectCallMethO.proto */
+#if CYTHON_COMPILING_IN_CPYTHON
+static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg);
+#endif
+
+/* PyObjectFastCall.proto */
+#define __Pyx_PyObject_FastCall(func, args, nargs) __Pyx_PyObject_FastCallDict(func, args, (size_t)(nargs), NULL)
+static CYTHON_INLINE PyObject* __Pyx_PyObject_FastCallDict(PyObject *func, PyObject * const*args, size_t nargs, PyObject *kwargs);
+
+/* PyLongCompare.proto */
+static CYTHON_INLINE int __Pyx_PyLong_BoolEqObjC(PyObject *op1, PyObject *op2, long intval, long inplace);
+
+/* RaiseTooManyValuesToUnpack.proto */
+static CYTHON_INLINE void __Pyx_RaiseTooManyValuesError(Py_ssize_t expected);
+
+/* RaiseNeedMoreValuesToUnpack.proto */
+static CYTHON_INLINE void __Pyx_RaiseNeedMoreValuesError(Py_ssize_t index);
+
+/* IterFinish.proto */
+static CYTHON_INLINE int __Pyx_IterFinish(void);
+
+/* UnpackItemEndCheck.proto */
+static int __Pyx_IternextUnpackEndCheck(PyObject *retval, Py_ssize_t expected);
+
+/* GetItemInt.proto */
+#define __Pyx_GetItemInt(o, i, type, is_signed, to_py_func, is_list, wraparound, boundscheck, has_gil)\
+ (__Pyx_fits_Py_ssize_t(i, type, is_signed) ?\
+ __Pyx_GetItemInt_Fast(o, (Py_ssize_t)i, is_list, wraparound, boundscheck) :\
+ (is_list ? (PyErr_SetString(PyExc_IndexError, "list index out of range"), (PyObject*)NULL) :\
+ __Pyx_GetItemInt_Generic(o, to_py_func(i))))
+#define __Pyx_GetItemInt_List(o, i, type, is_signed, to_py_func, is_list, wraparound, boundscheck, has_gil)\
+ (__Pyx_fits_Py_ssize_t(i, type, is_signed) ?\
+ __Pyx_GetItemInt_List_Fast(o, (Py_ssize_t)i, wraparound, boundscheck) :\
+ (PyErr_SetString(PyExc_IndexError, "list index out of range"), (PyObject*)NULL))
+static CYTHON_INLINE PyObject *__Pyx_GetItemInt_List_Fast(PyObject *o, Py_ssize_t i,
+ int wraparound, int boundscheck);
+#define __Pyx_GetItemInt_Tuple(o, i, type, is_signed, to_py_func, is_list, wraparound, boundscheck, has_gil)\
+ (__Pyx_fits_Py_ssize_t(i, type, is_signed) ?\
+ __Pyx_GetItemInt_Tuple_Fast(o, (Py_ssize_t)i, wraparound, boundscheck) :\
+ (PyErr_SetString(PyExc_IndexError, "tuple index out of range"), (PyObject*)NULL))
+static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Tuple_Fast(PyObject *o, Py_ssize_t i,
+ int wraparound, int boundscheck);
+static PyObject *__Pyx_GetItemInt_Generic(PyObject *o, PyObject* j);
+static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Fast(PyObject *o, Py_ssize_t i,
+ int is_list, int wraparound, int boundscheck);
+
+/* PyDictVersioning.proto */
+#if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_TYPE_SLOTS
+#define __PYX_DICT_VERSION_INIT ((PY_UINT64_T) -1)
+#define __PYX_GET_DICT_VERSION(dict) (((PyDictObject*)(dict))->ma_version_tag)
+#define __PYX_UPDATE_DICT_CACHE(dict, value, cache_var, version_var)\
+ (version_var) = __PYX_GET_DICT_VERSION(dict);\
+ (cache_var) = (value);
+#define __PYX_PY_DICT_LOOKUP_IF_MODIFIED(VAR, DICT, LOOKUP) {\
+ static PY_UINT64_T __pyx_dict_version = 0;\
+ static PyObject *__pyx_dict_cached_value = NULL;\
+ if (likely(__PYX_GET_DICT_VERSION(DICT) == __pyx_dict_version)) {\
+ (VAR) = __pyx_dict_cached_value;\
+ } else {\
+ (VAR) = __pyx_dict_cached_value = (LOOKUP);\
+ __pyx_dict_version = __PYX_GET_DICT_VERSION(DICT);\
+ }\
+}
+static CYTHON_INLINE PY_UINT64_T __Pyx_get_tp_dict_version(PyObject *obj);
+static CYTHON_INLINE PY_UINT64_T __Pyx_get_object_dict_version(PyObject *obj);
+static CYTHON_INLINE int __Pyx_object_dict_version_matches(PyObject* obj, PY_UINT64_T tp_dict_version, PY_UINT64_T obj_dict_version);
+#else
+#define __PYX_GET_DICT_VERSION(dict) (0)
+#define __PYX_UPDATE_DICT_CACHE(dict, value, cache_var, version_var)
+#define __PYX_PY_DICT_LOOKUP_IF_MODIFIED(VAR, DICT, LOOKUP) (VAR) = (LOOKUP);
+#endif
+
+/* GetModuleGlobalName.proto */
+#if CYTHON_USE_DICT_VERSIONS
+#define __Pyx_GetModuleGlobalName(var, name) do {\
+ static PY_UINT64_T __pyx_dict_version = 0;\
+ static PyObject *__pyx_dict_cached_value = NULL;\
+ (var) = (likely(__pyx_dict_version == __PYX_GET_DICT_VERSION(__pyx_mstate_global->__pyx_d))) ?\
+ (likely(__pyx_dict_cached_value) ? __Pyx_NewRef(__pyx_dict_cached_value) : __Pyx_GetBuiltinName(name)) :\
+ __Pyx__GetModuleGlobalName(name, &__pyx_dict_version, &__pyx_dict_cached_value);\
+} while(0)
+#define __Pyx_GetModuleGlobalNameUncached(var, name) do {\
+ PY_UINT64_T __pyx_dict_version;\
+ PyObject *__pyx_dict_cached_value;\
+ (var) = __Pyx__GetModuleGlobalName(name, &__pyx_dict_version, &__pyx_dict_cached_value);\
+} while(0)
+static PyObject *__Pyx__GetModuleGlobalName(PyObject *name, PY_UINT64_T *dict_version, PyObject **dict_cached_value);
+#else
+#define __Pyx_GetModuleGlobalName(var, name) (var) = __Pyx__GetModuleGlobalName(name)
+#define __Pyx_GetModuleGlobalNameUncached(var, name) (var) = __Pyx__GetModuleGlobalName(name)
+static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name);
+#endif
+
+/* TupleAndListFromArray.proto */
+#if CYTHON_COMPILING_IN_CPYTHON
+static CYTHON_INLINE PyObject* __Pyx_PyList_FromArray(PyObject *const *src, Py_ssize_t n);
+#endif
+#if CYTHON_COMPILING_IN_CPYTHON || CYTHON_METH_FASTCALL
+static CYTHON_INLINE PyObject* __Pyx_PyTuple_FromArray(PyObject *const *src, Py_ssize_t n);
+#endif
+
+/* IncludeStringH.proto */
+#include
+
+/* BytesEquals.proto */
+static CYTHON_INLINE int __Pyx_PyBytes_Equals(PyObject* s1, PyObject* s2, int equals);
+
+/* UnicodeEquals.proto */
+static CYTHON_INLINE int __Pyx_PyUnicode_Equals(PyObject* s1, PyObject* s2, int equals);
+
+/* fastcall.proto */
+#if CYTHON_AVOID_BORROWED_REFS
+ #define __Pyx_ArgRef_VARARGS(args, i) __Pyx_PySequence_ITEM(args, i)
+#elif CYTHON_ASSUME_SAFE_MACROS
+ #define __Pyx_ArgRef_VARARGS(args, i) __Pyx_NewRef(__Pyx_PyTuple_GET_ITEM(args, i))
+#else
+ #define __Pyx_ArgRef_VARARGS(args, i) __Pyx_XNewRef(PyTuple_GetItem(args, i))
+#endif
+#define __Pyx_NumKwargs_VARARGS(kwds) PyDict_Size(kwds)
+#define __Pyx_KwValues_VARARGS(args, nargs) NULL
+#define __Pyx_GetKwValue_VARARGS(kw, kwvalues, s) __Pyx_PyDict_GetItemStrWithError(kw, s)
+#define __Pyx_KwargsAsDict_VARARGS(kw, kwvalues) PyDict_Copy(kw)
+#if CYTHON_METH_FASTCALL
+ #define __Pyx_ArgRef_FASTCALL(args, i) __Pyx_NewRef(args[i])
+ #define __Pyx_NumKwargs_FASTCALL(kwds) __Pyx_PyTuple_GET_SIZE(kwds)
+ #define __Pyx_KwValues_FASTCALL(args, nargs) ((args) + (nargs))
+ static CYTHON_INLINE PyObject * __Pyx_GetKwValue_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues, PyObject *s);
+ #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030d0000 || CYTHON_COMPILING_IN_LIMITED_API
+ CYTHON_UNUSED static PyObject *__Pyx_KwargsAsDict_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues);
+ #else
+ #define __Pyx_KwargsAsDict_FASTCALL(kw, kwvalues) _PyStack_AsDict(kwvalues, kw)
+ #endif
+#else
+ #define __Pyx_ArgRef_FASTCALL __Pyx_ArgRef_VARARGS
+ #define __Pyx_NumKwargs_FASTCALL __Pyx_NumKwargs_VARARGS
+ #define __Pyx_KwValues_FASTCALL __Pyx_KwValues_VARARGS
+ #define __Pyx_GetKwValue_FASTCALL __Pyx_GetKwValue_VARARGS
+ #define __Pyx_KwargsAsDict_FASTCALL __Pyx_KwargsAsDict_VARARGS
+#endif
+#define __Pyx_ArgsSlice_VARARGS(args, start, stop) PyTuple_GetSlice(args, start, stop)
+#if CYTHON_METH_FASTCALL || (CYTHON_COMPILING_IN_CPYTHON && CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS)
+#define __Pyx_ArgsSlice_FASTCALL(args, start, stop) __Pyx_PyTuple_FromArray(args + start, stop - start)
+#else
+#define __Pyx_ArgsSlice_FASTCALL(args, start, stop) PyTuple_GetSlice(args, start, stop)
+#endif
+
+/* RaiseDoubleKeywords.proto */
+static void __Pyx_RaiseDoubleKeywordsError(const char* func_name, PyObject* kw_name);
+
+/* ParseKeywords.proto */
+static CYTHON_INLINE int __Pyx_ParseKeywords(
+ PyObject *kwds, PyObject *const *kwvalues, PyObject ** const argnames[],
+ PyObject *kwds2, PyObject *values[],
+ Py_ssize_t num_pos_args, Py_ssize_t num_kwargs,
+ const char* function_name,
+ int ignore_unknown_kwargs
+);
+
+/* CallCFunction.proto */
+#define __Pyx_CallCFunction(cfunc, self, args)\
+ ((PyCFunction)(void(*)(void))(cfunc)->func)(self, args)
+#define __Pyx_CallCFunctionWithKeywords(cfunc, self, args, kwargs)\
+ ((PyCFunctionWithKeywords)(void(*)(void))(cfunc)->func)(self, args, kwargs)
+#define __Pyx_CallCFunctionFast(cfunc, self, args, nargs)\
+ ((__Pyx_PyCFunctionFast)(void(*)(void))(PyCFunction)(cfunc)->func)(self, args, nargs)
+#define __Pyx_CallCFunctionFastWithKeywords(cfunc, self, args, nargs, kwnames)\
+ ((__Pyx_PyCFunctionFastWithKeywords)(void(*)(void))(PyCFunction)(cfunc)->func)(self, args, nargs, kwnames)
+
+/* UnpackUnboundCMethod.proto */
+typedef struct {
+ PyObject *type;
+ PyObject **method_name;
+#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING && CYTHON_ATOMICS
+ __pyx_atomic_int_type initialized;
+#endif
+ PyCFunction func;
+ PyObject *method;
+ int flag;
+} __Pyx_CachedCFunction;
+#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING
+static CYTHON_INLINE int __Pyx_CachedCFunction_GetAndSetInitializing(__Pyx_CachedCFunction *cfunc) {
+#if !CYTHON_ATOMICS
+ return 1;
+#else
+ __pyx_nonatomic_int_type expected = 0;
+ if (__pyx_atomic_int_cmp_exchange(&cfunc->initialized, &expected, 1)) {
+ return 0;
+ }
+ return expected;
+#endif
+}
+static CYTHON_INLINE void __Pyx_CachedCFunction_SetFinishedInitializing(__Pyx_CachedCFunction *cfunc) {
+#if CYTHON_ATOMICS
+ __pyx_atomic_store(&cfunc->initialized, 2);
+#endif
+}
+#else
+#define __Pyx_CachedCFunction_GetAndSetInitializing(cfunc) 2
+#define __Pyx_CachedCFunction_SetFinishedInitializing(cfunc)
+#endif
+
+/* CallUnboundCMethod2.proto */
+CYTHON_UNUSED
+static PyObject* __Pyx__CallUnboundCMethod2(__Pyx_CachedCFunction* cfunc, PyObject* self, PyObject* arg1, PyObject* arg2);
+#if CYTHON_COMPILING_IN_CPYTHON
+static CYTHON_INLINE PyObject *__Pyx_CallUnboundCMethod2(__Pyx_CachedCFunction *cfunc, PyObject *self, PyObject *arg1, PyObject *arg2);
+#else
+#define __Pyx_CallUnboundCMethod2(cfunc, self, arg1, arg2) __Pyx__CallUnboundCMethod2(cfunc, self, arg1, arg2)
+#endif
+
+/* RaiseArgTupleInvalid.proto */
+static void __Pyx_RaiseArgtupleInvalid(const char* func_name, int exact,
+ Py_ssize_t num_min, Py_ssize_t num_max, Py_ssize_t num_found);
+
+/* GetException.proto */
+#if CYTHON_FAST_THREAD_STATE
+#define __Pyx_GetException(type, value, tb) __Pyx__GetException(__pyx_tstate, type, value, tb)
+static int __Pyx__GetException(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb);
+#else
+static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb);
+#endif
+
+/* pep479.proto */
+static void __Pyx_Generator_Replace_StopIteration(int in_async_gen);
+
+/* GetTopmostException.proto */
+#if CYTHON_USE_EXC_INFO_STACK && CYTHON_FAST_THREAD_STATE
+static _PyErr_StackItem * __Pyx_PyErr_GetTopmostException(PyThreadState *tstate);
+#endif
+
+/* SaveResetException.proto */
+#if CYTHON_FAST_THREAD_STATE
+#define __Pyx_ExceptionSave(type, value, tb) __Pyx__ExceptionSave(__pyx_tstate, type, value, tb)
+static CYTHON_INLINE void __Pyx__ExceptionSave(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb);
+#define __Pyx_ExceptionReset(type, value, tb) __Pyx__ExceptionReset(__pyx_tstate, type, value, tb)
+static CYTHON_INLINE void __Pyx__ExceptionReset(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb);
+#else
+#define __Pyx_ExceptionSave(type, value, tb) PyErr_GetExcInfo(type, value, tb)
+#define __Pyx_ExceptionReset(type, value, tb) PyErr_SetExcInfo(type, value, tb)
+#endif
+
+/* IterNextPlain.proto */
+static CYTHON_INLINE PyObject *__Pyx_PyIter_Next_Plain(PyObject *iterator);
+#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030A0000
+static PyObject *__Pyx_GetBuiltinNext_LimitedAPI(void);
+#endif
+
+/* IterNext.proto */
+#define __Pyx_PyIter_Next(obj) __Pyx_PyIter_Next2(obj, NULL)
+static CYTHON_INLINE PyObject *__Pyx_PyIter_Next2(PyObject *, PyObject *);
+
+/* ListAppend.proto */
+#if CYTHON_USE_PYLIST_INTERNALS && CYTHON_ASSUME_SAFE_MACROS
+static CYTHON_INLINE int __Pyx_PyList_Append(PyObject* list, PyObject* x) {
+ PyListObject* L = (PyListObject*) list;
+ Py_ssize_t len = Py_SIZE(list);
+ if (likely(L->allocated > len) & likely(len > (L->allocated >> 1))) {
+ Py_INCREF(x);
+ #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030d0000
+ L->ob_item[len] = x;
+ #else
+ PyList_SET_ITEM(list, len, x);
+ #endif
+ __Pyx_SET_SIZE(list, len + 1);
+ return 0;
+ }
+ return PyList_Append(list, x);
+}
+#else
+#define __Pyx_PyList_Append(L,x) PyList_Append(L,x)
+#endif
+
+/* ListCompAppend.proto */
+#if CYTHON_USE_PYLIST_INTERNALS && CYTHON_ASSUME_SAFE_MACROS
+static CYTHON_INLINE int __Pyx_ListComp_Append(PyObject* list, PyObject* x) {
+ PyListObject* L = (PyListObject*) list;
+ Py_ssize_t len = Py_SIZE(list);
+ if (likely(L->allocated > len)) {
+ Py_INCREF(x);
+ #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030d0000
+ L->ob_item[len] = x;
+ #else
+ PyList_SET_ITEM(list, len, x);
+ #endif
+ __Pyx_SET_SIZE(list, len + 1);
+ return 0;
+ }
+ return PyList_Append(list, x);
+}
+#else
+#define __Pyx_ListComp_Append(L,x) PyList_Append(L,x)
+#endif
+
+/* PyLongBinop.proto */
+#if !CYTHON_COMPILING_IN_PYPY
+static CYTHON_INLINE PyObject* __Pyx_PyLong_AddObjC(PyObject *op1, PyObject *op2, long intval, int inplace, int zerodivision_check);
+#else
+#define __Pyx_PyLong_AddObjC(op1, op2, intval, inplace, zerodivision_check)\
+ (inplace ? PyNumber_InPlaceAdd(op1, op2) : PyNumber_Add(op1, op2))
+#endif
+
+/* RaiseException.proto */
+static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause);
+
+/* AssertionsEnabled.proto */
+#if CYTHON_COMPILING_IN_LIMITED_API || (CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030C0000)
+ static int __pyx_assertions_enabled_flag;
+ #define __pyx_assertions_enabled() (__pyx_assertions_enabled_flag)
+ static int __Pyx_init_assertions_enabled(void) {
+ PyObject *builtins, *debug, *debug_str;
+ int flag;
+ builtins = PyEval_GetBuiltins();
+ if (!builtins) goto bad;
+ debug_str = PyUnicode_FromStringAndSize("__debug__", 9);
+ if (!debug_str) goto bad;
+ debug = PyObject_GetItem(builtins, debug_str);
+ Py_DECREF(debug_str);
+ if (!debug) goto bad;
+ flag = PyObject_IsTrue(debug);
+ Py_DECREF(debug);
+ if (flag == -1) goto bad;
+ __pyx_assertions_enabled_flag = flag;
+ return 0;
+ bad:
+ __pyx_assertions_enabled_flag = 1;
+ return -1;
+ }
+#else
+ #define __Pyx_init_assertions_enabled() (0)
+ #define __pyx_assertions_enabled() (!Py_OptimizeFlag)
+#endif
+
+/* SetItemInt.proto */
+#define __Pyx_SetItemInt(o, i, v, type, is_signed, to_py_func, is_list, wraparound, boundscheck, has_gil)\
+ (__Pyx_fits_Py_ssize_t(i, type, is_signed) ?\
+ __Pyx_SetItemInt_Fast(o, (Py_ssize_t)i, v, is_list, wraparound, boundscheck) :\
+ (is_list ? (PyErr_SetString(PyExc_IndexError, "list assignment index out of range"), -1) :\
+ __Pyx_SetItemInt_Generic(o, to_py_func(i), v)))
+static int __Pyx_SetItemInt_Generic(PyObject *o, PyObject *j, PyObject *v);
+static CYTHON_INLINE int __Pyx_SetItemInt_Fast(PyObject *o, Py_ssize_t i, PyObject *v,
+ int is_list, int wraparound, int boundscheck);
+
+/* ModInt[long].proto */
+static CYTHON_INLINE long __Pyx_mod_long(long, long, int b_is_constant);
+
+/* LimitedApiGetTypeDict.proto */
+#if CYTHON_COMPILING_IN_LIMITED_API
+static PyObject *__Pyx_GetTypeDict(PyTypeObject *tp);
+#endif
+
+/* SetItemOnTypeDict.proto */
+static int __Pyx__SetItemOnTypeDict(PyTypeObject *tp, PyObject *k, PyObject *v);
+#define __Pyx_SetItemOnTypeDict(tp, k, v) __Pyx__SetItemOnTypeDict((PyTypeObject*)tp, k, v)
+
+/* FixUpExtensionType.proto */
+static CYTHON_INLINE int __Pyx_fix_up_extension_type_from_spec(PyType_Spec *spec, PyTypeObject *type);
+
+/* PyObjectCallNoArg.proto */
+static CYTHON_INLINE PyObject* __Pyx_PyObject_CallNoArg(PyObject *func);
+
+/* PyObjectCallOneArg.proto */
+static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg);
+
+/* PyObjectGetMethod.proto */
+static int __Pyx_PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method);
+
+/* PyObjectCallMethod0.proto */
+static PyObject* __Pyx_PyObject_CallMethod0(PyObject* obj, PyObject* method_name);
+
+/* ValidateBasesTuple.proto */
+#if CYTHON_COMPILING_IN_CPYTHON || CYTHON_COMPILING_IN_LIMITED_API || CYTHON_USE_TYPE_SPECS
+static int __Pyx_validate_bases_tuple(const char *type_name, Py_ssize_t dictoffset, PyObject *bases);
+#endif
+
+/* PyType_Ready.proto */
+CYTHON_UNUSED static int __Pyx_PyType_Ready(PyTypeObject *t);
+
+/* Import.proto */
+static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level);
+
+/* ImportDottedModule.proto */
+static PyObject *__Pyx_ImportDottedModule(PyObject *name, PyObject *parts_tuple);
+static PyObject *__Pyx_ImportDottedModule_WalkParts(PyObject *module, PyObject *name, PyObject *parts_tuple);
+
+/* ListPack.proto */
+static PyObject *__Pyx_PyList_Pack(Py_ssize_t n, ...);
+
+/* ImportFrom.proto */
+static PyObject* __Pyx_ImportFrom(PyObject* module, PyObject* name);
+
+/* pybytes_as_double.proto */
+static double __Pyx_SlowPyString_AsDouble(PyObject *obj);
+static double __Pyx__PyBytes_AsDouble(PyObject *obj, const char* start, Py_ssize_t length);
+static CYTHON_INLINE double __Pyx_PyBytes_AsDouble(PyObject *obj) {
+ char* as_c_string;
+ Py_ssize_t size;
+#if CYTHON_ASSUME_SAFE_MACROS && CYTHON_ASSUME_SAFE_SIZE
+ as_c_string = PyBytes_AS_STRING(obj);
+ size = PyBytes_GET_SIZE(obj);
+#else
+ if (PyBytes_AsStringAndSize(obj, &as_c_string, &size) < 0) {
+ return (double)-1;
+ }
+#endif
+ return __Pyx__PyBytes_AsDouble(obj, as_c_string, size);
+}
+static CYTHON_INLINE double __Pyx_PyByteArray_AsDouble(PyObject *obj) {
+ char* as_c_string;
+ Py_ssize_t size;
+#if CYTHON_ASSUME_SAFE_MACROS && CYTHON_ASSUME_SAFE_SIZE
+ as_c_string = PyByteArray_AS_STRING(obj);
+ size = PyByteArray_GET_SIZE(obj);
+#else
+ as_c_string = PyByteArray_AsString(obj);
+ if (as_c_string == NULL) {
+ return (double)-1;
+ }
+ size = PyByteArray_Size(obj);
+#endif
+ return __Pyx__PyBytes_AsDouble(obj, as_c_string, size);
+}
+
+/* pyunicode_as_double.proto */
+#if !CYTHON_COMPILING_IN_PYPY && CYTHON_ASSUME_SAFE_MACROS
+static const char* __Pyx__PyUnicode_AsDouble_Copy(const void* data, const int kind, char* buffer, Py_ssize_t start, Py_ssize_t end) {
+ int last_was_punctuation;
+ Py_ssize_t i;
+ last_was_punctuation = 1;
+ for (i=start; i <= end; i++) {
+ Py_UCS4 chr = PyUnicode_READ(kind, data, i);
+ int is_punctuation = (chr == '_') | (chr == '.');
+ *buffer = (char)chr;
+ buffer += (chr != '_');
+ if (unlikely(chr > 127)) goto parse_failure;
+ if (unlikely(last_was_punctuation & is_punctuation)) goto parse_failure;
+ last_was_punctuation = is_punctuation;
+ }
+ if (unlikely(last_was_punctuation)) goto parse_failure;
+ *buffer = '\0';
+ return buffer;
+parse_failure:
+ return NULL;
+}
+static double __Pyx__PyUnicode_AsDouble_inf_nan(const void* data, int kind, Py_ssize_t start, Py_ssize_t length) {
+ int matches = 1;
+ Py_UCS4 chr;
+ Py_UCS4 sign = PyUnicode_READ(kind, data, start);
+ int is_signed = (sign == '-') | (sign == '+');
+ start += is_signed;
+ length -= is_signed;
+ switch (PyUnicode_READ(kind, data, start)) {
+ #ifdef Py_NAN
+ case 'n':
+ case 'N':
+ if (unlikely(length != 3)) goto parse_failure;
+ chr = PyUnicode_READ(kind, data, start+1);
+ matches &= (chr == 'a') | (chr == 'A');
+ chr = PyUnicode_READ(kind, data, start+2);
+ matches &= (chr == 'n') | (chr == 'N');
+ if (unlikely(!matches)) goto parse_failure;
+ return (sign == '-') ? -Py_NAN : Py_NAN;
+ #endif
+ case 'i':
+ case 'I':
+ if (unlikely(length < 3)) goto parse_failure;
+ chr = PyUnicode_READ(kind, data, start+1);
+ matches &= (chr == 'n') | (chr == 'N');
+ chr = PyUnicode_READ(kind, data, start+2);
+ matches &= (chr == 'f') | (chr == 'F');
+ if (likely(length == 3 && matches))
+ return (sign == '-') ? -Py_HUGE_VAL : Py_HUGE_VAL;
+ if (unlikely(length != 8)) goto parse_failure;
+ chr = PyUnicode_READ(kind, data, start+3);
+ matches &= (chr == 'i') | (chr == 'I');
+ chr = PyUnicode_READ(kind, data, start+4);
+ matches &= (chr == 'n') | (chr == 'N');
+ chr = PyUnicode_READ(kind, data, start+5);
+ matches &= (chr == 'i') | (chr == 'I');
+ chr = PyUnicode_READ(kind, data, start+6);
+ matches &= (chr == 't') | (chr == 'T');
+ chr = PyUnicode_READ(kind, data, start+7);
+ matches &= (chr == 'y') | (chr == 'Y');
+ if (unlikely(!matches)) goto parse_failure;
+ return (sign == '-') ? -Py_HUGE_VAL : Py_HUGE_VAL;
+ case '.': case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9':
+ break;
+ default:
+ goto parse_failure;
+ }
+ return 0.0;
+parse_failure:
+ return -1.0;
+}
+static double __Pyx_PyUnicode_AsDouble_WithSpaces(PyObject *obj) {
+ double value;
+ const char *last;
+ char *end;
+ Py_ssize_t start, length = PyUnicode_GET_LENGTH(obj);
+ const int kind = PyUnicode_KIND(obj);
+ const void* data = PyUnicode_DATA(obj);
+ start = 0;
+ while (Py_UNICODE_ISSPACE(PyUnicode_READ(kind, data, start)))
+ start++;
+ while (start < length - 1 && Py_UNICODE_ISSPACE(PyUnicode_READ(kind, data, length - 1)))
+ length--;
+ length -= start;
+ if (unlikely(length <= 0)) goto fallback;
+ value = __Pyx__PyUnicode_AsDouble_inf_nan(data, kind, start, length);
+ if (unlikely(value == -1.0)) goto fallback;
+ if (value != 0.0) return value;
+ if (length < 40) {
+ char number[40];
+ last = __Pyx__PyUnicode_AsDouble_Copy(data, kind, number, start, start + length);
+ if (unlikely(!last)) goto fallback;
+ value = PyOS_string_to_double(number, &end, NULL);
+ } else {
+ char *number = (char*) PyMem_Malloc((length + 1) * sizeof(char));
+ if (unlikely(!number)) goto fallback;
+ last = __Pyx__PyUnicode_AsDouble_Copy(data, kind, number, start, start + length);
+ if (unlikely(!last)) {
+ PyMem_Free(number);
+ goto fallback;
+ }
+ value = PyOS_string_to_double(number, &end, NULL);
+ PyMem_Free(number);
+ }
+ if (likely(end == last) || (value == (double)-1 && PyErr_Occurred())) {
+ return value;
+ }
+fallback:
+ return __Pyx_SlowPyString_AsDouble(obj);
+}
+#endif
+static CYTHON_INLINE double __Pyx_PyUnicode_AsDouble(PyObject *obj) {
+#if !CYTHON_COMPILING_IN_PYPY && CYTHON_ASSUME_SAFE_MACROS
+ if (unlikely(__Pyx_PyUnicode_READY(obj) == -1))
+ return (double)-1;
+ if (likely(PyUnicode_IS_ASCII(obj))) {
+ const char *s;
+ Py_ssize_t length;
+ s = PyUnicode_AsUTF8AndSize(obj, &length);
+ return __Pyx__PyBytes_AsDouble(obj, s, length);
+ }
+ return __Pyx_PyUnicode_AsDouble_WithSpaces(obj);
+#else
+ return __Pyx_SlowPyString_AsDouble(obj);
+#endif
+}
+
+/* FetchSharedCythonModule.proto */
+static PyObject *__Pyx_FetchSharedCythonABIModule(void);
+
+/* dict_setdefault.proto */
+static CYTHON_INLINE PyObject *__Pyx_PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *default_value, int is_safe_type);
+
+/* FetchCommonType.proto */
+static PyTypeObject* __Pyx_FetchCommonTypeFromSpec(PyTypeObject *metaclass, PyObject *module, PyType_Spec *spec, PyObject *bases);
+
+/* CommonTypesMetaclass.proto */
+static int __pyx_CommonTypesMetaclass_init(PyObject *module);
+#define __Pyx_CommonTypesMetaclass_USED
+
+/* CallTypeTraverse.proto */
+#if !CYTHON_USE_TYPE_SPECS || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x03090000)
+#define __Pyx_call_type_traverse(o, always_call, visit, arg) 0
+#else
+static int __Pyx_call_type_traverse(PyObject *o, int always_call, visitproc visit, void *arg);
+#endif
+
+/* PyMethodNew.proto */
+static PyObject *__Pyx_PyMethod_New(PyObject *func, PyObject *self, PyObject *typ);
+
+/* PyVectorcallFastCallDict.proto */
+#if CYTHON_METH_FASTCALL && (CYTHON_VECTORCALL || CYTHON_BACKPORT_VECTORCALL)
+static CYTHON_INLINE PyObject *__Pyx_PyVectorcall_FastCallDict(PyObject *func, __pyx_vectorcallfunc vc, PyObject *const *args, size_t nargs, PyObject *kw);
+#endif
+
+/* CythonFunctionShared.proto */
+#define __Pyx_CyFunction_USED
+#define __Pyx_CYFUNCTION_STATICMETHOD 0x01
+#define __Pyx_CYFUNCTION_CLASSMETHOD 0x02
+#define __Pyx_CYFUNCTION_CCLASS 0x04
+#define __Pyx_CYFUNCTION_COROUTINE 0x08
+#define __Pyx_CyFunction_GetClosure(f)\
+ (((__pyx_CyFunctionObject *) (f))->func_closure)
+#if PY_VERSION_HEX < 0x030900B1 || CYTHON_COMPILING_IN_LIMITED_API
+ #define __Pyx_CyFunction_GetClassObj(f)\
+ (((__pyx_CyFunctionObject *) (f))->func_classobj)
+#else
+ #define __Pyx_CyFunction_GetClassObj(f)\
+ ((PyObject*) ((PyCMethodObject *) (f))->mm_class)
+#endif
+#define __Pyx_CyFunction_SetClassObj(f, classobj)\
+ __Pyx__CyFunction_SetClassObj((__pyx_CyFunctionObject *) (f), (classobj))
+#define __Pyx_CyFunction_Defaults(type, f)\
+ ((type *)(((__pyx_CyFunctionObject *) (f))->defaults))
+#define __Pyx_CyFunction_SetDefaultsGetter(f, g)\
+ ((__pyx_CyFunctionObject *) (f))->defaults_getter = (g)
+typedef struct {
+#if CYTHON_COMPILING_IN_LIMITED_API
+ PyObject_HEAD
+ PyObject *func;
+#elif PY_VERSION_HEX < 0x030900B1
+ PyCFunctionObject func;
+#else
+ PyCMethodObject func;
+#endif
+#if CYTHON_BACKPORT_VECTORCALL ||\
+ (CYTHON_COMPILING_IN_LIMITED_API && CYTHON_METH_FASTCALL)
+ __pyx_vectorcallfunc func_vectorcall;
+#endif
+#if CYTHON_COMPILING_IN_LIMITED_API
+ PyObject *func_weakreflist;
+#endif
+ PyObject *func_dict;
+ PyObject *func_name;
+ PyObject *func_qualname;
+ PyObject *func_doc;
+ PyObject *func_globals;
+ PyObject *func_code;
+ PyObject *func_closure;
+#if PY_VERSION_HEX < 0x030900B1 || CYTHON_COMPILING_IN_LIMITED_API
+ PyObject *func_classobj;
+#endif
+ PyObject *defaults;
+ int flags;
+ PyObject *defaults_tuple;
+ PyObject *defaults_kwdict;
+ PyObject *(*defaults_getter)(PyObject *);
+ PyObject *func_annotations;
+ PyObject *func_is_coroutine;
+} __pyx_CyFunctionObject;
+#undef __Pyx_CyOrPyCFunction_Check
+#define __Pyx_CyFunction_Check(obj) __Pyx_TypeCheck(obj, __pyx_mstate_global->__pyx_CyFunctionType)
+#define __Pyx_CyOrPyCFunction_Check(obj) __Pyx_TypeCheck2(obj, __pyx_mstate_global->__pyx_CyFunctionType, &PyCFunction_Type)
+#define __Pyx_CyFunction_CheckExact(obj) __Pyx_IS_TYPE(obj, __pyx_mstate_global->__pyx_CyFunctionType)
+static CYTHON_INLINE int __Pyx__IsSameCyOrCFunction(PyObject *func, void (*cfunc)(void));
+#undef __Pyx_IsSameCFunction
+#define __Pyx_IsSameCFunction(func, cfunc) __Pyx__IsSameCyOrCFunction(func, cfunc)
+static PyObject *__Pyx_CyFunction_Init(__pyx_CyFunctionObject* op, PyMethodDef *ml,
+ int flags, PyObject* qualname,
+ PyObject *closure,
+ PyObject *module, PyObject *globals,
+ PyObject* code);
+static CYTHON_INLINE void __Pyx__CyFunction_SetClassObj(__pyx_CyFunctionObject* f, PyObject* classobj);
+static CYTHON_INLINE PyObject *__Pyx_CyFunction_InitDefaults(PyObject *func,
+ PyTypeObject *defaults_type);
+static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsTuple(PyObject *m,
+ PyObject *tuple);
+static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsKwDict(PyObject *m,
+ PyObject *dict);
+static CYTHON_INLINE void __Pyx_CyFunction_SetAnnotationsDict(PyObject *m,
+ PyObject *dict);
+static int __pyx_CyFunction_init(PyObject *module);
+#if CYTHON_METH_FASTCALL
+static PyObject * __Pyx_CyFunction_Vectorcall_NOARGS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames);
+static PyObject * __Pyx_CyFunction_Vectorcall_O(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames);
+static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames);
+static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS_METHOD(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames);
+#if CYTHON_BACKPORT_VECTORCALL || CYTHON_COMPILING_IN_LIMITED_API
+#define __Pyx_CyFunction_func_vectorcall(f) (((__pyx_CyFunctionObject*)f)->func_vectorcall)
+#else
+#define __Pyx_CyFunction_func_vectorcall(f) (((PyCFunctionObject*)f)->vectorcall)
+#endif
+#endif
+
+/* CythonFunction.proto */
+static PyObject *__Pyx_CyFunction_New(PyMethodDef *ml,
+ int flags, PyObject* qualname,
+ PyObject *closure,
+ PyObject *module, PyObject *globals,
+ PyObject* code);
+
+/* CLineInTraceback.proto */
+#if CYTHON_CLINE_IN_TRACEBACK && CYTHON_CLINE_IN_TRACEBACK_RUNTIME
+static int __Pyx_CLineForTraceback(PyThreadState *tstate, int c_line);
+#else
+#define __Pyx_CLineForTraceback(tstate, c_line) (((CYTHON_CLINE_IN_TRACEBACK)) ? c_line : 0)
+#endif
+
+/* CodeObjectCache.proto */
+#if CYTHON_COMPILING_IN_LIMITED_API
+typedef PyObject __Pyx_CachedCodeObjectType;
+#else
+typedef PyCodeObject __Pyx_CachedCodeObjectType;
+#endif
+typedef struct {
+ __Pyx_CachedCodeObjectType* code_object;
+ int code_line;
+} __Pyx_CodeObjectCacheEntry;
+struct __Pyx_CodeObjectCache {
+ int count;
+ int max_count;
+ __Pyx_CodeObjectCacheEntry* entries;
+ #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING
+ __pyx_atomic_int_type accessor_count;
+ #endif
+};
+static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line);
+static __Pyx_CachedCodeObjectType *__pyx_find_code_object(int code_line);
+static void __pyx_insert_code_object(int code_line, __Pyx_CachedCodeObjectType* code_object);
+
+/* AddTraceback.proto */
+static void __Pyx_AddTraceback(const char *funcname, int c_line,
+ int py_line, const char *filename);
+
+/* RealImag.proto */
+#if CYTHON_CCOMPLEX
+ #ifdef __cplusplus
+ #define __Pyx_CREAL(z) ((z).real())
+ #define __Pyx_CIMAG(z) ((z).imag())
+ #else
+ #define __Pyx_CREAL(z) (__real__(z))
+ #define __Pyx_CIMAG(z) (__imag__(z))
+ #endif
+#else
+ #define __Pyx_CREAL(z) ((z).real)
+ #define __Pyx_CIMAG(z) ((z).imag)
+#endif
+#if defined(__cplusplus) && CYTHON_CCOMPLEX\
+ && (defined(_WIN32) || defined(__clang__) || (defined(__GNUC__) && (__GNUC__ >= 5 || __GNUC__ == 4 && __GNUC_MINOR__ >= 4 )) || __cplusplus >= 201103)
+ #define __Pyx_SET_CREAL(z,x) ((z).real(x))
+ #define __Pyx_SET_CIMAG(z,y) ((z).imag(y))
+#else
+ #define __Pyx_SET_CREAL(z,x) __Pyx_CREAL(z) = (x)
+ #define __Pyx_SET_CIMAG(z,y) __Pyx_CIMAG(z) = (y)
+#endif
+
+/* Arithmetic.proto */
+#if CYTHON_CCOMPLEX && (1) && (!0 || __cplusplus)
+ #define __Pyx_c_eq_double(a, b) ((a)==(b))
+ #define __Pyx_c_sum_double(a, b) ((a)+(b))
+ #define __Pyx_c_diff_double(a, b) ((a)-(b))
+ #define __Pyx_c_prod_double(a, b) ((a)*(b))
+ #define __Pyx_c_quot_double(a, b) ((a)/(b))
+ #define __Pyx_c_neg_double(a) (-(a))
+ #ifdef __cplusplus
+ #define __Pyx_c_is_zero_double(z) ((z)==(double)0)
+ #define __Pyx_c_conj_double(z) (::std::conj(z))
+ #if 1
+ #define __Pyx_c_abs_double(z) (::std::abs(z))
+ #define __Pyx_c_pow_double(a, b) (::std::pow(a, b))
+ #endif
+ #else
+ #define __Pyx_c_is_zero_double(z) ((z)==0)
+ #define __Pyx_c_conj_double(z) (conj(z))
+ #if 1
+ #define __Pyx_c_abs_double(z) (cabs(z))
+ #define __Pyx_c_pow_double(a, b) (cpow(a, b))
+ #endif
+ #endif
+#else
+ static CYTHON_INLINE int __Pyx_c_eq_double(__pyx_t_double_complex, __pyx_t_double_complex);
+ static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_sum_double(__pyx_t_double_complex, __pyx_t_double_complex);
+ static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_diff_double(__pyx_t_double_complex, __pyx_t_double_complex);
+ static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_prod_double(__pyx_t_double_complex, __pyx_t_double_complex);
+ static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_quot_double(__pyx_t_double_complex, __pyx_t_double_complex);
+ static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_neg_double(__pyx_t_double_complex);
+ static CYTHON_INLINE int __Pyx_c_is_zero_double(__pyx_t_double_complex);
+ static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_conj_double(__pyx_t_double_complex);
+ #if 1
+ static CYTHON_INLINE double __Pyx_c_abs_double(__pyx_t_double_complex);
+ static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_pow_double(__pyx_t_double_complex, __pyx_t_double_complex);
+ #endif
+#endif
+
+/* FromPy.proto */
+static __pyx_t_double_complex __Pyx_PyComplex_As___pyx_t_double_complex(PyObject*);
+
+/* GCCDiagnostics.proto */
+#if !defined(__INTEL_COMPILER) && defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6))
+#define __Pyx_HAS_GCC_DIAGNOSTIC
+#endif
+
+/* ToPy.proto */
+#define __pyx_PyComplex_FromComplex(z)\
+ PyComplex_FromDoubles((double)__Pyx_CREAL(z),\
+ (double)__Pyx_CIMAG(z))
+
+/* CIntFromPy.proto */
+static CYTHON_INLINE int __Pyx_PyLong_As_int(PyObject *);
+
+/* PyObjectVectorCallKwBuilder.proto */
+CYTHON_UNUSED static int __Pyx_VectorcallBuilder_AddArg_Check(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n);
+#if CYTHON_VECTORCALL
+#if PY_VERSION_HEX >= 0x03090000
+#define __Pyx_Object_Vectorcall_CallFromBuilder PyObject_Vectorcall
+#else
+#define __Pyx_Object_Vectorcall_CallFromBuilder _PyObject_Vectorcall
+#endif
+#define __Pyx_MakeVectorcallBuilderKwds(n) PyTuple_New(n)
+static int __Pyx_VectorcallBuilder_AddArg(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n);
+static int __Pyx_VectorcallBuilder_AddArgStr(const char *key, PyObject *value, PyObject *builder, PyObject **args, int n);
+#else
+#define __Pyx_Object_Vectorcall_CallFromBuilder __Pyx_PyObject_FastCallDict
+#define __Pyx_MakeVectorcallBuilderKwds(n) __Pyx_PyDict_NewPresized(n)
+#define __Pyx_VectorcallBuilder_AddArg(key, value, builder, args, n) PyDict_SetItem(builder, key, value)
+#define __Pyx_VectorcallBuilder_AddArgStr(key, value, builder, args, n) PyDict_SetItemString(builder, key, value)
+#endif
+
+/* CIntToPy.proto */
+static CYTHON_INLINE PyObject* __Pyx_PyLong_From_long(long value);
+
+/* CIntToPy.proto */
+static CYTHON_INLINE PyObject* __Pyx_PyLong_From_int(int value);
+
+/* FormatTypeName.proto */
+#if CYTHON_COMPILING_IN_LIMITED_API
+typedef PyObject *__Pyx_TypeName;
+#define __Pyx_FMT_TYPENAME "%U"
+#define __Pyx_DECREF_TypeName(obj) Py_XDECREF(obj)
+#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000
+#define __Pyx_PyType_GetFullyQualifiedName PyType_GetFullyQualifiedName
+#else
+static __Pyx_TypeName __Pyx_PyType_GetFullyQualifiedName(PyTypeObject* tp);
+#endif
+#else // !LIMITED_API
+typedef const char *__Pyx_TypeName;
+#define __Pyx_FMT_TYPENAME "%.200s"
+#define __Pyx_PyType_GetFullyQualifiedName(tp) ((tp)->tp_name)
+#define __Pyx_DECREF_TypeName(obj)
+#endif
+
+/* CIntFromPy.proto */
+static CYTHON_INLINE long __Pyx_PyLong_As_long(PyObject *);
+
+/* FastTypeChecks.proto */
+#if CYTHON_COMPILING_IN_CPYTHON
+#define __Pyx_TypeCheck(obj, type) __Pyx_IsSubtype(Py_TYPE(obj), (PyTypeObject *)type)
+#define __Pyx_TypeCheck2(obj, type1, type2) __Pyx_IsAnySubtype2(Py_TYPE(obj), (PyTypeObject *)type1, (PyTypeObject *)type2)
+static CYTHON_INLINE int __Pyx_IsSubtype(PyTypeObject *a, PyTypeObject *b);
+static CYTHON_INLINE int __Pyx_IsAnySubtype2(PyTypeObject *cls, PyTypeObject *a, PyTypeObject *b);
+static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches(PyObject *err, PyObject *type);
+static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches2(PyObject *err, PyObject *type1, PyObject *type2);
+#else
+#define __Pyx_TypeCheck(obj, type) PyObject_TypeCheck(obj, (PyTypeObject *)type)
+#define __Pyx_TypeCheck2(obj, type1, type2) (PyObject_TypeCheck(obj, (PyTypeObject *)type1) || PyObject_TypeCheck(obj, (PyTypeObject *)type2))
+#define __Pyx_PyErr_GivenExceptionMatches(err, type) PyErr_GivenExceptionMatches(err, type)
+static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches2(PyObject *err, PyObject *type1, PyObject *type2) {
+ return PyErr_GivenExceptionMatches(err, type1) || PyErr_GivenExceptionMatches(err, type2);
+}
+#endif
+#define __Pyx_PyErr_ExceptionMatches2(err1, err2) __Pyx_PyErr_GivenExceptionMatches2(__Pyx_PyErr_CurrentExceptionType(), err1, err2)
+#define __Pyx_PyException_Check(obj) __Pyx_TypeCheck(obj, PyExc_Exception)
+#ifdef PyExceptionInstance_Check
+ #define __Pyx_PyBaseException_Check(obj) PyExceptionInstance_Check(obj)
+#else
+ #define __Pyx_PyBaseException_Check(obj) __Pyx_TypeCheck(obj, PyExc_BaseException)
+#endif
+
+/* SwapException.proto */
+#if CYTHON_FAST_THREAD_STATE
+#define __Pyx_ExceptionSwap(type, value, tb) __Pyx__ExceptionSwap(__pyx_tstate, type, value, tb)
+static CYTHON_INLINE void __Pyx__ExceptionSwap(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb);
+#else
+static CYTHON_INLINE void __Pyx_ExceptionSwap(PyObject **type, PyObject **value, PyObject **tb);
+#endif
+
+/* PyObjectCall2Args.proto */
+static CYTHON_INLINE PyObject* __Pyx_PyObject_Call2Args(PyObject* function, PyObject* arg1, PyObject* arg2);
+
+/* PyObjectCallMethod1.proto */
+static PyObject* __Pyx_PyObject_CallMethod1(PyObject* obj, PyObject* method_name, PyObject* arg);
+
+/* ReturnWithStopIteration.proto */
+static CYTHON_INLINE void __Pyx_ReturnWithStopIteration(PyObject* value, int async, int iternext);
+
+/* CoroutineBase.proto */
+struct __pyx_CoroutineObject;
+typedef PyObject *(*__pyx_coroutine_body_t)(struct __pyx_CoroutineObject *, PyThreadState *, PyObject *);
+#if CYTHON_USE_EXC_INFO_STACK
+#define __Pyx_ExcInfoStruct _PyErr_StackItem
+#else
+typedef struct {
+ PyObject *exc_type;
+ PyObject *exc_value;
+ PyObject *exc_traceback;
+} __Pyx_ExcInfoStruct;
+#endif
+typedef struct __pyx_CoroutineObject {
+ PyObject_HEAD
+ __pyx_coroutine_body_t body;
+ PyObject *closure;
+ __Pyx_ExcInfoStruct gi_exc_state;
+ PyObject *gi_weakreflist;
+ PyObject *classobj;
+ PyObject *yieldfrom;
+ __Pyx_pyiter_sendfunc yieldfrom_am_send;
+ PyObject *gi_name;
+ PyObject *gi_qualname;
+ PyObject *gi_modulename;
+ PyObject *gi_code;
+ PyObject *gi_frame;
+#if CYTHON_USE_SYS_MONITORING && (CYTHON_PROFILE || CYTHON_TRACE)
+ PyMonitoringState __pyx_pymonitoring_state[__Pyx_MonitoringEventTypes_CyGen_count];
+ uint64_t __pyx_pymonitoring_version;
+#endif
+ int resume_label;
+ char is_running;
+} __pyx_CoroutineObject;
+static __pyx_CoroutineObject *__Pyx__Coroutine_New(
+ PyTypeObject *type, __pyx_coroutine_body_t body, PyObject *code, PyObject *closure,
+ PyObject *name, PyObject *qualname, PyObject *module_name);
+static __pyx_CoroutineObject *__Pyx__Coroutine_NewInit(
+ __pyx_CoroutineObject *gen, __pyx_coroutine_body_t body, PyObject *code, PyObject *closure,
+ PyObject *name, PyObject *qualname, PyObject *module_name);
+static CYTHON_INLINE void __Pyx_Coroutine_ExceptionClear(__Pyx_ExcInfoStruct *self);
+static int __Pyx_Coroutine_clear(PyObject *self);
+static __Pyx_PySendResult __Pyx_Coroutine_AmSend(PyObject *self, PyObject *value, PyObject **retval);
+static PyObject *__Pyx_Coroutine_Send(PyObject *self, PyObject *value);
+static __Pyx_PySendResult __Pyx_Coroutine_Close(PyObject *self, PyObject **retval);
+static PyObject *__Pyx_Coroutine_Throw(PyObject *gen, PyObject *args);
+#if CYTHON_USE_EXC_INFO_STACK
+#define __Pyx_Coroutine_SwapException(self)
+#define __Pyx_Coroutine_ResetAndClearException(self) __Pyx_Coroutine_ExceptionClear(&(self)->gi_exc_state)
+#else
+#define __Pyx_Coroutine_SwapException(self) {\
+ __Pyx_ExceptionSwap(&(self)->gi_exc_state.exc_type, &(self)->gi_exc_state.exc_value, &(self)->gi_exc_state.exc_traceback);\
+ __Pyx_Coroutine_ResetFrameBackpointer(&(self)->gi_exc_state);\
+ }
+#define __Pyx_Coroutine_ResetAndClearException(self) {\
+ __Pyx_ExceptionReset((self)->gi_exc_state.exc_type, (self)->gi_exc_state.exc_value, (self)->gi_exc_state.exc_traceback);\
+ (self)->gi_exc_state.exc_type = (self)->gi_exc_state.exc_value = (self)->gi_exc_state.exc_traceback = NULL;\
+ }
+#endif
+#if CYTHON_FAST_THREAD_STATE
+#define __Pyx_PyGen_FetchStopIterationValue(pvalue)\
+ __Pyx_PyGen__FetchStopIterationValue(__pyx_tstate, pvalue)
+#else
+#define __Pyx_PyGen_FetchStopIterationValue(pvalue)\
+ __Pyx_PyGen__FetchStopIterationValue(__Pyx_PyThreadState_Current, pvalue)
+#endif
+static int __Pyx_PyGen__FetchStopIterationValue(PyThreadState *tstate, PyObject **pvalue);
+static CYTHON_INLINE void __Pyx_Coroutine_ResetFrameBackpointer(__Pyx_ExcInfoStruct *exc_state);
+static char __Pyx_Coroutine_test_and_set_is_running(__pyx_CoroutineObject *gen);
+static void __Pyx_Coroutine_unset_is_running(__pyx_CoroutineObject *gen);
+static char __Pyx_Coroutine_get_is_running(__pyx_CoroutineObject *gen);
+static PyObject *__Pyx_Coroutine_get_is_running_getter(PyObject *gen, void *closure);
+#if __PYX_HAS_PY_AM_SEND == 2
+static void __Pyx_SetBackportTypeAmSend(PyTypeObject *type, __Pyx_PyAsyncMethodsStruct *static_amsend_methods, __Pyx_pyiter_sendfunc am_send);
+#endif
+static PyObject *__Pyx_Coroutine_fail_reduce_ex(PyObject *self, PyObject *arg);
+
+/* Generator.proto */
+#define __Pyx_Generator_USED
+#define __Pyx_Generator_CheckExact(obj) __Pyx_IS_TYPE(obj, __pyx_mstate_global->__pyx_GeneratorType)
+#define __Pyx_Generator_New(body, code, closure, name, qualname, module_name)\
+ __Pyx__Coroutine_New(__pyx_mstate_global->__pyx_GeneratorType, body, code, closure, name, qualname, module_name)
+static PyObject *__Pyx_Generator_Next(PyObject *self);
+static int __pyx_Generator_init(PyObject *module);
+static CYTHON_INLINE PyObject *__Pyx_Generator_GetInlinedResult(PyObject *self);
+
+/* GetRuntimeVersion.proto */
+static unsigned long __Pyx_get_runtime_version(void);
+
+/* CheckBinaryVersion.proto */
+static int __Pyx_check_binary_version(unsigned long ct_version, unsigned long rt_version, int allow_newer);
+
+/* MultiPhaseInitModuleState.proto */
+#if CYTHON_PEP489_MULTI_PHASE_INIT && CYTHON_USE_MODULE_STATE
+static PyObject *__Pyx_State_FindModule(void*);
+static int __Pyx_State_AddModule(PyObject* module, void*);
+static int __Pyx_State_RemoveModule(void*);
+#elif CYTHON_USE_MODULE_STATE
+#define __Pyx_State_FindModule PyState_FindModule
+#define __Pyx_State_AddModule PyState_AddModule
+#define __Pyx_State_RemoveModule PyState_RemoveModule
+#endif
+
+/* #### Code section: module_declarations ### */
+/* CythonABIVersion.proto */
+#if CYTHON_COMPILING_IN_LIMITED_API
+ #if CYTHON_METH_FASTCALL
+ #define __PYX_FASTCALL_ABI_SUFFIX "_fastcall"
+ #else
+ #define __PYX_FASTCALL_ABI_SUFFIX
+ #endif
+ #define __PYX_LIMITED_ABI_SUFFIX "limited" __PYX_FASTCALL_ABI_SUFFIX __PYX_AM_SEND_ABI_SUFFIX
+#else
+ #define __PYX_LIMITED_ABI_SUFFIX
+#endif
+#if __PYX_HAS_PY_AM_SEND == 1
+ #define __PYX_AM_SEND_ABI_SUFFIX
+#elif __PYX_HAS_PY_AM_SEND == 2
+ #define __PYX_AM_SEND_ABI_SUFFIX "amsendbackport"
+#else
+ #define __PYX_AM_SEND_ABI_SUFFIX "noamsend"
+#endif
+#ifndef __PYX_MONITORING_ABI_SUFFIX
+ #define __PYX_MONITORING_ABI_SUFFIX
+#endif
+#if CYTHON_USE_TP_FINALIZE
+ #define __PYX_TP_FINALIZE_ABI_SUFFIX
+#else
+ #define __PYX_TP_FINALIZE_ABI_SUFFIX "nofinalize"
+#endif
+#if CYTHON_USE_FREELISTS || !defined(__Pyx_AsyncGen_USED)
+ #define __PYX_FREELISTS_ABI_SUFFIX
+#else
+ #define __PYX_FREELISTS_ABI_SUFFIX "nofreelists"
+#endif
+#define CYTHON_ABI __PYX_ABI_VERSION __PYX_LIMITED_ABI_SUFFIX __PYX_MONITORING_ABI_SUFFIX __PYX_TP_FINALIZE_ABI_SUFFIX __PYX_FREELISTS_ABI_SUFFIX __PYX_AM_SEND_ABI_SUFFIX
+#define __PYX_ABI_MODULE_NAME "_cython_" CYTHON_ABI
+#define __PYX_TYPE_MODULE_PREFIX __PYX_ABI_MODULE_NAME "."
+
+
+/* Module declarations from "cython" */
+
+/* Module declarations from "fontTools.cu2qu.cu2qu" */
+static CYTHON_INLINE double __pyx_f_9fontTools_5cu2qu_5cu2qu_dot(__pyx_t_double_complex, __pyx_t_double_complex); /*proto*/
+static PyObject *__pyx_f_9fontTools_5cu2qu_5cu2qu__complex_div_by_real(__pyx_t_double_complex, double); /*proto*/
+static CYTHON_INLINE PyObject *__pyx_f_9fontTools_5cu2qu_5cu2qu_calc_cubic_points(__pyx_t_double_complex, __pyx_t_double_complex, __pyx_t_double_complex, __pyx_t_double_complex); /*proto*/
+static CYTHON_INLINE PyObject *__pyx_f_9fontTools_5cu2qu_5cu2qu_calc_cubic_parameters(__pyx_t_double_complex, __pyx_t_double_complex, __pyx_t_double_complex, __pyx_t_double_complex); /*proto*/
+static CYTHON_INLINE PyObject *__pyx_f_9fontTools_5cu2qu_5cu2qu_split_cubic_into_n_iter(__pyx_t_double_complex, __pyx_t_double_complex, __pyx_t_double_complex, __pyx_t_double_complex, PyObject *); /*proto*/
+static CYTHON_INLINE PyObject *__pyx_f_9fontTools_5cu2qu_5cu2qu_split_cubic_into_two(__pyx_t_double_complex, __pyx_t_double_complex, __pyx_t_double_complex, __pyx_t_double_complex); /*proto*/
+static CYTHON_INLINE PyObject *__pyx_f_9fontTools_5cu2qu_5cu2qu_split_cubic_into_three(__pyx_t_double_complex, __pyx_t_double_complex, __pyx_t_double_complex, __pyx_t_double_complex); /*proto*/
+static CYTHON_INLINE __pyx_t_double_complex __pyx_f_9fontTools_5cu2qu_5cu2qu_cubic_approx_control(double, __pyx_t_double_complex, __pyx_t_double_complex, __pyx_t_double_complex, __pyx_t_double_complex); /*proto*/
+static CYTHON_INLINE __pyx_t_double_complex __pyx_f_9fontTools_5cu2qu_5cu2qu_calc_intersect(__pyx_t_double_complex, __pyx_t_double_complex, __pyx_t_double_complex, __pyx_t_double_complex); /*proto*/
+static int __pyx_f_9fontTools_5cu2qu_5cu2qu_cubic_farthest_fit_inside(__pyx_t_double_complex, __pyx_t_double_complex, __pyx_t_double_complex, __pyx_t_double_complex, double); /*proto*/
+static CYTHON_INLINE PyObject *__pyx_f_9fontTools_5cu2qu_5cu2qu_cubic_approx_quadratic(PyObject *, double); /*proto*/
+static PyObject *__pyx_f_9fontTools_5cu2qu_5cu2qu_cubic_approx_spline(PyObject *, int, double, int); /*proto*/
+/* #### Code section: typeinfo ### */
+/* #### Code section: before_global_var ### */
+#define __Pyx_MODULE_NAME "fontTools.cu2qu.cu2qu"
+extern int __pyx_module_is_main_fontTools__cu2qu__cu2qu;
+int __pyx_module_is_main_fontTools__cu2qu__cu2qu = 0;
+
+/* Implementation of "fontTools.cu2qu.cu2qu" */
+/* #### Code section: global_var ### */
+static PyObject *__pyx_builtin_AttributeError;
+static PyObject *__pyx_builtin_ImportError;
+static PyObject *__pyx_builtin_range;
+static PyObject *__pyx_builtin_ZeroDivisionError;
+static PyObject *__pyx_builtin_AssertionError;
+/* #### Code section: string_decls ### */
+static const char __pyx_k_[] = ".";
+static const char __pyx_k_a[] = "a";
+static const char __pyx_k_b[] = "b";
+static const char __pyx_k_c[] = "c";
+static const char __pyx_k_d[] = "d";
+static const char __pyx_k_i[] = "i";
+static const char __pyx_k_l[] = "l";
+static const char __pyx_k_n[] = "n";
+static const char __pyx_k_p[] = "p";
+static const char __pyx_k_s[] = "s";
+static const char __pyx_k__2[] = "?";
+static const char __pyx_k__3[] = "\200\001";
+static const char __pyx_k_a1[] = "a1";
+static const char __pyx_k_b1[] = "b1";
+static const char __pyx_k_c1[] = "c1";
+static const char __pyx_k_d1[] = "d1";
+static const char __pyx_k_dt[] = "dt";
+static const char __pyx_k_gc[] = "gc";
+static const char __pyx_k_p0[] = "p0";
+static const char __pyx_k_p1[] = "p1";
+static const char __pyx_k_p2[] = "p2";
+static const char __pyx_k_p3[] = "p3";
+static const char __pyx_k_t1[] = "t1";
+static const char __pyx_k_NAN[] = "NAN";
+static const char __pyx_k_NaN[] = "NaN";
+static const char __pyx_k_all[] = "__all__";
+static const char __pyx_k_pop[] = "pop";
+static const char __pyx_k_func[] = "__func__";
+static const char __pyx_k_imag[] = "imag";
+static const char __pyx_k_main[] = "__main__";
+static const char __pyx_k_math[] = "math";
+static const char __pyx_k_name[] = "__name__";
+static const char __pyx_k_next[] = "next";
+static const char __pyx_k_real[] = "real";
+static const char __pyx_k_send[] = "send";
+static const char __pyx_k_spec[] = "__spec__";
+static const char __pyx_k_t1_2[] = "t1_2";
+static const char __pyx_k_test[] = "__test__";
+static const char __pyx_k_Error[] = "Error";
+static const char __pyx_k_MAX_N[] = "MAX_N";
+static const char __pyx_k_close[] = "close";
+static const char __pyx_k_curve[] = "curve";
+static const char __pyx_k_isnan[] = "isnan";
+static const char __pyx_k_range[] = "range";
+static const char __pyx_k_throw[] = "throw";
+static const char __pyx_k_value[] = "value";
+static const char __pyx_k_curves[] = "curves";
+static const char __pyx_k_enable[] = "enable";
+static const char __pyx_k_errors[] = "errors";
+static const char __pyx_k_last_i[] = "last_i";
+static const char __pyx_k_module[] = "__module__";
+static const char __pyx_k_spline[] = "spline";
+static const char __pyx_k_delta_2[] = "delta_2";
+static const char __pyx_k_delta_3[] = "delta_3";
+static const char __pyx_k_disable[] = "disable";
+static const char __pyx_k_max_err[] = "max_err";
+static const char __pyx_k_splines[] = "splines";
+static const char __pyx_k_COMPILED[] = "COMPILED";
+static const char __pyx_k_qualname[] = "__qualname__";
+static const char __pyx_k_set_name[] = "__set_name__";
+static const char __pyx_k_isenabled[] = "isenabled";
+static const char __pyx_k_Cu2QuError[] = "Cu2QuError";
+static const char __pyx_k_max_errors[] = "max_errors";
+static const char __pyx_k_ImportError[] = "ImportError";
+static const char __pyx_k_initializing[] = "_initializing";
+static const char __pyx_k_is_coroutine[] = "_is_coroutine";
+static const char __pyx_k_all_quadratic[] = "all_quadratic";
+static const char __pyx_k_AssertionError[] = "AssertionError";
+static const char __pyx_k_AttributeError[] = "AttributeError";
+static const char __pyx_k_ZeroDivisionError[] = "ZeroDivisionError";
+static const char __pyx_k_asyncio_coroutines[] = "asyncio.coroutines";
+static const char __pyx_k_cline_in_traceback[] = "cline_in_traceback";
+static const char __pyx_k_curve_to_quadratic[] = "curve_to_quadratic";
+static const char __pyx_k_ApproxNotFoundError[] = "ApproxNotFoundError";
+static const char __pyx_k_curves_to_quadratic[] = "curves_to_quadratic";
+static const char __pyx_k_fontTools_cu2qu_cu2qu[] = "fontTools.cu2qu.cu2qu";
+static const char __pyx_k_split_cubic_into_n_gen[] = "_split_cubic_into_n_gen";
+static const char __pyx_k_Lib_fontTools_cu2qu_cu2qu_py[] = "Lib/fontTools/cu2qu/cu2qu.py";
+static const char __pyx_k_curves_to_quadratic_line_503[] = "curves_to_quadratic (line 503)";
+static const char __pyx_k_AWBc_U_U_3fBa_AWCy_7_2QgQgT_a_Q[] = "\200\001\360\006\000()\360*\000\005\r\210A\210W\220B\220c\230\024\230U\240!\340\004\010\210\005\210U\220!\2203\220f\230B\230a\330\010\021\320\021$\240A\240W\250C\250y\270\001\330\010\013\2107\220'\230\021\340\014\023\2202\220Q\220g\230Q\230g\240T\250\025\250a\340\004\n\320\n\035\230Q\230a";
+static const char __pyx_k_J_Qawb_4uG4y_3a_3c_1A_avRq_T_AV[] = "\200\001\340,-\360J\001\000\005\016\210Q\210a\210w\220b\230\003\2304\230u\240G\2504\250y\270\001\330\004\013\2103\210a\210|\2303\230c\240\021\240!\340\004\010\210\003\2101\210A\330\004\016\210a\210v\220R\220q\330\004\r\210T\220\021\330\004\010\210\001\330\004\005\330\010\021\320\021$\240A\240V\2501\250D\260\003\260:\270Q\270d\300!\330\010\013\2107\220#\220Q\330\014\017\210r\220\023\220A\330\020\021\330\014\021\220\021\330\014\025\220Q\330\014\r\330\010\017\210q\220\005\220Q\330\010\r\210R\210r\220\023\220B\220a\330\010\013\2102\210S\220\001\340\014\023\2201\220B\220a\220w\230a\230w\240d\250%\250x\260t\270:\300Q\340\004\n\320\n\035\230Q\230a";
+static const char __pyx_k_Return_quadratic_Bezier_splines[] = "Return quadratic Bezier splines approximating the input cubic Beziers.\n\n Args:\n curves: A sequence of *n* curves, each curve being a sequence of four\n 2D tuples.\n max_errors: A sequence of *n* floats representing the maximum permissible\n deviation from each of the cubic Bezier curves.\n all_quadratic (bool): If True (default) returned values are a\n quadratic spline. If False, they are either a single quadratic\n curve or a single cubic curve.\n\n Example::\n\n >>> curves_to_quadratic( [\n ... [ (50,50), (100,100), (150,100), (200,50) ],\n ... [ (75,50), (120,100), (150,75), (200,60) ]\n ... ], [1,1] )\n [[(50.0, 50.0), (75.0, 75.0), (125.0, 91.66666666666666), (175.0, 75.0), (200.0, 50.0)], [(75.0, 50.0), (97.5, 75.0), (135.41666666666666, 82.08333333333333), (175.0, 67.5), (200.0, 60.0)]]\n\n The returned splines have \"implied oncurve points\" suitable for use in\n TrueType ``glif`` outlines - i.e. in the first spline returned above,\n the first quadratic segment runs from (50,50) to\n ( (75 + 125)/2 , (120 + 91.666..)/2 ) = (100, 83.333...).\n\n Returns:\n If all_quadratic is True, a list of splines, each spline being a list\n of 2D tuples.\n\n If all_quadratic is False, a list of curves, each curve being a quadratic\n (length 3), or cubic (length 4).\n\n Raises:\n fontTools.cu2qu.Errors.ApproxNotFoundError: if no suitable approximation\n can be found for all curves with the given parameters.\n ";
+/* #### Code section: decls ### */
+static PyObject *__pyx_pf_9fontTools_5cu2qu_5cu2qu__split_cubic_into_n_gen(CYTHON_UNUSED PyObject *__pyx_self, __pyx_t_double_complex __pyx_v_p0, __pyx_t_double_complex __pyx_v_p1, __pyx_t_double_complex __pyx_v_p2, __pyx_t_double_complex __pyx_v_p3, int __pyx_v_n); /* proto */
+static PyObject *__pyx_pf_9fontTools_5cu2qu_5cu2qu_3curve_to_quadratic(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_curve, double __pyx_v_max_err, int __pyx_v_all_quadratic); /* proto */
+static PyObject *__pyx_pf_9fontTools_5cu2qu_5cu2qu_5curves_to_quadratic(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_curves, PyObject *__pyx_v_max_errors, int __pyx_v_all_quadratic); /* proto */
+static PyObject *__pyx_tp_new_9fontTools_5cu2qu_5cu2qu___pyx_scope_struct___split_cubic_into_n_gen(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/
+/* #### Code section: late_includes ### */
+/* #### Code section: module_state ### */
+/* SmallCodeConfig */
+#ifndef CYTHON_SMALL_CODE
+#if defined(__clang__)
+ #define CYTHON_SMALL_CODE
+#elif defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3))
+ #define CYTHON_SMALL_CODE __attribute__((cold))
+#else
+ #define CYTHON_SMALL_CODE
+#endif
+#endif
+
+typedef struct {
+ PyObject *__pyx_d;
+ PyObject *__pyx_b;
+ PyObject *__pyx_cython_runtime;
+ PyObject *__pyx_empty_tuple;
+ PyObject *__pyx_empty_bytes;
+ PyObject *__pyx_empty_unicode;
+ #ifdef __Pyx_CyFunction_USED
+ PyTypeObject *__pyx_CyFunctionType;
+ #endif
+ #ifdef __Pyx_FusedFunction_USED
+ PyTypeObject *__pyx_FusedFunctionType;
+ #endif
+ #ifdef __Pyx_Generator_USED
+ PyTypeObject *__pyx_GeneratorType;
+ #endif
+ #ifdef __Pyx_IterableCoroutine_USED
+ PyTypeObject *__pyx_IterableCoroutineType;
+ #endif
+ #ifdef __Pyx_Coroutine_USED
+ PyTypeObject *__pyx_CoroutineAwaitType;
+ #endif
+ #ifdef __Pyx_Coroutine_USED
+ PyTypeObject *__pyx_CoroutineType;
+ #endif
+ PyObject *__pyx_type_9fontTools_5cu2qu_5cu2qu___pyx_scope_struct___split_cubic_into_n_gen;
+ PyTypeObject *__pyx_ptype_9fontTools_5cu2qu_5cu2qu___pyx_scope_struct___split_cubic_into_n_gen;
+ __Pyx_CachedCFunction __pyx_umethod_PyDict_Type_pop;
+ PyObject *__pyx_codeobj_tab[3];
+ PyObject *__pyx_string_tab[79];
+ PyObject *__pyx_int_1;
+ PyObject *__pyx_int_2;
+ PyObject *__pyx_int_3;
+ PyObject *__pyx_int_4;
+ PyObject *__pyx_int_6;
+ PyObject *__pyx_int_100;
+/* #### Code section: module_state_contents ### */
+/* IterNextPlain.module_state_decls */
+#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030A0000
+PyObject *__Pyx_GetBuiltinNext_LimitedAPI_cache;
+#endif
+
+
+#if CYTHON_USE_FREELISTS
+struct __pyx_obj_9fontTools_5cu2qu_5cu2qu___pyx_scope_struct___split_cubic_into_n_gen *__pyx_freelist_9fontTools_5cu2qu_5cu2qu___pyx_scope_struct___split_cubic_into_n_gen[8];
+int __pyx_freecount_9fontTools_5cu2qu_5cu2qu___pyx_scope_struct___split_cubic_into_n_gen;
+#endif
+/* CommonTypesMetaclass.module_state_decls */
+PyTypeObject *__pyx_CommonTypesMetaclassType;
+
+/* CachedMethodType.module_state_decls */
+#if CYTHON_COMPILING_IN_LIMITED_API
+PyObject *__Pyx_CachedMethodType;
+#endif
+
+/* CodeObjectCache.module_state_decls */
+struct __Pyx_CodeObjectCache __pyx_code_cache;
+
+/* #### Code section: module_state_end ### */
+} __pyx_mstatetype;
+
+#if CYTHON_USE_MODULE_STATE
+#ifdef __cplusplus
+namespace {
+extern struct PyModuleDef __pyx_moduledef;
+} /* anonymous namespace */
+#else
+static struct PyModuleDef __pyx_moduledef;
+#endif
+
+#define __pyx_mstate_global (__Pyx_PyModule_GetState(__Pyx_State_FindModule(&__pyx_moduledef)))
+
+#define __pyx_m (__Pyx_State_FindModule(&__pyx_moduledef))
+#else
+static __pyx_mstatetype __pyx_mstate_global_static =
+#ifdef __cplusplus
+ {};
+#else
+ {0};
+#endif
+static __pyx_mstatetype * const __pyx_mstate_global = &__pyx_mstate_global_static;
+#endif
+/* #### Code section: constant_name_defines ### */
+#define __pyx_kp_u_ __pyx_string_tab[0]
+#define __pyx_n_u_ApproxNotFoundError __pyx_string_tab[1]
+#define __pyx_n_u_AssertionError __pyx_string_tab[2]
+#define __pyx_n_u_AttributeError __pyx_string_tab[3]
+#define __pyx_n_u_COMPILED __pyx_string_tab[4]
+#define __pyx_n_u_Cu2QuError __pyx_string_tab[5]
+#define __pyx_n_u_Error __pyx_string_tab[6]
+#define __pyx_n_u_ImportError __pyx_string_tab[7]
+#define __pyx_kp_u_Lib_fontTools_cu2qu_cu2qu_py __pyx_string_tab[8]
+#define __pyx_n_u_MAX_N __pyx_string_tab[9]
+#define __pyx_n_u_NAN __pyx_string_tab[10]
+#define __pyx_n_u_NaN __pyx_string_tab[11]
+#define __pyx_kp_u_Return_quadratic_Bezier_splines __pyx_string_tab[12]
+#define __pyx_n_u_ZeroDivisionError __pyx_string_tab[13]
+#define __pyx_kp_u__2 __pyx_string_tab[14]
+#define __pyx_n_u_a __pyx_string_tab[15]
+#define __pyx_n_u_a1 __pyx_string_tab[16]
+#define __pyx_n_u_all __pyx_string_tab[17]
+#define __pyx_n_u_all_quadratic __pyx_string_tab[18]
+#define __pyx_n_u_asyncio_coroutines __pyx_string_tab[19]
+#define __pyx_n_u_b __pyx_string_tab[20]
+#define __pyx_n_u_b1 __pyx_string_tab[21]
+#define __pyx_n_u_c __pyx_string_tab[22]
+#define __pyx_n_u_c1 __pyx_string_tab[23]
+#define __pyx_n_u_cline_in_traceback __pyx_string_tab[24]
+#define __pyx_n_u_close __pyx_string_tab[25]
+#define __pyx_n_u_curve __pyx_string_tab[26]
+#define __pyx_n_u_curve_to_quadratic __pyx_string_tab[27]
+#define __pyx_n_u_curves __pyx_string_tab[28]
+#define __pyx_n_u_curves_to_quadratic __pyx_string_tab[29]
+#define __pyx_kp_u_curves_to_quadratic_line_503 __pyx_string_tab[30]
+#define __pyx_n_u_d __pyx_string_tab[31]
+#define __pyx_n_u_d1 __pyx_string_tab[32]
+#define __pyx_n_u_delta_2 __pyx_string_tab[33]
+#define __pyx_n_u_delta_3 __pyx_string_tab[34]
+#define __pyx_kp_u_disable __pyx_string_tab[35]
+#define __pyx_n_u_dt __pyx_string_tab[36]
+#define __pyx_kp_u_enable __pyx_string_tab[37]
+#define __pyx_n_u_errors __pyx_string_tab[38]
+#define __pyx_n_u_fontTools_cu2qu_cu2qu __pyx_string_tab[39]
+#define __pyx_n_u_func __pyx_string_tab[40]
+#define __pyx_kp_u_gc __pyx_string_tab[41]
+#define __pyx_n_u_i __pyx_string_tab[42]
+#define __pyx_n_u_imag __pyx_string_tab[43]
+#define __pyx_n_u_initializing __pyx_string_tab[44]
+#define __pyx_n_u_is_coroutine __pyx_string_tab[45]
+#define __pyx_kp_u_isenabled __pyx_string_tab[46]
+#define __pyx_n_u_isnan __pyx_string_tab[47]
+#define __pyx_n_u_l __pyx_string_tab[48]
+#define __pyx_n_u_last_i __pyx_string_tab[49]
+#define __pyx_n_u_main __pyx_string_tab[50]
+#define __pyx_n_u_math __pyx_string_tab[51]
+#define __pyx_n_u_max_err __pyx_string_tab[52]
+#define __pyx_n_u_max_errors __pyx_string_tab[53]
+#define __pyx_n_u_module __pyx_string_tab[54]
+#define __pyx_n_u_n __pyx_string_tab[55]
+#define __pyx_n_u_name __pyx_string_tab[56]
+#define __pyx_n_u_next __pyx_string_tab[57]
+#define __pyx_n_u_p __pyx_string_tab[58]
+#define __pyx_n_u_p0 __pyx_string_tab[59]
+#define __pyx_n_u_p1 __pyx_string_tab[60]
+#define __pyx_n_u_p2 __pyx_string_tab[61]
+#define __pyx_n_u_p3 __pyx_string_tab[62]
+#define __pyx_n_u_pop __pyx_string_tab[63]
+#define __pyx_n_u_qualname __pyx_string_tab[64]
+#define __pyx_n_u_range __pyx_string_tab[65]
+#define __pyx_n_u_real __pyx_string_tab[66]
+#define __pyx_n_u_s __pyx_string_tab[67]
+#define __pyx_n_u_send __pyx_string_tab[68]
+#define __pyx_n_u_set_name __pyx_string_tab[69]
+#define __pyx_n_u_spec __pyx_string_tab[70]
+#define __pyx_n_u_spline __pyx_string_tab[71]
+#define __pyx_n_u_splines __pyx_string_tab[72]
+#define __pyx_n_u_split_cubic_into_n_gen __pyx_string_tab[73]
+#define __pyx_n_u_t1 __pyx_string_tab[74]
+#define __pyx_n_u_t1_2 __pyx_string_tab[75]
+#define __pyx_n_u_test __pyx_string_tab[76]
+#define __pyx_n_u_throw __pyx_string_tab[77]
+#define __pyx_n_u_value __pyx_string_tab[78]
+/* #### Code section: module_state_clear ### */
+#if CYTHON_USE_MODULE_STATE
+static CYTHON_SMALL_CODE int __pyx_m_clear(PyObject *m) {
+ __pyx_mstatetype *clear_module_state = __Pyx_PyModule_GetState(m);
+ if (!clear_module_state) return 0;
+ Py_CLEAR(clear_module_state->__pyx_d);
+ Py_CLEAR(clear_module_state->__pyx_b);
+ Py_CLEAR(clear_module_state->__pyx_cython_runtime);
+ Py_CLEAR(clear_module_state->__pyx_empty_tuple);
+ Py_CLEAR(clear_module_state->__pyx_empty_bytes);
+ Py_CLEAR(clear_module_state->__pyx_empty_unicode);
+ #ifdef __Pyx_CyFunction_USED
+ Py_CLEAR(clear_module_state->__pyx_CyFunctionType);
+ #endif
+ #ifdef __Pyx_FusedFunction_USED
+ Py_CLEAR(clear_module_state->__pyx_FusedFunctionType);
+ #endif
+ #if CYTHON_PEP489_MULTI_PHASE_INIT
+ __Pyx_State_RemoveModule(NULL);
+ #endif
+ Py_CLEAR(clear_module_state->__pyx_ptype_9fontTools_5cu2qu_5cu2qu___pyx_scope_struct___split_cubic_into_n_gen);
+ Py_CLEAR(clear_module_state->__pyx_type_9fontTools_5cu2qu_5cu2qu___pyx_scope_struct___split_cubic_into_n_gen);
+ for (int i=0; i<3; ++i) { Py_CLEAR(clear_module_state->__pyx_codeobj_tab[i]); }
+ for (int i=0; i<79; ++i) { Py_CLEAR(clear_module_state->__pyx_string_tab[i]); }
+ Py_CLEAR(clear_module_state->__pyx_int_1);
+ Py_CLEAR(clear_module_state->__pyx_int_2);
+ Py_CLEAR(clear_module_state->__pyx_int_3);
+ Py_CLEAR(clear_module_state->__pyx_int_4);
+ Py_CLEAR(clear_module_state->__pyx_int_6);
+ Py_CLEAR(clear_module_state->__pyx_int_100);
+ return 0;
+}
+#endif
+/* #### Code section: module_state_traverse ### */
+#if CYTHON_USE_MODULE_STATE
+static CYTHON_SMALL_CODE int __pyx_m_traverse(PyObject *m, visitproc visit, void *arg) {
+ __pyx_mstatetype *traverse_module_state = __Pyx_PyModule_GetState(m);
+ if (!traverse_module_state) return 0;
+ Py_VISIT(traverse_module_state->__pyx_d);
+ Py_VISIT(traverse_module_state->__pyx_b);
+ Py_VISIT(traverse_module_state->__pyx_cython_runtime);
+ __Pyx_VISIT_CONST(traverse_module_state->__pyx_empty_tuple);
+ __Pyx_VISIT_CONST(traverse_module_state->__pyx_empty_bytes);
+ __Pyx_VISIT_CONST(traverse_module_state->__pyx_empty_unicode);
+ #ifdef __Pyx_CyFunction_USED
+ Py_VISIT(traverse_module_state->__pyx_CyFunctionType);
+ #endif
+ #ifdef __Pyx_FusedFunction_USED
+ Py_VISIT(traverse_module_state->__pyx_FusedFunctionType);
+ #endif
+ Py_VISIT(traverse_module_state->__pyx_ptype_9fontTools_5cu2qu_5cu2qu___pyx_scope_struct___split_cubic_into_n_gen);
+ Py_VISIT(traverse_module_state->__pyx_type_9fontTools_5cu2qu_5cu2qu___pyx_scope_struct___split_cubic_into_n_gen);
+ for (int i=0; i<3; ++i) { __Pyx_VISIT_CONST(traverse_module_state->__pyx_codeobj_tab[i]); }
+ for (int i=0; i<79; ++i) { __Pyx_VISIT_CONST(traverse_module_state->__pyx_string_tab[i]); }
+ __Pyx_VISIT_CONST(traverse_module_state->__pyx_int_1);
+ __Pyx_VISIT_CONST(traverse_module_state->__pyx_int_2);
+ __Pyx_VISIT_CONST(traverse_module_state->__pyx_int_3);
+ __Pyx_VISIT_CONST(traverse_module_state->__pyx_int_4);
+ __Pyx_VISIT_CONST(traverse_module_state->__pyx_int_6);
+ __Pyx_VISIT_CONST(traverse_module_state->__pyx_int_100);
+ return 0;
+}
+#endif
+/* #### Code section: module_code ### */
+
+/* "fontTools/cu2qu/cu2qu.py":37
+ *
+ *
+ * @cython.cfunc # <<<<<<<<<<<<<<
+ * @cython.inline
+ * @cython.returns(cython.double)
+*/
+
+static CYTHON_INLINE double __pyx_f_9fontTools_5cu2qu_5cu2qu_dot(__pyx_t_double_complex __pyx_v_v1, __pyx_t_double_complex __pyx_v_v2) {
+ double __pyx_v_result;
+ double __pyx_r;
+ double __pyx_t_1;
+ int __pyx_t_2;
+
+ /* "fontTools/cu2qu/cu2qu.py":51
+ * double: Dot product.
+ * """
+ * result = (v1 * v2.conjugate()).real # <<<<<<<<<<<<<<
+ * # When vectors are perpendicular (i.e. dot product is 0), the above expression may
+ * # yield slightly different results when running in pure Python vs C/Cython,
+*/
+ __pyx_t_1 = __Pyx_CREAL(__Pyx_c_prod_double(__pyx_v_v1, __Pyx_c_conj_double(__pyx_v_v2)));
+ __pyx_v_result = __pyx_t_1;
+
+ /* "fontTools/cu2qu/cu2qu.py":58
+ * # implementation. Because we are using the result in a denominator and catching
+ * # ZeroDivisionError (see `calc_intersect`), it's best to normalize the result here.
+ * if abs(result) < 1e-15: # <<<<<<<<<<<<<<
+ * result = 0.0
+ * return result
+*/
+ __pyx_t_1 = fabs(__pyx_v_result);
+ __pyx_t_2 = (__pyx_t_1 < 1e-15);
+ if (__pyx_t_2) {
+
+ /* "fontTools/cu2qu/cu2qu.py":59
+ * # ZeroDivisionError (see `calc_intersect`), it's best to normalize the result here.
+ * if abs(result) < 1e-15:
+ * result = 0.0 # <<<<<<<<<<<<<<
+ * return result
+ *
+*/
+ __pyx_v_result = 0.0;
+
+ /* "fontTools/cu2qu/cu2qu.py":58
+ * # implementation. Because we are using the result in a denominator and catching
+ * # ZeroDivisionError (see `calc_intersect`), it's best to normalize the result here.
+ * if abs(result) < 1e-15: # <<<<<<<<<<<<<<
+ * result = 0.0
+ * return result
+*/
+ }
+
+ /* "fontTools/cu2qu/cu2qu.py":60
+ * if abs(result) < 1e-15:
+ * result = 0.0
+ * return result # <<<<<<<<<<<<<<
+ *
+ *
+*/
+ __pyx_r = __pyx_v_result;
+ goto __pyx_L0;
+
+ /* "fontTools/cu2qu/cu2qu.py":37
+ *
+ *
+ * @cython.cfunc # <<<<<<<<<<<<<<
+ * @cython.inline
+ * @cython.returns(cython.double)
+*/
+
+ /* function exit code */
+ __pyx_L0:;
+ return __pyx_r;
+}
+
+/* "fontTools/cu2qu/cu2qu.py":63
+ *
+ *
+ * @cython.cfunc # <<<<<<<<<<<<<<
+ * @cython.locals(z=cython.complex, den=cython.double)
+ * @cython.locals(zr=cython.double, zi=cython.double)
+*/
+
+static PyObject *__pyx_f_9fontTools_5cu2qu_5cu2qu__complex_div_by_real(__pyx_t_double_complex __pyx_v_z, double __pyx_v_den) {
+ double __pyx_v_zr;
+ double __pyx_v_zi;
+ PyObject *__pyx_r = NULL;
+ __Pyx_RefNannyDeclarations
+ double __pyx_t_1;
+ PyObject *__pyx_t_2 = NULL;
+ PyObject *__pyx_t_3 = NULL;
+ PyObject *__pyx_t_4 = NULL;
+ PyObject *__pyx_t_5 = NULL;
+ PyObject *__pyx_t_6 = NULL;
+ size_t __pyx_t_7;
+ int __pyx_lineno = 0;
+ const char *__pyx_filename = NULL;
+ int __pyx_clineno = 0;
+ __Pyx_RefNannySetupContext("_complex_div_by_real", 0);
+
+ /* "fontTools/cu2qu/cu2qu.py":75
+ * https://github.com/fonttools/fonttools/issues/3928
+ * """
+ * zr = z.real # <<<<<<<<<<<<<<
+ * zi = z.imag
+ * return complex(zr / den, zi / den)
+*/
+ __pyx_t_1 = __Pyx_CREAL(__pyx_v_z);
+ __pyx_v_zr = __pyx_t_1;
+
+ /* "fontTools/cu2qu/cu2qu.py":76
+ * """
+ * zr = z.real
+ * zi = z.imag # <<<<<<<<<<<<<<
+ * return complex(zr / den, zi / den)
+ *
+*/
+ __pyx_t_1 = __Pyx_CIMAG(__pyx_v_z);
+ __pyx_v_zi = __pyx_t_1;
+
+ /* "fontTools/cu2qu/cu2qu.py":77
+ * zr = z.real
+ * zi = z.imag
+ * return complex(zr / den, zi / den) # <<<<<<<<<<<<<<
+ *
+ *
+*/
+ __Pyx_XDECREF(__pyx_r);
+ __pyx_t_3 = NULL;
+ __Pyx_INCREF((PyObject *)(&PyComplex_Type));
+ __pyx_t_4 = ((PyObject *)(&PyComplex_Type));
+ if (unlikely(__pyx_v_den == 0)) {
+ PyErr_SetString(PyExc_ZeroDivisionError, "float division");
+ __PYX_ERR(0, 77, __pyx_L1_error)
+ }
+ __pyx_t_5 = PyFloat_FromDouble((__pyx_v_zr / __pyx_v_den)); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 77, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_5);
+ if (unlikely(__pyx_v_den == 0)) {
+ PyErr_SetString(PyExc_ZeroDivisionError, "float division");
+ __PYX_ERR(0, 77, __pyx_L1_error)
+ }
+ __pyx_t_6 = PyFloat_FromDouble((__pyx_v_zi / __pyx_v_den)); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 77, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_6);
+ __pyx_t_7 = 1;
+ {
+ PyObject *__pyx_callargs[3] = {__pyx_t_3, __pyx_t_5, __pyx_t_6};
+ __pyx_t_2 = __Pyx_PyObject_FastCall(__pyx_t_4, __pyx_callargs+__pyx_t_7, (3-__pyx_t_7) | (__pyx_t_7*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
+ __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0;
+ __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
+ __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
+ __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
+ if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 77, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_2);
+ }
+ __pyx_r = __pyx_t_2;
+ __pyx_t_2 = 0;
+ goto __pyx_L0;
+
+ /* "fontTools/cu2qu/cu2qu.py":63
+ *
+ *
+ * @cython.cfunc # <<<<<<<<<<<<<<
+ * @cython.locals(z=cython.complex, den=cython.double)
+ * @cython.locals(zr=cython.double, zi=cython.double)
+*/
+
+ /* function exit code */
+ __pyx_L1_error:;
+ __Pyx_XDECREF(__pyx_t_2);
+ __Pyx_XDECREF(__pyx_t_3);
+ __Pyx_XDECREF(__pyx_t_4);
+ __Pyx_XDECREF(__pyx_t_5);
+ __Pyx_XDECREF(__pyx_t_6);
+ __Pyx_AddTraceback("fontTools.cu2qu.cu2qu._complex_div_by_real", __pyx_clineno, __pyx_lineno, __pyx_filename);
+ __pyx_r = 0;
+ __pyx_L0:;
+ __Pyx_XGIVEREF(__pyx_r);
+ __Pyx_RefNannyFinishContext();
+ return __pyx_r;
+}
+
+/* "fontTools/cu2qu/cu2qu.py":80
+ *
+ *
+ * @cython.cfunc # <<<<<<<<<<<<<<
+ * @cython.inline
+ * @cython.locals(a=cython.complex, b=cython.complex, c=cython.complex, d=cython.complex)
+*/
+
+static CYTHON_INLINE PyObject *__pyx_f_9fontTools_5cu2qu_5cu2qu_calc_cubic_points(__pyx_t_double_complex __pyx_v_a, __pyx_t_double_complex __pyx_v_b, __pyx_t_double_complex __pyx_v_c, __pyx_t_double_complex __pyx_v_d) {
+ __pyx_t_double_complex __pyx_v__1;
+ __pyx_t_double_complex __pyx_v__2;
+ __pyx_t_double_complex __pyx_v__3;
+ __pyx_t_double_complex __pyx_v__4;
+ PyObject *__pyx_r = NULL;
+ __Pyx_RefNannyDeclarations
+ PyObject *__pyx_t_1 = NULL;
+ PyObject *__pyx_t_2 = NULL;
+ PyObject *__pyx_t_3 = NULL;
+ __pyx_t_double_complex __pyx_t_4;
+ PyObject *__pyx_t_5 = NULL;
+ PyObject *__pyx_t_6 = NULL;
+ int __pyx_lineno = 0;
+ const char *__pyx_filename = NULL;
+ int __pyx_clineno = 0;
+ __Pyx_RefNannySetupContext("calc_cubic_points", 0);
+
+ /* "fontTools/cu2qu/cu2qu.py":87
+ * )
+ * def calc_cubic_points(a, b, c, d):
+ * _1 = d # <<<<<<<<<<<<<<
+ * _2 = _complex_div_by_real(c, 3.0) + d
+ * _3 = _complex_div_by_real(b + c, 3.0) + _2
+*/
+ __pyx_v__1 = __pyx_v_d;
+
+ /* "fontTools/cu2qu/cu2qu.py":88
+ * def calc_cubic_points(a, b, c, d):
+ * _1 = d
+ * _2 = _complex_div_by_real(c, 3.0) + d # <<<<<<<<<<<<<<
+ * _3 = _complex_div_by_real(b + c, 3.0) + _2
+ * _4 = a + d + c + b
+*/
+ __pyx_t_1 = __pyx_f_9fontTools_5cu2qu_5cu2qu__complex_div_by_real(__pyx_v_c, 3.0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 88, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_1);
+ __pyx_t_2 = __pyx_PyComplex_FromComplex(__pyx_v_d); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 88, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_2);
+ __pyx_t_3 = PyNumber_Add(__pyx_t_1, __pyx_t_2); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 88, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_3);
+ __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
+ __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+ __pyx_t_4 = __Pyx_PyComplex_As___pyx_t_double_complex(__pyx_t_3); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 88, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
+ __pyx_v__2 = __pyx_t_4;
+
+ /* "fontTools/cu2qu/cu2qu.py":89
+ * _1 = d
+ * _2 = _complex_div_by_real(c, 3.0) + d
+ * _3 = _complex_div_by_real(b + c, 3.0) + _2 # <<<<<<<<<<<<<<
+ * _4 = a + d + c + b
+ * return _1, _2, _3, _4
+*/
+ __pyx_t_3 = __pyx_f_9fontTools_5cu2qu_5cu2qu__complex_div_by_real(__Pyx_c_sum_double(__pyx_v_b, __pyx_v_c), 3.0); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 89, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_3);
+ __pyx_t_2 = __pyx_PyComplex_FromComplex(__pyx_v__2); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 89, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_2);
+ __pyx_t_1 = PyNumber_Add(__pyx_t_3, __pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 89, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_1);
+ __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
+ __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+ __pyx_t_4 = __Pyx_PyComplex_As___pyx_t_double_complex(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 89, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
+ __pyx_v__3 = __pyx_t_4;
+
+ /* "fontTools/cu2qu/cu2qu.py":90
+ * _2 = _complex_div_by_real(c, 3.0) + d
+ * _3 = _complex_div_by_real(b + c, 3.0) + _2
+ * _4 = a + d + c + b # <<<<<<<<<<<<<<
+ * return _1, _2, _3, _4
+ *
+*/
+ __pyx_v__4 = __Pyx_c_sum_double(__Pyx_c_sum_double(__Pyx_c_sum_double(__pyx_v_a, __pyx_v_d), __pyx_v_c), __pyx_v_b);
+
+ /* "fontTools/cu2qu/cu2qu.py":91
+ * _3 = _complex_div_by_real(b + c, 3.0) + _2
+ * _4 = a + d + c + b
+ * return _1, _2, _3, _4 # <<<<<<<<<<<<<<
+ *
+ *
+*/
+ __Pyx_XDECREF(__pyx_r);
+ __pyx_t_1 = __pyx_PyComplex_FromComplex(__pyx_v__1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 91, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_1);
+ __pyx_t_2 = __pyx_PyComplex_FromComplex(__pyx_v__2); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 91, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_2);
+ __pyx_t_3 = __pyx_PyComplex_FromComplex(__pyx_v__3); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 91, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_3);
+ __pyx_t_5 = __pyx_PyComplex_FromComplex(__pyx_v__4); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 91, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_5);
+ __pyx_t_6 = PyTuple_New(4); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 91, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_6);
+ __Pyx_GIVEREF(__pyx_t_1);
+ if (__Pyx_PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_1) != (0)) __PYX_ERR(0, 91, __pyx_L1_error);
+ __Pyx_GIVEREF(__pyx_t_2);
+ if (__Pyx_PyTuple_SET_ITEM(__pyx_t_6, 1, __pyx_t_2) != (0)) __PYX_ERR(0, 91, __pyx_L1_error);
+ __Pyx_GIVEREF(__pyx_t_3);
+ if (__Pyx_PyTuple_SET_ITEM(__pyx_t_6, 2, __pyx_t_3) != (0)) __PYX_ERR(0, 91, __pyx_L1_error);
+ __Pyx_GIVEREF(__pyx_t_5);
+ if (__Pyx_PyTuple_SET_ITEM(__pyx_t_6, 3, __pyx_t_5) != (0)) __PYX_ERR(0, 91, __pyx_L1_error);
+ __pyx_t_1 = 0;
+ __pyx_t_2 = 0;
+ __pyx_t_3 = 0;
+ __pyx_t_5 = 0;
+ __pyx_r = __pyx_t_6;
+ __pyx_t_6 = 0;
+ goto __pyx_L0;
+
+ /* "fontTools/cu2qu/cu2qu.py":80
+ *
+ *
+ * @cython.cfunc # <<<<<<<<<<<<<<
+ * @cython.inline
+ * @cython.locals(a=cython.complex, b=cython.complex, c=cython.complex, d=cython.complex)
+*/
+
+ /* function exit code */
+ __pyx_L1_error:;
+ __Pyx_XDECREF(__pyx_t_1);
+ __Pyx_XDECREF(__pyx_t_2);
+ __Pyx_XDECREF(__pyx_t_3);
+ __Pyx_XDECREF(__pyx_t_5);
+ __Pyx_XDECREF(__pyx_t_6);
+ __Pyx_AddTraceback("fontTools.cu2qu.cu2qu.calc_cubic_points", __pyx_clineno, __pyx_lineno, __pyx_filename);
+ __pyx_r = 0;
+ __pyx_L0:;
+ __Pyx_XGIVEREF(__pyx_r);
+ __Pyx_RefNannyFinishContext();
+ return __pyx_r;
+}
+
+/* "fontTools/cu2qu/cu2qu.py":94
+ *
+ *
+ * @cython.cfunc # <<<<<<<<<<<<<<
+ * @cython.inline
+ * @cython.locals(
+*/
+
+static CYTHON_INLINE PyObject *__pyx_f_9fontTools_5cu2qu_5cu2qu_calc_cubic_parameters(__pyx_t_double_complex __pyx_v_p0, __pyx_t_double_complex __pyx_v_p1, __pyx_t_double_complex __pyx_v_p2, __pyx_t_double_complex __pyx_v_p3) {
+ __pyx_t_double_complex __pyx_v_a;
+ __pyx_t_double_complex __pyx_v_b;
+ __pyx_t_double_complex __pyx_v_c;
+ __pyx_t_double_complex __pyx_v_d;
+ PyObject *__pyx_r = NULL;
+ __Pyx_RefNannyDeclarations
+ PyObject *__pyx_t_1 = NULL;
+ PyObject *__pyx_t_2 = NULL;
+ PyObject *__pyx_t_3 = NULL;
+ PyObject *__pyx_t_4 = NULL;
+ PyObject *__pyx_t_5 = NULL;
+ int __pyx_lineno = 0;
+ const char *__pyx_filename = NULL;
+ int __pyx_clineno = 0;
+ __Pyx_RefNannySetupContext("calc_cubic_parameters", 0);
+
+ /* "fontTools/cu2qu/cu2qu.py":101
+ * @cython.locals(a=cython.complex, b=cython.complex, c=cython.complex, d=cython.complex)
+ * def calc_cubic_parameters(p0, p1, p2, p3):
+ * c = (p1 - p0) * 3.0 # <<<<<<<<<<<<<<
+ * b = (p2 - p1) * 3.0 - c
+ * d = p0
+*/
+ __pyx_v_c = __Pyx_c_prod_double(__Pyx_c_diff_double(__pyx_v_p1, __pyx_v_p0), __pyx_t_double_complex_from_parts(3.0, 0));
+
+ /* "fontTools/cu2qu/cu2qu.py":102
+ * def calc_cubic_parameters(p0, p1, p2, p3):
+ * c = (p1 - p0) * 3.0
+ * b = (p2 - p1) * 3.0 - c # <<<<<<<<<<<<<<
+ * d = p0
+ * a = p3 - d - c - b
+*/
+ __pyx_v_b = __Pyx_c_diff_double(__Pyx_c_prod_double(__Pyx_c_diff_double(__pyx_v_p2, __pyx_v_p1), __pyx_t_double_complex_from_parts(3.0, 0)), __pyx_v_c);
+
+ /* "fontTools/cu2qu/cu2qu.py":103
+ * c = (p1 - p0) * 3.0
+ * b = (p2 - p1) * 3.0 - c
+ * d = p0 # <<<<<<<<<<<<<<
+ * a = p3 - d - c - b
+ * return a, b, c, d
+*/
+ __pyx_v_d = __pyx_v_p0;
+
+ /* "fontTools/cu2qu/cu2qu.py":104
+ * b = (p2 - p1) * 3.0 - c
+ * d = p0
+ * a = p3 - d - c - b # <<<<<<<<<<<<<<
+ * return a, b, c, d
+ *
+*/
+ __pyx_v_a = __Pyx_c_diff_double(__Pyx_c_diff_double(__Pyx_c_diff_double(__pyx_v_p3, __pyx_v_d), __pyx_v_c), __pyx_v_b);
+
+ /* "fontTools/cu2qu/cu2qu.py":105
+ * d = p0
+ * a = p3 - d - c - b
+ * return a, b, c, d # <<<<<<<<<<<<<<
+ *
+ *
+*/
+ __Pyx_XDECREF(__pyx_r);
+ __pyx_t_1 = __pyx_PyComplex_FromComplex(__pyx_v_a); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 105, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_1);
+ __pyx_t_2 = __pyx_PyComplex_FromComplex(__pyx_v_b); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 105, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_2);
+ __pyx_t_3 = __pyx_PyComplex_FromComplex(__pyx_v_c); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 105, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_3);
+ __pyx_t_4 = __pyx_PyComplex_FromComplex(__pyx_v_d); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 105, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_4);
+ __pyx_t_5 = PyTuple_New(4); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 105, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_5);
+ __Pyx_GIVEREF(__pyx_t_1);
+ if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_1) != (0)) __PYX_ERR(0, 105, __pyx_L1_error);
+ __Pyx_GIVEREF(__pyx_t_2);
+ if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 1, __pyx_t_2) != (0)) __PYX_ERR(0, 105, __pyx_L1_error);
+ __Pyx_GIVEREF(__pyx_t_3);
+ if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 2, __pyx_t_3) != (0)) __PYX_ERR(0, 105, __pyx_L1_error);
+ __Pyx_GIVEREF(__pyx_t_4);
+ if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 3, __pyx_t_4) != (0)) __PYX_ERR(0, 105, __pyx_L1_error);
+ __pyx_t_1 = 0;
+ __pyx_t_2 = 0;
+ __pyx_t_3 = 0;
+ __pyx_t_4 = 0;
+ __pyx_r = __pyx_t_5;
+ __pyx_t_5 = 0;
+ goto __pyx_L0;
+
+ /* "fontTools/cu2qu/cu2qu.py":94
+ *
+ *
+ * @cython.cfunc # <<<<<<<<<<<<<<
+ * @cython.inline
+ * @cython.locals(
+*/
+
+ /* function exit code */
+ __pyx_L1_error:;
+ __Pyx_XDECREF(__pyx_t_1);
+ __Pyx_XDECREF(__pyx_t_2);
+ __Pyx_XDECREF(__pyx_t_3);
+ __Pyx_XDECREF(__pyx_t_4);
+ __Pyx_XDECREF(__pyx_t_5);
+ __Pyx_AddTraceback("fontTools.cu2qu.cu2qu.calc_cubic_parameters", __pyx_clineno, __pyx_lineno, __pyx_filename);
+ __pyx_r = 0;
+ __pyx_L0:;
+ __Pyx_XGIVEREF(__pyx_r);
+ __Pyx_RefNannyFinishContext();
+ return __pyx_r;
+}
+
+/* "fontTools/cu2qu/cu2qu.py":108
+ *
+ *
+ * @cython.cfunc # <<<<<<<<<<<<<<
+ * @cython.inline
+ * @cython.locals(
+*/
+
+static CYTHON_INLINE PyObject *__pyx_f_9fontTools_5cu2qu_5cu2qu_split_cubic_into_n_iter(__pyx_t_double_complex __pyx_v_p0, __pyx_t_double_complex __pyx_v_p1, __pyx_t_double_complex __pyx_v_p2, __pyx_t_double_complex __pyx_v_p3, PyObject *__pyx_v_n) {
+ PyObject *__pyx_v_a = NULL;
+ PyObject *__pyx_v_b = NULL;
+ PyObject *__pyx_r = NULL;
+ __Pyx_RefNannyDeclarations
+ int __pyx_t_1;
+ PyObject *__pyx_t_2 = NULL;
+ PyObject *__pyx_t_3 = NULL;
+ PyObject *__pyx_t_4 = NULL;
+ PyObject *__pyx_t_5 = NULL;
+ PyObject *(*__pyx_t_6)(PyObject *);
+ __pyx_t_double_complex __pyx_t_7;
+ __pyx_t_double_complex __pyx_t_8;
+ __pyx_t_double_complex __pyx_t_9;
+ __pyx_t_double_complex __pyx_t_10;
+ PyObject *__pyx_t_11 = NULL;
+ PyObject *__pyx_t_12 = NULL;
+ PyObject *__pyx_t_13 = NULL;
+ size_t __pyx_t_14;
+ int __pyx_lineno = 0;
+ const char *__pyx_filename = NULL;
+ int __pyx_clineno = 0;
+ __Pyx_RefNannySetupContext("split_cubic_into_n_iter", 0);
+
+ /* "fontTools/cu2qu/cu2qu.py":130
+ * """
+ * # Hand-coded special-cases
+ * if n == 2: # <<<<<<<<<<<<<<
+ * return iter(split_cubic_into_two(p0, p1, p2, p3))
+ * if n == 3:
+*/
+ __pyx_t_1 = (__Pyx_PyLong_BoolEqObjC(__pyx_v_n, __pyx_mstate_global->__pyx_int_2, 2, 0)); if (unlikely((__pyx_t_1 < 0))) __PYX_ERR(0, 130, __pyx_L1_error)
+ if (__pyx_t_1) {
+
+ /* "fontTools/cu2qu/cu2qu.py":131
+ * # Hand-coded special-cases
+ * if n == 2:
+ * return iter(split_cubic_into_two(p0, p1, p2, p3)) # <<<<<<<<<<<<<<
+ * if n == 3:
+ * return iter(split_cubic_into_three(p0, p1, p2, p3))
+*/
+ __Pyx_XDECREF(__pyx_r);
+ __pyx_t_2 = __pyx_f_9fontTools_5cu2qu_5cu2qu_split_cubic_into_two(__pyx_v_p0, __pyx_v_p1, __pyx_v_p2, __pyx_v_p3); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 131, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_2);
+ __pyx_t_3 = PyObject_GetIter(__pyx_t_2); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 131, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_3);
+ __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+ __pyx_r = __pyx_t_3;
+ __pyx_t_3 = 0;
+ goto __pyx_L0;
+
+ /* "fontTools/cu2qu/cu2qu.py":130
+ * """
+ * # Hand-coded special-cases
+ * if n == 2: # <<<<<<<<<<<<<<
+ * return iter(split_cubic_into_two(p0, p1, p2, p3))
+ * if n == 3:
+*/
+ }
+
+ /* "fontTools/cu2qu/cu2qu.py":132
+ * if n == 2:
+ * return iter(split_cubic_into_two(p0, p1, p2, p3))
+ * if n == 3: # <<<<<<<<<<<<<<
+ * return iter(split_cubic_into_three(p0, p1, p2, p3))
+ * if n == 4:
+*/
+ __pyx_t_1 = (__Pyx_PyLong_BoolEqObjC(__pyx_v_n, __pyx_mstate_global->__pyx_int_3, 3, 0)); if (unlikely((__pyx_t_1 < 0))) __PYX_ERR(0, 132, __pyx_L1_error)
+ if (__pyx_t_1) {
+
+ /* "fontTools/cu2qu/cu2qu.py":133
+ * return iter(split_cubic_into_two(p0, p1, p2, p3))
+ * if n == 3:
+ * return iter(split_cubic_into_three(p0, p1, p2, p3)) # <<<<<<<<<<<<<<
+ * if n == 4:
+ * a, b = split_cubic_into_two(p0, p1, p2, p3)
+*/
+ __Pyx_XDECREF(__pyx_r);
+ __pyx_t_3 = __pyx_f_9fontTools_5cu2qu_5cu2qu_split_cubic_into_three(__pyx_v_p0, __pyx_v_p1, __pyx_v_p2, __pyx_v_p3); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 133, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_3);
+ __pyx_t_2 = PyObject_GetIter(__pyx_t_3); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 133, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_2);
+ __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
+ __pyx_r = __pyx_t_2;
+ __pyx_t_2 = 0;
+ goto __pyx_L0;
+
+ /* "fontTools/cu2qu/cu2qu.py":132
+ * if n == 2:
+ * return iter(split_cubic_into_two(p0, p1, p2, p3))
+ * if n == 3: # <<<<<<<<<<<<<<
+ * return iter(split_cubic_into_three(p0, p1, p2, p3))
+ * if n == 4:
+*/
+ }
+
+ /* "fontTools/cu2qu/cu2qu.py":134
+ * if n == 3:
+ * return iter(split_cubic_into_three(p0, p1, p2, p3))
+ * if n == 4: # <<<<<<<<<<<<<<
+ * a, b = split_cubic_into_two(p0, p1, p2, p3)
+ * return iter(
+*/
+ __pyx_t_1 = (__Pyx_PyLong_BoolEqObjC(__pyx_v_n, __pyx_mstate_global->__pyx_int_4, 4, 0)); if (unlikely((__pyx_t_1 < 0))) __PYX_ERR(0, 134, __pyx_L1_error)
+ if (__pyx_t_1) {
+
+ /* "fontTools/cu2qu/cu2qu.py":135
+ * return iter(split_cubic_into_three(p0, p1, p2, p3))
+ * if n == 4:
+ * a, b = split_cubic_into_two(p0, p1, p2, p3) # <<<<<<<<<<<<<<
+ * return iter(
+ * split_cubic_into_two(a[0], a[1], a[2], a[3])
+*/
+ __pyx_t_2 = __pyx_f_9fontTools_5cu2qu_5cu2qu_split_cubic_into_two(__pyx_v_p0, __pyx_v_p1, __pyx_v_p2, __pyx_v_p3); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 135, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_2);
+ if ((likely(PyTuple_CheckExact(__pyx_t_2))) || (PyList_CheckExact(__pyx_t_2))) {
+ PyObject* sequence = __pyx_t_2;
+ Py_ssize_t size = __Pyx_PySequence_SIZE(sequence);
+ if (unlikely(size != 2)) {
+ if (size > 2) __Pyx_RaiseTooManyValuesError(2);
+ else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size);
+ __PYX_ERR(0, 135, __pyx_L1_error)
+ }
+ #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS
+ if (likely(PyTuple_CheckExact(sequence))) {
+ __pyx_t_3 = PyTuple_GET_ITEM(sequence, 0);
+ __Pyx_INCREF(__pyx_t_3);
+ __pyx_t_4 = PyTuple_GET_ITEM(sequence, 1);
+ __Pyx_INCREF(__pyx_t_4);
+ } else {
+ __pyx_t_3 = __Pyx_PyList_GetItemRef(sequence, 0);
+ if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 135, __pyx_L1_error)
+ __Pyx_XGOTREF(__pyx_t_3);
+ __pyx_t_4 = __Pyx_PyList_GetItemRef(sequence, 1);
+ if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 135, __pyx_L1_error)
+ __Pyx_XGOTREF(__pyx_t_4);
+ }
+ #else
+ __pyx_t_3 = __Pyx_PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 135, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_3);
+ __pyx_t_4 = __Pyx_PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 135, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_4);
+ #endif
+ __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+ } else {
+ Py_ssize_t index = -1;
+ __pyx_t_5 = PyObject_GetIter(__pyx_t_2); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 135, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_5);
+ __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+ __pyx_t_6 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_5);
+ index = 0; __pyx_t_3 = __pyx_t_6(__pyx_t_5); if (unlikely(!__pyx_t_3)) goto __pyx_L6_unpacking_failed;
+ __Pyx_GOTREF(__pyx_t_3);
+ index = 1; __pyx_t_4 = __pyx_t_6(__pyx_t_5); if (unlikely(!__pyx_t_4)) goto __pyx_L6_unpacking_failed;
+ __Pyx_GOTREF(__pyx_t_4);
+ if (__Pyx_IternextUnpackEndCheck(__pyx_t_6(__pyx_t_5), 2) < 0) __PYX_ERR(0, 135, __pyx_L1_error)
+ __pyx_t_6 = NULL;
+ __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
+ goto __pyx_L7_unpacking_done;
+ __pyx_L6_unpacking_failed:;
+ __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
+ __pyx_t_6 = NULL;
+ if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index);
+ __PYX_ERR(0, 135, __pyx_L1_error)
+ __pyx_L7_unpacking_done:;
+ }
+ __pyx_v_a = __pyx_t_3;
+ __pyx_t_3 = 0;
+ __pyx_v_b = __pyx_t_4;
+ __pyx_t_4 = 0;
+
+ /* "fontTools/cu2qu/cu2qu.py":136
+ * if n == 4:
+ * a, b = split_cubic_into_two(p0, p1, p2, p3)
+ * return iter( # <<<<<<<<<<<<<<
+ * split_cubic_into_two(a[0], a[1], a[2], a[3])
+ * + split_cubic_into_two(b[0], b[1], b[2], b[3])
+*/
+ __Pyx_XDECREF(__pyx_r);
+
+ /* "fontTools/cu2qu/cu2qu.py":137
+ * a, b = split_cubic_into_two(p0, p1, p2, p3)
+ * return iter(
+ * split_cubic_into_two(a[0], a[1], a[2], a[3]) # <<<<<<<<<<<<<<
+ * + split_cubic_into_two(b[0], b[1], b[2], b[3])
+ * )
+*/
+ __pyx_t_2 = __Pyx_GetItemInt(__pyx_v_a, 0, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 137, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_2);
+ __pyx_t_7 = __Pyx_PyComplex_As___pyx_t_double_complex(__pyx_t_2); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 137, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+ __pyx_t_2 = __Pyx_GetItemInt(__pyx_v_a, 1, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 137, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_2);
+ __pyx_t_8 = __Pyx_PyComplex_As___pyx_t_double_complex(__pyx_t_2); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 137, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+ __pyx_t_2 = __Pyx_GetItemInt(__pyx_v_a, 2, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 137, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_2);
+ __pyx_t_9 = __Pyx_PyComplex_As___pyx_t_double_complex(__pyx_t_2); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 137, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+ __pyx_t_2 = __Pyx_GetItemInt(__pyx_v_a, 3, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 137, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_2);
+ __pyx_t_10 = __Pyx_PyComplex_As___pyx_t_double_complex(__pyx_t_2); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 137, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+ __pyx_t_2 = __pyx_f_9fontTools_5cu2qu_5cu2qu_split_cubic_into_two(__pyx_t_7, __pyx_t_8, __pyx_t_9, __pyx_t_10); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 137, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_2);
+
+ /* "fontTools/cu2qu/cu2qu.py":138
+ * return iter(
+ * split_cubic_into_two(a[0], a[1], a[2], a[3])
+ * + split_cubic_into_two(b[0], b[1], b[2], b[3]) # <<<<<<<<<<<<<<
+ * )
+ * if n == 6:
+*/
+ __pyx_t_4 = __Pyx_GetItemInt(__pyx_v_b, 0, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 138, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_4);
+ __pyx_t_10 = __Pyx_PyComplex_As___pyx_t_double_complex(__pyx_t_4); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 138, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
+ __pyx_t_4 = __Pyx_GetItemInt(__pyx_v_b, 1, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 138, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_4);
+ __pyx_t_9 = __Pyx_PyComplex_As___pyx_t_double_complex(__pyx_t_4); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 138, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
+ __pyx_t_4 = __Pyx_GetItemInt(__pyx_v_b, 2, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 138, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_4);
+ __pyx_t_8 = __Pyx_PyComplex_As___pyx_t_double_complex(__pyx_t_4); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 138, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
+ __pyx_t_4 = __Pyx_GetItemInt(__pyx_v_b, 3, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 138, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_4);
+ __pyx_t_7 = __Pyx_PyComplex_As___pyx_t_double_complex(__pyx_t_4); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 138, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
+ __pyx_t_4 = __pyx_f_9fontTools_5cu2qu_5cu2qu_split_cubic_into_two(__pyx_t_10, __pyx_t_9, __pyx_t_8, __pyx_t_7); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 138, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_4);
+ __pyx_t_3 = PyNumber_Add(__pyx_t_2, __pyx_t_4); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 138, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_3);
+ __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+ __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
+
+ /* "fontTools/cu2qu/cu2qu.py":136
+ * if n == 4:
+ * a, b = split_cubic_into_two(p0, p1, p2, p3)
+ * return iter( # <<<<<<<<<<<<<<
+ * split_cubic_into_two(a[0], a[1], a[2], a[3])
+ * + split_cubic_into_two(b[0], b[1], b[2], b[3])
+*/
+ __pyx_t_4 = PyObject_GetIter(__pyx_t_3); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 136, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_4);
+ __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
+ __pyx_r = __pyx_t_4;
+ __pyx_t_4 = 0;
+ goto __pyx_L0;
+
+ /* "fontTools/cu2qu/cu2qu.py":134
+ * if n == 3:
+ * return iter(split_cubic_into_three(p0, p1, p2, p3))
+ * if n == 4: # <<<<<<<<<<<<<<
+ * a, b = split_cubic_into_two(p0, p1, p2, p3)
+ * return iter(
+*/
+ }
+
+ /* "fontTools/cu2qu/cu2qu.py":140
+ * + split_cubic_into_two(b[0], b[1], b[2], b[3])
+ * )
+ * if n == 6: # <<<<<<<<<<<<<<
+ * a, b = split_cubic_into_two(p0, p1, p2, p3)
+ * return iter(
+*/
+ __pyx_t_1 = (__Pyx_PyLong_BoolEqObjC(__pyx_v_n, __pyx_mstate_global->__pyx_int_6, 6, 0)); if (unlikely((__pyx_t_1 < 0))) __PYX_ERR(0, 140, __pyx_L1_error)
+ if (__pyx_t_1) {
+
+ /* "fontTools/cu2qu/cu2qu.py":141
+ * )
+ * if n == 6:
+ * a, b = split_cubic_into_two(p0, p1, p2, p3) # <<<<<<<<<<<<<<
+ * return iter(
+ * split_cubic_into_three(a[0], a[1], a[2], a[3])
+*/
+ __pyx_t_4 = __pyx_f_9fontTools_5cu2qu_5cu2qu_split_cubic_into_two(__pyx_v_p0, __pyx_v_p1, __pyx_v_p2, __pyx_v_p3); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 141, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_4);
+ if ((likely(PyTuple_CheckExact(__pyx_t_4))) || (PyList_CheckExact(__pyx_t_4))) {
+ PyObject* sequence = __pyx_t_4;
+ Py_ssize_t size = __Pyx_PySequence_SIZE(sequence);
+ if (unlikely(size != 2)) {
+ if (size > 2) __Pyx_RaiseTooManyValuesError(2);
+ else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size);
+ __PYX_ERR(0, 141, __pyx_L1_error)
+ }
+ #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS
+ if (likely(PyTuple_CheckExact(sequence))) {
+ __pyx_t_3 = PyTuple_GET_ITEM(sequence, 0);
+ __Pyx_INCREF(__pyx_t_3);
+ __pyx_t_2 = PyTuple_GET_ITEM(sequence, 1);
+ __Pyx_INCREF(__pyx_t_2);
+ } else {
+ __pyx_t_3 = __Pyx_PyList_GetItemRef(sequence, 0);
+ if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 141, __pyx_L1_error)
+ __Pyx_XGOTREF(__pyx_t_3);
+ __pyx_t_2 = __Pyx_PyList_GetItemRef(sequence, 1);
+ if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 141, __pyx_L1_error)
+ __Pyx_XGOTREF(__pyx_t_2);
+ }
+ #else
+ __pyx_t_3 = __Pyx_PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 141, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_3);
+ __pyx_t_2 = __Pyx_PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 141, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_2);
+ #endif
+ __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
+ } else {
+ Py_ssize_t index = -1;
+ __pyx_t_5 = PyObject_GetIter(__pyx_t_4); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 141, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_5);
+ __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
+ __pyx_t_6 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_5);
+ index = 0; __pyx_t_3 = __pyx_t_6(__pyx_t_5); if (unlikely(!__pyx_t_3)) goto __pyx_L9_unpacking_failed;
+ __Pyx_GOTREF(__pyx_t_3);
+ index = 1; __pyx_t_2 = __pyx_t_6(__pyx_t_5); if (unlikely(!__pyx_t_2)) goto __pyx_L9_unpacking_failed;
+ __Pyx_GOTREF(__pyx_t_2);
+ if (__Pyx_IternextUnpackEndCheck(__pyx_t_6(__pyx_t_5), 2) < 0) __PYX_ERR(0, 141, __pyx_L1_error)
+ __pyx_t_6 = NULL;
+ __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
+ goto __pyx_L10_unpacking_done;
+ __pyx_L9_unpacking_failed:;
+ __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
+ __pyx_t_6 = NULL;
+ if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index);
+ __PYX_ERR(0, 141, __pyx_L1_error)
+ __pyx_L10_unpacking_done:;
+ }
+ __pyx_v_a = __pyx_t_3;
+ __pyx_t_3 = 0;
+ __pyx_v_b = __pyx_t_2;
+ __pyx_t_2 = 0;
+
+ /* "fontTools/cu2qu/cu2qu.py":142
+ * if n == 6:
+ * a, b = split_cubic_into_two(p0, p1, p2, p3)
+ * return iter( # <<<<<<<<<<<<<<
+ * split_cubic_into_three(a[0], a[1], a[2], a[3])
+ * + split_cubic_into_three(b[0], b[1], b[2], b[3])
+*/
+ __Pyx_XDECREF(__pyx_r);
+
+ /* "fontTools/cu2qu/cu2qu.py":143
+ * a, b = split_cubic_into_two(p0, p1, p2, p3)
+ * return iter(
+ * split_cubic_into_three(a[0], a[1], a[2], a[3]) # <<<<<<<<<<<<<<
+ * + split_cubic_into_three(b[0], b[1], b[2], b[3])
+ * )
+*/
+ __pyx_t_4 = __Pyx_GetItemInt(__pyx_v_a, 0, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 143, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_4);
+ __pyx_t_7 = __Pyx_PyComplex_As___pyx_t_double_complex(__pyx_t_4); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 143, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
+ __pyx_t_4 = __Pyx_GetItemInt(__pyx_v_a, 1, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 143, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_4);
+ __pyx_t_8 = __Pyx_PyComplex_As___pyx_t_double_complex(__pyx_t_4); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 143, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
+ __pyx_t_4 = __Pyx_GetItemInt(__pyx_v_a, 2, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 143, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_4);
+ __pyx_t_9 = __Pyx_PyComplex_As___pyx_t_double_complex(__pyx_t_4); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 143, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
+ __pyx_t_4 = __Pyx_GetItemInt(__pyx_v_a, 3, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 143, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_4);
+ __pyx_t_10 = __Pyx_PyComplex_As___pyx_t_double_complex(__pyx_t_4); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 143, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
+ __pyx_t_4 = __pyx_f_9fontTools_5cu2qu_5cu2qu_split_cubic_into_three(__pyx_t_7, __pyx_t_8, __pyx_t_9, __pyx_t_10); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 143, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_4);
+
+ /* "fontTools/cu2qu/cu2qu.py":144
+ * return iter(
+ * split_cubic_into_three(a[0], a[1], a[2], a[3])
+ * + split_cubic_into_three(b[0], b[1], b[2], b[3]) # <<<<<<<<<<<<<<
+ * )
+ *
+*/
+ __pyx_t_2 = __Pyx_GetItemInt(__pyx_v_b, 0, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 144, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_2);
+ __pyx_t_10 = __Pyx_PyComplex_As___pyx_t_double_complex(__pyx_t_2); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 144, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+ __pyx_t_2 = __Pyx_GetItemInt(__pyx_v_b, 1, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 144, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_2);
+ __pyx_t_9 = __Pyx_PyComplex_As___pyx_t_double_complex(__pyx_t_2); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 144, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+ __pyx_t_2 = __Pyx_GetItemInt(__pyx_v_b, 2, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 144, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_2);
+ __pyx_t_8 = __Pyx_PyComplex_As___pyx_t_double_complex(__pyx_t_2); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 144, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+ __pyx_t_2 = __Pyx_GetItemInt(__pyx_v_b, 3, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 144, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_2);
+ __pyx_t_7 = __Pyx_PyComplex_As___pyx_t_double_complex(__pyx_t_2); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 144, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+ __pyx_t_2 = __pyx_f_9fontTools_5cu2qu_5cu2qu_split_cubic_into_three(__pyx_t_10, __pyx_t_9, __pyx_t_8, __pyx_t_7); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 144, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_2);
+ __pyx_t_3 = PyNumber_Add(__pyx_t_4, __pyx_t_2); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 144, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_3);
+ __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
+ __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+
+ /* "fontTools/cu2qu/cu2qu.py":142
+ * if n == 6:
+ * a, b = split_cubic_into_two(p0, p1, p2, p3)
+ * return iter( # <<<<<<<<<<<<<<
+ * split_cubic_into_three(a[0], a[1], a[2], a[3])
+ * + split_cubic_into_three(b[0], b[1], b[2], b[3])
+*/
+ __pyx_t_2 = PyObject_GetIter(__pyx_t_3); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 142, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_2);
+ __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
+ __pyx_r = __pyx_t_2;
+ __pyx_t_2 = 0;
+ goto __pyx_L0;
+
+ /* "fontTools/cu2qu/cu2qu.py":140
+ * + split_cubic_into_two(b[0], b[1], b[2], b[3])
+ * )
+ * if n == 6: # <<<<<<<<<<<<<<
+ * a, b = split_cubic_into_two(p0, p1, p2, p3)
+ * return iter(
+*/
+ }
+
+ /* "fontTools/cu2qu/cu2qu.py":147
+ * )
+ *
+ * return _split_cubic_into_n_gen(p0, p1, p2, p3, n) # <<<<<<<<<<<<<<
+ *
+ *
+*/
+ __Pyx_XDECREF(__pyx_r);
+ __pyx_t_3 = NULL;
+ __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_split_cubic_into_n_gen); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 147, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_4);
+ __pyx_t_5 = __pyx_PyComplex_FromComplex(__pyx_v_p0); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 147, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_5);
+ __pyx_t_11 = __pyx_PyComplex_FromComplex(__pyx_v_p1); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 147, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_11);
+ __pyx_t_12 = __pyx_PyComplex_FromComplex(__pyx_v_p2); if (unlikely(!__pyx_t_12)) __PYX_ERR(0, 147, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_12);
+ __pyx_t_13 = __pyx_PyComplex_FromComplex(__pyx_v_p3); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 147, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_13);
+ __pyx_t_14 = 1;
+ #if CYTHON_UNPACK_METHODS
+ if (unlikely(PyMethod_Check(__pyx_t_4))) {
+ __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_4);
+ assert(__pyx_t_3);
+ PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_4);
+ __Pyx_INCREF(__pyx_t_3);
+ __Pyx_INCREF(__pyx__function);
+ __Pyx_DECREF_SET(__pyx_t_4, __pyx__function);
+ __pyx_t_14 = 0;
+ }
+ #endif
+ {
+ PyObject *__pyx_callargs[6] = {__pyx_t_3, __pyx_t_5, __pyx_t_11, __pyx_t_12, __pyx_t_13, __pyx_v_n};
+ __pyx_t_2 = __Pyx_PyObject_FastCall(__pyx_t_4, __pyx_callargs+__pyx_t_14, (6-__pyx_t_14) | (__pyx_t_14*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
+ __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0;
+ __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
+ __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0;
+ __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0;
+ __Pyx_DECREF(__pyx_t_13); __pyx_t_13 = 0;
+ __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
+ if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 147, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_2);
+ }
+ __pyx_r = __pyx_t_2;
+ __pyx_t_2 = 0;
+ goto __pyx_L0;
+
+ /* "fontTools/cu2qu/cu2qu.py":108
+ *
+ *
+ * @cython.cfunc # <<<<<<<<<<<<<<
+ * @cython.inline
+ * @cython.locals(
+*/
+
+ /* function exit code */
+ __pyx_L1_error:;
+ __Pyx_XDECREF(__pyx_t_2);
+ __Pyx_XDECREF(__pyx_t_3);
+ __Pyx_XDECREF(__pyx_t_4);
+ __Pyx_XDECREF(__pyx_t_5);
+ __Pyx_XDECREF(__pyx_t_11);
+ __Pyx_XDECREF(__pyx_t_12);
+ __Pyx_XDECREF(__pyx_t_13);
+ __Pyx_AddTraceback("fontTools.cu2qu.cu2qu.split_cubic_into_n_iter", __pyx_clineno, __pyx_lineno, __pyx_filename);
+ __pyx_r = 0;
+ __pyx_L0:;
+ __Pyx_XDECREF(__pyx_v_a);
+ __Pyx_XDECREF(__pyx_v_b);
+ __Pyx_XGIVEREF(__pyx_r);
+ __Pyx_RefNannyFinishContext();
+ return __pyx_r;
+}
+static PyObject *__pyx_gb_9fontTools_5cu2qu_5cu2qu_2generator(__pyx_CoroutineObject *__pyx_generator, CYTHON_UNUSED PyThreadState *__pyx_tstate, PyObject *__pyx_sent_value); /* proto */
+
+/* "fontTools/cu2qu/cu2qu.py":150
+ *
+ *
+ * @cython.locals( # <<<<<<<<<<<<<<
+ * p0=cython.complex,
+ * p1=cython.complex,
+*/
+
+/* Python wrapper */
+static PyObject *__pyx_pw_9fontTools_5cu2qu_5cu2qu_1_split_cubic_into_n_gen(PyObject *__pyx_self,
+#if CYTHON_METH_FASTCALL
+PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds
+#else
+PyObject *__pyx_args, PyObject *__pyx_kwds
+#endif
+); /*proto*/
+PyDoc_STRVAR(__pyx_doc_9fontTools_5cu2qu_5cu2qu__split_cubic_into_n_gen, "_split_cubic_into_n_gen(double complex p0, double complex p1, double complex p2, double complex p3, int n)");
+static PyMethodDef __pyx_mdef_9fontTools_5cu2qu_5cu2qu_1_split_cubic_into_n_gen = {"_split_cubic_into_n_gen", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_9fontTools_5cu2qu_5cu2qu_1_split_cubic_into_n_gen, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_9fontTools_5cu2qu_5cu2qu__split_cubic_into_n_gen};
+static PyObject *__pyx_pw_9fontTools_5cu2qu_5cu2qu_1_split_cubic_into_n_gen(PyObject *__pyx_self,
+#if CYTHON_METH_FASTCALL
+PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds
+#else
+PyObject *__pyx_args, PyObject *__pyx_kwds
+#endif
+) {
+ __pyx_t_double_complex __pyx_v_p0;
+ __pyx_t_double_complex __pyx_v_p1;
+ __pyx_t_double_complex __pyx_v_p2;
+ __pyx_t_double_complex __pyx_v_p3;
+ int __pyx_v_n;
+ #if !CYTHON_METH_FASTCALL
+ CYTHON_UNUSED Py_ssize_t __pyx_nargs;
+ #endif
+ CYTHON_UNUSED PyObject *const *__pyx_kwvalues;
+ PyObject* values[5] = {0,0,0,0,0};
+ int __pyx_lineno = 0;
+ const char *__pyx_filename = NULL;
+ int __pyx_clineno = 0;
+ PyObject *__pyx_r = 0;
+ __Pyx_RefNannyDeclarations
+ __Pyx_RefNannySetupContext("_split_cubic_into_n_gen (wrapper)", 0);
+ #if !CYTHON_METH_FASTCALL
+ #if CYTHON_ASSUME_SAFE_SIZE
+ __pyx_nargs = PyTuple_GET_SIZE(__pyx_args);
+ #else
+ __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL;
+ #endif
+ #endif
+ __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs);
+ {
+ PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_p0,&__pyx_mstate_global->__pyx_n_u_p1,&__pyx_mstate_global->__pyx_n_u_p2,&__pyx_mstate_global->__pyx_n_u_p3,&__pyx_mstate_global->__pyx_n_u_n,0};
+ const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0;
+ if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 150, __pyx_L3_error)
+ if (__pyx_kwds_len > 0) {
+ switch (__pyx_nargs) {
+ case 5:
+ values[4] = __Pyx_ArgRef_FASTCALL(__pyx_args, 4);
+ if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[4])) __PYX_ERR(0, 150, __pyx_L3_error)
+ CYTHON_FALLTHROUGH;
+ case 4:
+ values[3] = __Pyx_ArgRef_FASTCALL(__pyx_args, 3);
+ if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 150, __pyx_L3_error)
+ CYTHON_FALLTHROUGH;
+ case 3:
+ values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2);
+ if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 150, __pyx_L3_error)
+ CYTHON_FALLTHROUGH;
+ case 2:
+ values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1);
+ if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 150, __pyx_L3_error)
+ CYTHON_FALLTHROUGH;
+ case 1:
+ values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0);
+ if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 150, __pyx_L3_error)
+ CYTHON_FALLTHROUGH;
+ case 0: break;
+ default: goto __pyx_L5_argtuple_error;
+ }
+ const Py_ssize_t kwd_pos_args = __pyx_nargs;
+ if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "_split_cubic_into_n_gen", 0) < 0) __PYX_ERR(0, 150, __pyx_L3_error)
+ for (Py_ssize_t i = __pyx_nargs; i < 5; i++) {
+ if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("_split_cubic_into_n_gen", 1, 5, 5, i); __PYX_ERR(0, 150, __pyx_L3_error) }
+ }
+ } else if (unlikely(__pyx_nargs != 5)) {
+ goto __pyx_L5_argtuple_error;
+ } else {
+ values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0);
+ if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 150, __pyx_L3_error)
+ values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1);
+ if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 150, __pyx_L3_error)
+ values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2);
+ if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 150, __pyx_L3_error)
+ values[3] = __Pyx_ArgRef_FASTCALL(__pyx_args, 3);
+ if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 150, __pyx_L3_error)
+ values[4] = __Pyx_ArgRef_FASTCALL(__pyx_args, 4);
+ if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[4])) __PYX_ERR(0, 150, __pyx_L3_error)
+ }
+ __pyx_v_p0 = __Pyx_PyComplex_As___pyx_t_double_complex(values[0]); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 164, __pyx_L3_error)
+ __pyx_v_p1 = __Pyx_PyComplex_As___pyx_t_double_complex(values[1]); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 164, __pyx_L3_error)
+ __pyx_v_p2 = __Pyx_PyComplex_As___pyx_t_double_complex(values[2]); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 164, __pyx_L3_error)
+ __pyx_v_p3 = __Pyx_PyComplex_As___pyx_t_double_complex(values[3]); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 164, __pyx_L3_error)
+ __pyx_v_n = __Pyx_PyLong_As_int(values[4]); if (unlikely((__pyx_v_n == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 164, __pyx_L3_error)
+ }
+ goto __pyx_L6_skip;
+ __pyx_L5_argtuple_error:;
+ __Pyx_RaiseArgtupleInvalid("_split_cubic_into_n_gen", 1, 5, 5, __pyx_nargs); __PYX_ERR(0, 150, __pyx_L3_error)
+ __pyx_L6_skip:;
+ goto __pyx_L4_argument_unpacking_done;
+ __pyx_L3_error:;
+ for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) {
+ Py_XDECREF(values[__pyx_temp]);
+ }
+ __Pyx_AddTraceback("fontTools.cu2qu.cu2qu._split_cubic_into_n_gen", __pyx_clineno, __pyx_lineno, __pyx_filename);
+ __Pyx_RefNannyFinishContext();
+ return NULL;
+ __pyx_L4_argument_unpacking_done:;
+ __pyx_r = __pyx_pf_9fontTools_5cu2qu_5cu2qu__split_cubic_into_n_gen(__pyx_self, __pyx_v_p0, __pyx_v_p1, __pyx_v_p2, __pyx_v_p3, __pyx_v_n);
+
+ /* function exit code */
+ for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) {
+ Py_XDECREF(values[__pyx_temp]);
+ }
+ __Pyx_RefNannyFinishContext();
+ return __pyx_r;
+}
+
+static PyObject *__pyx_pf_9fontTools_5cu2qu_5cu2qu__split_cubic_into_n_gen(CYTHON_UNUSED PyObject *__pyx_self, __pyx_t_double_complex __pyx_v_p0, __pyx_t_double_complex __pyx_v_p1, __pyx_t_double_complex __pyx_v_p2, __pyx_t_double_complex __pyx_v_p3, int __pyx_v_n) {
+ struct __pyx_obj_9fontTools_5cu2qu_5cu2qu___pyx_scope_struct___split_cubic_into_n_gen *__pyx_cur_scope;
+ PyObject *__pyx_r = NULL;
+ __Pyx_RefNannyDeclarations
+ int __pyx_lineno = 0;
+ const char *__pyx_filename = NULL;
+ int __pyx_clineno = 0;
+ __Pyx_RefNannySetupContext("_split_cubic_into_n_gen", 0);
+ __pyx_cur_scope = (struct __pyx_obj_9fontTools_5cu2qu_5cu2qu___pyx_scope_struct___split_cubic_into_n_gen *)__pyx_tp_new_9fontTools_5cu2qu_5cu2qu___pyx_scope_struct___split_cubic_into_n_gen(__pyx_mstate_global->__pyx_ptype_9fontTools_5cu2qu_5cu2qu___pyx_scope_struct___split_cubic_into_n_gen, __pyx_mstate_global->__pyx_empty_tuple, NULL);
+ if (unlikely(!__pyx_cur_scope)) {
+ __pyx_cur_scope = ((struct __pyx_obj_9fontTools_5cu2qu_5cu2qu___pyx_scope_struct___split_cubic_into_n_gen *)Py_None);
+ __Pyx_INCREF(Py_None);
+ __PYX_ERR(0, 150, __pyx_L1_error)
+ } else {
+ __Pyx_GOTREF((PyObject *)__pyx_cur_scope);
+ }
+ __pyx_cur_scope->__pyx_v_p0 = __pyx_v_p0;
+ __pyx_cur_scope->__pyx_v_p1 = __pyx_v_p1;
+ __pyx_cur_scope->__pyx_v_p2 = __pyx_v_p2;
+ __pyx_cur_scope->__pyx_v_p3 = __pyx_v_p3;
+ __pyx_cur_scope->__pyx_v_n = __pyx_v_n;
+ {
+ __pyx_CoroutineObject *gen = __Pyx_Generator_New((__pyx_coroutine_body_t) __pyx_gb_9fontTools_5cu2qu_5cu2qu_2generator, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[0]), (PyObject *) __pyx_cur_scope, __pyx_mstate_global->__pyx_n_u_split_cubic_into_n_gen, __pyx_mstate_global->__pyx_n_u_split_cubic_into_n_gen, __pyx_mstate_global->__pyx_n_u_fontTools_cu2qu_cu2qu); if (unlikely(!gen)) __PYX_ERR(0, 150, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_cur_scope);
+ __Pyx_RefNannyFinishContext();
+ return (PyObject *) gen;
+ }
+
+ /* function exit code */
+ __pyx_L1_error:;
+ __Pyx_AddTraceback("fontTools.cu2qu.cu2qu._split_cubic_into_n_gen", __pyx_clineno, __pyx_lineno, __pyx_filename);
+ __pyx_r = NULL;
+ __Pyx_DECREF((PyObject *)__pyx_cur_scope);
+ __Pyx_XGIVEREF(__pyx_r);
+ __Pyx_RefNannyFinishContext();
+ return __pyx_r;
+}
+
+static PyObject *__pyx_gb_9fontTools_5cu2qu_5cu2qu_2generator(__pyx_CoroutineObject *__pyx_generator, CYTHON_UNUSED PyThreadState *__pyx_tstate, PyObject *__pyx_sent_value) /* generator body */
+{
+ struct __pyx_obj_9fontTools_5cu2qu_5cu2qu___pyx_scope_struct___split_cubic_into_n_gen *__pyx_cur_scope = ((struct __pyx_obj_9fontTools_5cu2qu_5cu2qu___pyx_scope_struct___split_cubic_into_n_gen *)__pyx_generator->closure);
+ PyObject *__pyx_r = NULL;
+ PyObject *__pyx_t_1 = NULL;
+ PyObject *__pyx_t_2 = NULL;
+ PyObject *__pyx_t_3 = NULL;
+ PyObject *__pyx_t_4 = NULL;
+ PyObject *__pyx_t_5 = NULL;
+ PyObject *__pyx_t_6 = NULL;
+ PyObject *(*__pyx_t_7)(PyObject *);
+ __pyx_t_double_complex __pyx_t_8;
+ __pyx_t_double_complex __pyx_t_9;
+ __pyx_t_double_complex __pyx_t_10;
+ __pyx_t_double_complex __pyx_t_11;
+ int __pyx_t_12;
+ int __pyx_t_13;
+ int __pyx_t_14;
+ int __pyx_lineno = 0;
+ const char *__pyx_filename = NULL;
+ int __pyx_clineno = 0;
+ __Pyx_RefNannyDeclarations
+ __Pyx_RefNannySetupContext("_split_cubic_into_n_gen", 0);
+ switch (__pyx_generator->resume_label) {
+ case 0: goto __pyx_L3_first_run;
+ case 1: goto __pyx_L8_resume_from_yield;
+ default: /* CPython raises the right error here */
+ __Pyx_RefNannyFinishContext();
+ return NULL;
+ }
+ __pyx_L3_first_run:;
+ if (unlikely(__pyx_sent_value != Py_None)) {
+ if (unlikely(__pyx_sent_value)) PyErr_SetString(PyExc_TypeError, "can't send non-None value to a just-started generator");
+ __PYX_ERR(0, 150, __pyx_L1_error)
+ }
+
+ /* "fontTools/cu2qu/cu2qu.py":165
+ * )
+ * def _split_cubic_into_n_gen(p0, p1, p2, p3, n):
+ * a, b, c, d = calc_cubic_parameters(p0, p1, p2, p3) # <<<<<<<<<<<<<<
+ * dt = 1 / n
+ * delta_2 = dt * dt
+*/
+ __pyx_t_1 = __pyx_f_9fontTools_5cu2qu_5cu2qu_calc_cubic_parameters(__pyx_cur_scope->__pyx_v_p0, __pyx_cur_scope->__pyx_v_p1, __pyx_cur_scope->__pyx_v_p2, __pyx_cur_scope->__pyx_v_p3); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 165, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_1);
+ if ((likely(PyTuple_CheckExact(__pyx_t_1))) || (PyList_CheckExact(__pyx_t_1))) {
+ PyObject* sequence = __pyx_t_1;
+ Py_ssize_t size = __Pyx_PySequence_SIZE(sequence);
+ if (unlikely(size != 4)) {
+ if (size > 4) __Pyx_RaiseTooManyValuesError(4);
+ else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size);
+ __PYX_ERR(0, 165, __pyx_L1_error)
+ }
+ #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS
+ if (likely(PyTuple_CheckExact(sequence))) {
+ __pyx_t_2 = PyTuple_GET_ITEM(sequence, 0);
+ __Pyx_INCREF(__pyx_t_2);
+ __pyx_t_3 = PyTuple_GET_ITEM(sequence, 1);
+ __Pyx_INCREF(__pyx_t_3);
+ __pyx_t_4 = PyTuple_GET_ITEM(sequence, 2);
+ __Pyx_INCREF(__pyx_t_4);
+ __pyx_t_5 = PyTuple_GET_ITEM(sequence, 3);
+ __Pyx_INCREF(__pyx_t_5);
+ } else {
+ __pyx_t_2 = __Pyx_PyList_GetItemRef(sequence, 0);
+ if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 165, __pyx_L1_error)
+ __Pyx_XGOTREF(__pyx_t_2);
+ __pyx_t_3 = __Pyx_PyList_GetItemRef(sequence, 1);
+ if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 165, __pyx_L1_error)
+ __Pyx_XGOTREF(__pyx_t_3);
+ __pyx_t_4 = __Pyx_PyList_GetItemRef(sequence, 2);
+ if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 165, __pyx_L1_error)
+ __Pyx_XGOTREF(__pyx_t_4);
+ __pyx_t_5 = __Pyx_PyList_GetItemRef(sequence, 3);
+ if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 165, __pyx_L1_error)
+ __Pyx_XGOTREF(__pyx_t_5);
+ }
+ #else
+ {
+ Py_ssize_t i;
+ PyObject** temps[4] = {&__pyx_t_2,&__pyx_t_3,&__pyx_t_4,&__pyx_t_5};
+ for (i=0; i < 4; i++) {
+ PyObject* item = __Pyx_PySequence_ITEM(sequence, i); if (unlikely(!item)) __PYX_ERR(0, 165, __pyx_L1_error)
+ __Pyx_GOTREF(item);
+ *(temps[i]) = item;
+ }
+ }
+ #endif
+ __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
+ } else {
+ Py_ssize_t index = -1;
+ PyObject** temps[4] = {&__pyx_t_2,&__pyx_t_3,&__pyx_t_4,&__pyx_t_5};
+ __pyx_t_6 = PyObject_GetIter(__pyx_t_1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 165, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_6);
+ __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
+ __pyx_t_7 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_6);
+ for (index=0; index < 4; index++) {
+ PyObject* item = __pyx_t_7(__pyx_t_6); if (unlikely(!item)) goto __pyx_L4_unpacking_failed;
+ __Pyx_GOTREF(item);
+ *(temps[index]) = item;
+ }
+ if (__Pyx_IternextUnpackEndCheck(__pyx_t_7(__pyx_t_6), 4) < 0) __PYX_ERR(0, 165, __pyx_L1_error)
+ __pyx_t_7 = NULL;
+ __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
+ goto __pyx_L5_unpacking_done;
+ __pyx_L4_unpacking_failed:;
+ __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
+ __pyx_t_7 = NULL;
+ if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index);
+ __PYX_ERR(0, 165, __pyx_L1_error)
+ __pyx_L5_unpacking_done:;
+ }
+ __pyx_t_8 = __Pyx_PyComplex_As___pyx_t_double_complex(__pyx_t_2); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 165, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+ __pyx_t_9 = __Pyx_PyComplex_As___pyx_t_double_complex(__pyx_t_3); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 165, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
+ __pyx_t_10 = __Pyx_PyComplex_As___pyx_t_double_complex(__pyx_t_4); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 165, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
+ __pyx_t_11 = __Pyx_PyComplex_As___pyx_t_double_complex(__pyx_t_5); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 165, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
+ __pyx_cur_scope->__pyx_v_a = __pyx_t_8;
+ __pyx_cur_scope->__pyx_v_b = __pyx_t_9;
+ __pyx_cur_scope->__pyx_v_c = __pyx_t_10;
+ __pyx_cur_scope->__pyx_v_d = __pyx_t_11;
+
+ /* "fontTools/cu2qu/cu2qu.py":166
+ * def _split_cubic_into_n_gen(p0, p1, p2, p3, n):
+ * a, b, c, d = calc_cubic_parameters(p0, p1, p2, p3)
+ * dt = 1 / n # <<<<<<<<<<<<<<
+ * delta_2 = dt * dt
+ * delta_3 = dt * delta_2
+*/
+ if (unlikely(__pyx_cur_scope->__pyx_v_n == 0)) {
+ PyErr_SetString(PyExc_ZeroDivisionError, "float division");
+ __PYX_ERR(0, 166, __pyx_L1_error)
+ }
+ __pyx_cur_scope->__pyx_v_dt = (1.0 / ((double)__pyx_cur_scope->__pyx_v_n));
+
+ /* "fontTools/cu2qu/cu2qu.py":167
+ * a, b, c, d = calc_cubic_parameters(p0, p1, p2, p3)
+ * dt = 1 / n
+ * delta_2 = dt * dt # <<<<<<<<<<<<<<
+ * delta_3 = dt * delta_2
+ * for i in range(n):
+*/
+ __pyx_cur_scope->__pyx_v_delta_2 = (__pyx_cur_scope->__pyx_v_dt * __pyx_cur_scope->__pyx_v_dt);
+
+ /* "fontTools/cu2qu/cu2qu.py":168
+ * dt = 1 / n
+ * delta_2 = dt * dt
+ * delta_3 = dt * delta_2 # <<<<<<<<<<<<<<
+ * for i in range(n):
+ * t1 = i * dt
+*/
+ __pyx_cur_scope->__pyx_v_delta_3 = (__pyx_cur_scope->__pyx_v_dt * __pyx_cur_scope->__pyx_v_delta_2);
+
+ /* "fontTools/cu2qu/cu2qu.py":169
+ * delta_2 = dt * dt
+ * delta_3 = dt * delta_2
+ * for i in range(n): # <<<<<<<<<<<<<<
+ * t1 = i * dt
+ * t1_2 = t1 * t1
+*/
+ __pyx_t_12 = __pyx_cur_scope->__pyx_v_n;
+ __pyx_t_13 = __pyx_t_12;
+ for (__pyx_t_14 = 0; __pyx_t_14 < __pyx_t_13; __pyx_t_14+=1) {
+ __pyx_cur_scope->__pyx_v_i = __pyx_t_14;
+
+ /* "fontTools/cu2qu/cu2qu.py":170
+ * delta_3 = dt * delta_2
+ * for i in range(n):
+ * t1 = i * dt # <<<<<<<<<<<<<<
+ * t1_2 = t1 * t1
+ * # calc new a, b, c and d
+*/
+ __pyx_cur_scope->__pyx_v_t1 = (__pyx_cur_scope->__pyx_v_i * __pyx_cur_scope->__pyx_v_dt);
+
+ /* "fontTools/cu2qu/cu2qu.py":171
+ * for i in range(n):
+ * t1 = i * dt
+ * t1_2 = t1 * t1 # <<<<<<<<<<<<<<
+ * # calc new a, b, c and d
+ * a1 = a * delta_3
+*/
+ __pyx_cur_scope->__pyx_v_t1_2 = (__pyx_cur_scope->__pyx_v_t1 * __pyx_cur_scope->__pyx_v_t1);
+
+ /* "fontTools/cu2qu/cu2qu.py":173
+ * t1_2 = t1 * t1
+ * # calc new a, b, c and d
+ * a1 = a * delta_3 # <<<<<<<<<<<<<<
+ * b1 = (3 * a * t1 + b) * delta_2
+ * c1 = (2 * b * t1 + c + 3 * a * t1_2) * dt
+*/
+ __pyx_cur_scope->__pyx_v_a1 = __Pyx_c_prod_double(__pyx_cur_scope->__pyx_v_a, __pyx_t_double_complex_from_parts(__pyx_cur_scope->__pyx_v_delta_3, 0));
+
+ /* "fontTools/cu2qu/cu2qu.py":174
+ * # calc new a, b, c and d
+ * a1 = a * delta_3
+ * b1 = (3 * a * t1 + b) * delta_2 # <<<<<<<<<<<<<<
+ * c1 = (2 * b * t1 + c + 3 * a * t1_2) * dt
+ * d1 = a * t1 * t1_2 + b * t1_2 + c * t1 + d
+*/
+ __pyx_cur_scope->__pyx_v_b1 = __Pyx_c_prod_double(__Pyx_c_sum_double(__Pyx_c_prod_double(__Pyx_c_prod_double(__pyx_t_double_complex_from_parts(3, 0), __pyx_cur_scope->__pyx_v_a), __pyx_t_double_complex_from_parts(__pyx_cur_scope->__pyx_v_t1, 0)), __pyx_cur_scope->__pyx_v_b), __pyx_t_double_complex_from_parts(__pyx_cur_scope->__pyx_v_delta_2, 0));
+
+ /* "fontTools/cu2qu/cu2qu.py":175
+ * a1 = a * delta_3
+ * b1 = (3 * a * t1 + b) * delta_2
+ * c1 = (2 * b * t1 + c + 3 * a * t1_2) * dt # <<<<<<<<<<<<<<
+ * d1 = a * t1 * t1_2 + b * t1_2 + c * t1 + d
+ * yield calc_cubic_points(a1, b1, c1, d1)
+*/
+ __pyx_cur_scope->__pyx_v_c1 = __Pyx_c_prod_double(__Pyx_c_sum_double(__Pyx_c_sum_double(__Pyx_c_prod_double(__Pyx_c_prod_double(__pyx_t_double_complex_from_parts(2, 0), __pyx_cur_scope->__pyx_v_b), __pyx_t_double_complex_from_parts(__pyx_cur_scope->__pyx_v_t1, 0)), __pyx_cur_scope->__pyx_v_c), __Pyx_c_prod_double(__Pyx_c_prod_double(__pyx_t_double_complex_from_parts(3, 0), __pyx_cur_scope->__pyx_v_a), __pyx_t_double_complex_from_parts(__pyx_cur_scope->__pyx_v_t1_2, 0))), __pyx_t_double_complex_from_parts(__pyx_cur_scope->__pyx_v_dt, 0));
+
+ /* "fontTools/cu2qu/cu2qu.py":176
+ * b1 = (3 * a * t1 + b) * delta_2
+ * c1 = (2 * b * t1 + c + 3 * a * t1_2) * dt
+ * d1 = a * t1 * t1_2 + b * t1_2 + c * t1 + d # <<<<<<<<<<<<<<
+ * yield calc_cubic_points(a1, b1, c1, d1)
+ *
+*/
+ __pyx_cur_scope->__pyx_v_d1 = __Pyx_c_sum_double(__Pyx_c_sum_double(__Pyx_c_sum_double(__Pyx_c_prod_double(__Pyx_c_prod_double(__pyx_cur_scope->__pyx_v_a, __pyx_t_double_complex_from_parts(__pyx_cur_scope->__pyx_v_t1, 0)), __pyx_t_double_complex_from_parts(__pyx_cur_scope->__pyx_v_t1_2, 0)), __Pyx_c_prod_double(__pyx_cur_scope->__pyx_v_b, __pyx_t_double_complex_from_parts(__pyx_cur_scope->__pyx_v_t1_2, 0))), __Pyx_c_prod_double(__pyx_cur_scope->__pyx_v_c, __pyx_t_double_complex_from_parts(__pyx_cur_scope->__pyx_v_t1, 0))), __pyx_cur_scope->__pyx_v_d);
+
+ /* "fontTools/cu2qu/cu2qu.py":177
+ * c1 = (2 * b * t1 + c + 3 * a * t1_2) * dt
+ * d1 = a * t1 * t1_2 + b * t1_2 + c * t1 + d
+ * yield calc_cubic_points(a1, b1, c1, d1) # <<<<<<<<<<<<<<
+ *
+ *
+*/
+ __pyx_t_1 = __pyx_f_9fontTools_5cu2qu_5cu2qu_calc_cubic_points(__pyx_cur_scope->__pyx_v_a1, __pyx_cur_scope->__pyx_v_b1, __pyx_cur_scope->__pyx_v_c1, __pyx_cur_scope->__pyx_v_d1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 177, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_1);
+ __pyx_r = __pyx_t_1;
+ __pyx_t_1 = 0;
+ __pyx_cur_scope->__pyx_t_0 = __pyx_t_12;
+ __pyx_cur_scope->__pyx_t_1 = __pyx_t_13;
+ __pyx_cur_scope->__pyx_t_2 = __pyx_t_14;
+ __Pyx_XGIVEREF(__pyx_r);
+ __Pyx_RefNannyFinishContext();
+ __Pyx_Coroutine_ResetAndClearException(__pyx_generator);
+ /* return from generator, yielding value */
+ __pyx_generator->resume_label = 1;
+ return __pyx_r;
+ __pyx_L8_resume_from_yield:;
+ __pyx_t_12 = __pyx_cur_scope->__pyx_t_0;
+ __pyx_t_13 = __pyx_cur_scope->__pyx_t_1;
+ __pyx_t_14 = __pyx_cur_scope->__pyx_t_2;
+ if (unlikely(!__pyx_sent_value)) __PYX_ERR(0, 177, __pyx_L1_error)
+ }
+ CYTHON_MAYBE_UNUSED_VAR(__pyx_cur_scope);
+
+ /* "fontTools/cu2qu/cu2qu.py":150
+ *
+ *
+ * @cython.locals( # <<<<<<<<<<<<<<
+ * p0=cython.complex,
+ * p1=cython.complex,
+*/
+
+ /* function exit code */
+ __pyx_r = Py_None; __Pyx_INCREF(Py_None);
+ goto __pyx_L0;
+ __pyx_L1_error:;
+ __Pyx_XDECREF(__pyx_t_1);
+ __Pyx_XDECREF(__pyx_t_2);
+ __Pyx_XDECREF(__pyx_t_3);
+ __Pyx_XDECREF(__pyx_t_4);
+ __Pyx_XDECREF(__pyx_t_5);
+ __Pyx_XDECREF(__pyx_t_6);
+ if (__Pyx_PyErr_Occurred()) {
+ __Pyx_Generator_Replace_StopIteration(0);
+ __Pyx_AddTraceback("_split_cubic_into_n_gen", __pyx_clineno, __pyx_lineno, __pyx_filename);
+ }
+ __pyx_L0:;
+ __Pyx_XGIVEREF(__pyx_r);
+ #if !CYTHON_USE_EXC_INFO_STACK
+ __Pyx_Coroutine_ResetAndClearException(__pyx_generator);
+ #endif
+ __pyx_generator->resume_label = -1;
+ __Pyx_Coroutine_clear((PyObject*)__pyx_generator);
+ __Pyx_RefNannyFinishContext();
+ return __pyx_r;
+}
+
+/* "fontTools/cu2qu/cu2qu.py":180
+ *
+ *
+ * @cython.cfunc # <<<<<<<<<<<<<<
+ * @cython.inline
+ * @cython.locals(
+*/
+
+static CYTHON_INLINE PyObject *__pyx_f_9fontTools_5cu2qu_5cu2qu_split_cubic_into_two(__pyx_t_double_complex __pyx_v_p0, __pyx_t_double_complex __pyx_v_p1, __pyx_t_double_complex __pyx_v_p2, __pyx_t_double_complex __pyx_v_p3) {
+ __pyx_t_double_complex __pyx_v_mid;
+ __pyx_t_double_complex __pyx_v_deriv3;
+ PyObject *__pyx_r = NULL;
+ __Pyx_RefNannyDeclarations
+ PyObject *__pyx_t_1 = NULL;
+ __pyx_t_double_complex __pyx_t_2;
+ PyObject *__pyx_t_3 = NULL;
+ PyObject *__pyx_t_4 = NULL;
+ PyObject *__pyx_t_5 = NULL;
+ PyObject *__pyx_t_6 = NULL;
+ PyObject *__pyx_t_7 = NULL;
+ int __pyx_lineno = 0;
+ const char *__pyx_filename = NULL;
+ int __pyx_clineno = 0;
+ __Pyx_RefNannySetupContext("split_cubic_into_two", 0);
+
+ /* "fontTools/cu2qu/cu2qu.py":201
+ * values).
+ * """
+ * mid = (p0 + 3 * (p1 + p2) + p3) * 0.125 # <<<<<<<<<<<<<<
+ * deriv3 = (p3 + p2 - p1 - p0) * 0.125
+ * return (
+*/
+ __pyx_v_mid = __Pyx_c_prod_double(__Pyx_c_sum_double(__Pyx_c_sum_double(__pyx_v_p0, __Pyx_c_prod_double(__pyx_t_double_complex_from_parts(3, 0), __Pyx_c_sum_double(__pyx_v_p1, __pyx_v_p2))), __pyx_v_p3), __pyx_t_double_complex_from_parts(0.125, 0));
+
+ /* "fontTools/cu2qu/cu2qu.py":202
+ * """
+ * mid = (p0 + 3 * (p1 + p2) + p3) * 0.125
+ * deriv3 = (p3 + p2 - p1 - p0) * 0.125 # <<<<<<<<<<<<<<
+ * return (
+ * (p0, (p0 + p1) * 0.5, mid - deriv3, mid),
+*/
+ __pyx_v_deriv3 = __Pyx_c_prod_double(__Pyx_c_diff_double(__Pyx_c_diff_double(__Pyx_c_sum_double(__pyx_v_p3, __pyx_v_p2), __pyx_v_p1), __pyx_v_p0), __pyx_t_double_complex_from_parts(0.125, 0));
+
+ /* "fontTools/cu2qu/cu2qu.py":203
+ * mid = (p0 + 3 * (p1 + p2) + p3) * 0.125
+ * deriv3 = (p3 + p2 - p1 - p0) * 0.125
+ * return ( # <<<<<<<<<<<<<<
+ * (p0, (p0 + p1) * 0.5, mid - deriv3, mid),
+ * (mid, mid + deriv3, (p2 + p3) * 0.5, p3),
+*/
+ __Pyx_XDECREF(__pyx_r);
+
+ /* "fontTools/cu2qu/cu2qu.py":204
+ * deriv3 = (p3 + p2 - p1 - p0) * 0.125
+ * return (
+ * (p0, (p0 + p1) * 0.5, mid - deriv3, mid), # <<<<<<<<<<<<<<
+ * (mid, mid + deriv3, (p2 + p3) * 0.5, p3),
+ * )
+*/
+ __pyx_t_1 = __pyx_PyComplex_FromComplex(__pyx_v_p0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 204, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_1);
+ __pyx_t_2 = __Pyx_c_prod_double(__Pyx_c_sum_double(__pyx_v_p0, __pyx_v_p1), __pyx_t_double_complex_from_parts(0.5, 0));
+ __pyx_t_3 = __pyx_PyComplex_FromComplex(__pyx_t_2); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 204, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_3);
+ __pyx_t_2 = __Pyx_c_diff_double(__pyx_v_mid, __pyx_v_deriv3);
+ __pyx_t_4 = __pyx_PyComplex_FromComplex(__pyx_t_2); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 204, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_4);
+ __pyx_t_5 = __pyx_PyComplex_FromComplex(__pyx_v_mid); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 204, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_5);
+ __pyx_t_6 = PyTuple_New(4); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 204, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_6);
+ __Pyx_GIVEREF(__pyx_t_1);
+ if (__Pyx_PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_1) != (0)) __PYX_ERR(0, 204, __pyx_L1_error);
+ __Pyx_GIVEREF(__pyx_t_3);
+ if (__Pyx_PyTuple_SET_ITEM(__pyx_t_6, 1, __pyx_t_3) != (0)) __PYX_ERR(0, 204, __pyx_L1_error);
+ __Pyx_GIVEREF(__pyx_t_4);
+ if (__Pyx_PyTuple_SET_ITEM(__pyx_t_6, 2, __pyx_t_4) != (0)) __PYX_ERR(0, 204, __pyx_L1_error);
+ __Pyx_GIVEREF(__pyx_t_5);
+ if (__Pyx_PyTuple_SET_ITEM(__pyx_t_6, 3, __pyx_t_5) != (0)) __PYX_ERR(0, 204, __pyx_L1_error);
+ __pyx_t_1 = 0;
+ __pyx_t_3 = 0;
+ __pyx_t_4 = 0;
+ __pyx_t_5 = 0;
+
+ /* "fontTools/cu2qu/cu2qu.py":205
+ * return (
+ * (p0, (p0 + p1) * 0.5, mid - deriv3, mid),
+ * (mid, mid + deriv3, (p2 + p3) * 0.5, p3), # <<<<<<<<<<<<<<
+ * )
+ *
+*/
+ __pyx_t_5 = __pyx_PyComplex_FromComplex(__pyx_v_mid); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 205, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_5);
+ __pyx_t_2 = __Pyx_c_sum_double(__pyx_v_mid, __pyx_v_deriv3);
+ __pyx_t_4 = __pyx_PyComplex_FromComplex(__pyx_t_2); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 205, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_4);
+ __pyx_t_2 = __Pyx_c_prod_double(__Pyx_c_sum_double(__pyx_v_p2, __pyx_v_p3), __pyx_t_double_complex_from_parts(0.5, 0));
+ __pyx_t_3 = __pyx_PyComplex_FromComplex(__pyx_t_2); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 205, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_3);
+ __pyx_t_1 = __pyx_PyComplex_FromComplex(__pyx_v_p3); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 205, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_1);
+ __pyx_t_7 = PyTuple_New(4); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 205, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_7);
+ __Pyx_GIVEREF(__pyx_t_5);
+ if (__Pyx_PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_5) != (0)) __PYX_ERR(0, 205, __pyx_L1_error);
+ __Pyx_GIVEREF(__pyx_t_4);
+ if (__Pyx_PyTuple_SET_ITEM(__pyx_t_7, 1, __pyx_t_4) != (0)) __PYX_ERR(0, 205, __pyx_L1_error);
+ __Pyx_GIVEREF(__pyx_t_3);
+ if (__Pyx_PyTuple_SET_ITEM(__pyx_t_7, 2, __pyx_t_3) != (0)) __PYX_ERR(0, 205, __pyx_L1_error);
+ __Pyx_GIVEREF(__pyx_t_1);
+ if (__Pyx_PyTuple_SET_ITEM(__pyx_t_7, 3, __pyx_t_1) != (0)) __PYX_ERR(0, 205, __pyx_L1_error);
+ __pyx_t_5 = 0;
+ __pyx_t_4 = 0;
+ __pyx_t_3 = 0;
+ __pyx_t_1 = 0;
+
+ /* "fontTools/cu2qu/cu2qu.py":204
+ * deriv3 = (p3 + p2 - p1 - p0) * 0.125
+ * return (
+ * (p0, (p0 + p1) * 0.5, mid - deriv3, mid), # <<<<<<<<<<<<<<
+ * (mid, mid + deriv3, (p2 + p3) * 0.5, p3),
+ * )
+*/
+ __pyx_t_1 = PyTuple_New(2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 204, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_1);
+ __Pyx_GIVEREF(__pyx_t_6);
+ if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_t_6) != (0)) __PYX_ERR(0, 204, __pyx_L1_error);
+ __Pyx_GIVEREF(__pyx_t_7);
+ if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 1, __pyx_t_7) != (0)) __PYX_ERR(0, 204, __pyx_L1_error);
+ __pyx_t_6 = 0;
+ __pyx_t_7 = 0;
+ __pyx_r = __pyx_t_1;
+ __pyx_t_1 = 0;
+ goto __pyx_L0;
+
+ /* "fontTools/cu2qu/cu2qu.py":180
+ *
+ *
+ * @cython.cfunc # <<<<<<<<<<<<<<
+ * @cython.inline
+ * @cython.locals(
+*/
+
+ /* function exit code */
+ __pyx_L1_error:;
+ __Pyx_XDECREF(__pyx_t_1);
+ __Pyx_XDECREF(__pyx_t_3);
+ __Pyx_XDECREF(__pyx_t_4);
+ __Pyx_XDECREF(__pyx_t_5);
+ __Pyx_XDECREF(__pyx_t_6);
+ __Pyx_XDECREF(__pyx_t_7);
+ __Pyx_AddTraceback("fontTools.cu2qu.cu2qu.split_cubic_into_two", __pyx_clineno, __pyx_lineno, __pyx_filename);
+ __pyx_r = 0;
+ __pyx_L0:;
+ __Pyx_XGIVEREF(__pyx_r);
+ __Pyx_RefNannyFinishContext();
+ return __pyx_r;
+}
+
+/* "fontTools/cu2qu/cu2qu.py":209
+ *
+ *
+ * @cython.cfunc # <<<<<<<<<<<<<<
+ * @cython.inline
+ * @cython.locals(
+*/
+
+static CYTHON_INLINE PyObject *__pyx_f_9fontTools_5cu2qu_5cu2qu_split_cubic_into_three(__pyx_t_double_complex __pyx_v_p0, __pyx_t_double_complex __pyx_v_p1, __pyx_t_double_complex __pyx_v_p2, __pyx_t_double_complex __pyx_v_p3) {
+ __pyx_t_double_complex __pyx_v_mid1;
+ __pyx_t_double_complex __pyx_v_deriv1;
+ __pyx_t_double_complex __pyx_v_mid2;
+ __pyx_t_double_complex __pyx_v_deriv2;
+ PyObject *__pyx_r = NULL;
+ __Pyx_RefNannyDeclarations
+ PyObject *__pyx_t_1 = NULL;
+ __pyx_t_double_complex __pyx_t_2;
+ __pyx_t_double_complex __pyx_t_3;
+ __pyx_t_double_complex __pyx_t_4;
+ PyObject *__pyx_t_5 = NULL;
+ PyObject *__pyx_t_6 = NULL;
+ PyObject *__pyx_t_7 = NULL;
+ PyObject *__pyx_t_8 = NULL;
+ PyObject *__pyx_t_9 = NULL;
+ PyObject *__pyx_t_10 = NULL;
+ int __pyx_lineno = 0;
+ const char *__pyx_filename = NULL;
+ int __pyx_clineno = 0;
+ __Pyx_RefNannySetupContext("split_cubic_into_three", 0);
+
+ /* "fontTools/cu2qu/cu2qu.py":238
+ * values).
+ * """
+ * mid1 = (8 * p0 + 12 * p1 + 6 * p2 + p3) * (1 / 27) # <<<<<<<<<<<<<<
+ * deriv1 = (p3 + 3 * p2 - 4 * p0) * (1 / 27)
+ * mid2 = (p0 + 6 * p1 + 12 * p2 + 8 * p3) * (1 / 27)
+*/
+ __pyx_v_mid1 = __Pyx_c_prod_double(__Pyx_c_sum_double(__Pyx_c_sum_double(__Pyx_c_sum_double(__Pyx_c_prod_double(__pyx_t_double_complex_from_parts(8, 0), __pyx_v_p0), __Pyx_c_prod_double(__pyx_t_double_complex_from_parts(12, 0), __pyx_v_p1)), __Pyx_c_prod_double(__pyx_t_double_complex_from_parts(6, 0), __pyx_v_p2)), __pyx_v_p3), __pyx_t_double_complex_from_parts((1.0 / 27.0), 0));
+
+ /* "fontTools/cu2qu/cu2qu.py":239
+ * """
+ * mid1 = (8 * p0 + 12 * p1 + 6 * p2 + p3) * (1 / 27)
+ * deriv1 = (p3 + 3 * p2 - 4 * p0) * (1 / 27) # <<<<<<<<<<<<<<
+ * mid2 = (p0 + 6 * p1 + 12 * p2 + 8 * p3) * (1 / 27)
+ * deriv2 = (4 * p3 - 3 * p1 - p0) * (1 / 27)
+*/
+ __pyx_v_deriv1 = __Pyx_c_prod_double(__Pyx_c_diff_double(__Pyx_c_sum_double(__pyx_v_p3, __Pyx_c_prod_double(__pyx_t_double_complex_from_parts(3, 0), __pyx_v_p2)), __Pyx_c_prod_double(__pyx_t_double_complex_from_parts(4, 0), __pyx_v_p0)), __pyx_t_double_complex_from_parts((1.0 / 27.0), 0));
+
+ /* "fontTools/cu2qu/cu2qu.py":240
+ * mid1 = (8 * p0 + 12 * p1 + 6 * p2 + p3) * (1 / 27)
+ * deriv1 = (p3 + 3 * p2 - 4 * p0) * (1 / 27)
+ * mid2 = (p0 + 6 * p1 + 12 * p2 + 8 * p3) * (1 / 27) # <<<<<<<<<<<<<<
+ * deriv2 = (4 * p3 - 3 * p1 - p0) * (1 / 27)
+ * return (
+*/
+ __pyx_v_mid2 = __Pyx_c_prod_double(__Pyx_c_sum_double(__Pyx_c_sum_double(__Pyx_c_sum_double(__pyx_v_p0, __Pyx_c_prod_double(__pyx_t_double_complex_from_parts(6, 0), __pyx_v_p1)), __Pyx_c_prod_double(__pyx_t_double_complex_from_parts(12, 0), __pyx_v_p2)), __Pyx_c_prod_double(__pyx_t_double_complex_from_parts(8, 0), __pyx_v_p3)), __pyx_t_double_complex_from_parts((1.0 / 27.0), 0));
+
+ /* "fontTools/cu2qu/cu2qu.py":241
+ * deriv1 = (p3 + 3 * p2 - 4 * p0) * (1 / 27)
+ * mid2 = (p0 + 6 * p1 + 12 * p2 + 8 * p3) * (1 / 27)
+ * deriv2 = (4 * p3 - 3 * p1 - p0) * (1 / 27) # <<<<<<<<<<<<<<
+ * return (
+ * (p0, (2 * p0 + p1) / 3.0, mid1 - deriv1, mid1),
+*/
+ __pyx_v_deriv2 = __Pyx_c_prod_double(__Pyx_c_diff_double(__Pyx_c_diff_double(__Pyx_c_prod_double(__pyx_t_double_complex_from_parts(4, 0), __pyx_v_p3), __Pyx_c_prod_double(__pyx_t_double_complex_from_parts(3, 0), __pyx_v_p1)), __pyx_v_p0), __pyx_t_double_complex_from_parts((1.0 / 27.0), 0));
+
+ /* "fontTools/cu2qu/cu2qu.py":242
+ * mid2 = (p0 + 6 * p1 + 12 * p2 + 8 * p3) * (1 / 27)
+ * deriv2 = (4 * p3 - 3 * p1 - p0) * (1 / 27)
+ * return ( # <<<<<<<<<<<<<<
+ * (p0, (2 * p0 + p1) / 3.0, mid1 - deriv1, mid1),
+ * (mid1, mid1 + deriv1, mid2 - deriv2, mid2),
+*/
+ __Pyx_XDECREF(__pyx_r);
+
+ /* "fontTools/cu2qu/cu2qu.py":243
+ * deriv2 = (4 * p3 - 3 * p1 - p0) * (1 / 27)
+ * return (
+ * (p0, (2 * p0 + p1) / 3.0, mid1 - deriv1, mid1), # <<<<<<<<<<<<<<
+ * (mid1, mid1 + deriv1, mid2 - deriv2, mid2),
+ * (mid2, mid2 + deriv2, (p2 + 2 * p3) / 3.0, p3),
+*/
+ __pyx_t_1 = __pyx_PyComplex_FromComplex(__pyx_v_p0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 243, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_1);
+ __pyx_t_2 = __Pyx_c_sum_double(__Pyx_c_prod_double(__pyx_t_double_complex_from_parts(2, 0), __pyx_v_p0), __pyx_v_p1);
+ __pyx_t_3 = __pyx_t_double_complex_from_parts(3.0, 0);
+ if (unlikely(__Pyx_c_is_zero_double(__pyx_t_3))) {
+ PyErr_SetString(PyExc_ZeroDivisionError, "float division");
+ __PYX_ERR(0, 243, __pyx_L1_error)
+ }
+ __pyx_t_4 = __Pyx_c_quot_double(__pyx_t_2, __pyx_t_3);
+ __pyx_t_5 = __pyx_PyComplex_FromComplex(__pyx_t_4); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 243, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_5);
+ __pyx_t_4 = __Pyx_c_diff_double(__pyx_v_mid1, __pyx_v_deriv1);
+ __pyx_t_6 = __pyx_PyComplex_FromComplex(__pyx_t_4); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 243, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_6);
+ __pyx_t_7 = __pyx_PyComplex_FromComplex(__pyx_v_mid1); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 243, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_7);
+ __pyx_t_8 = PyTuple_New(4); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 243, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_8);
+ __Pyx_GIVEREF(__pyx_t_1);
+ if (__Pyx_PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_t_1) != (0)) __PYX_ERR(0, 243, __pyx_L1_error);
+ __Pyx_GIVEREF(__pyx_t_5);
+ if (__Pyx_PyTuple_SET_ITEM(__pyx_t_8, 1, __pyx_t_5) != (0)) __PYX_ERR(0, 243, __pyx_L1_error);
+ __Pyx_GIVEREF(__pyx_t_6);
+ if (__Pyx_PyTuple_SET_ITEM(__pyx_t_8, 2, __pyx_t_6) != (0)) __PYX_ERR(0, 243, __pyx_L1_error);
+ __Pyx_GIVEREF(__pyx_t_7);
+ if (__Pyx_PyTuple_SET_ITEM(__pyx_t_8, 3, __pyx_t_7) != (0)) __PYX_ERR(0, 243, __pyx_L1_error);
+ __pyx_t_1 = 0;
+ __pyx_t_5 = 0;
+ __pyx_t_6 = 0;
+ __pyx_t_7 = 0;
+
+ /* "fontTools/cu2qu/cu2qu.py":244
+ * return (
+ * (p0, (2 * p0 + p1) / 3.0, mid1 - deriv1, mid1),
+ * (mid1, mid1 + deriv1, mid2 - deriv2, mid2), # <<<<<<<<<<<<<<
+ * (mid2, mid2 + deriv2, (p2 + 2 * p3) / 3.0, p3),
+ * )
+*/
+ __pyx_t_7 = __pyx_PyComplex_FromComplex(__pyx_v_mid1); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 244, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_7);
+ __pyx_t_4 = __Pyx_c_sum_double(__pyx_v_mid1, __pyx_v_deriv1);
+ __pyx_t_6 = __pyx_PyComplex_FromComplex(__pyx_t_4); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 244, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_6);
+ __pyx_t_4 = __Pyx_c_diff_double(__pyx_v_mid2, __pyx_v_deriv2);
+ __pyx_t_5 = __pyx_PyComplex_FromComplex(__pyx_t_4); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 244, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_5);
+ __pyx_t_1 = __pyx_PyComplex_FromComplex(__pyx_v_mid2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 244, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_1);
+ __pyx_t_9 = PyTuple_New(4); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 244, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_9);
+ __Pyx_GIVEREF(__pyx_t_7);
+ if (__Pyx_PyTuple_SET_ITEM(__pyx_t_9, 0, __pyx_t_7) != (0)) __PYX_ERR(0, 244, __pyx_L1_error);
+ __Pyx_GIVEREF(__pyx_t_6);
+ if (__Pyx_PyTuple_SET_ITEM(__pyx_t_9, 1, __pyx_t_6) != (0)) __PYX_ERR(0, 244, __pyx_L1_error);
+ __Pyx_GIVEREF(__pyx_t_5);
+ if (__Pyx_PyTuple_SET_ITEM(__pyx_t_9, 2, __pyx_t_5) != (0)) __PYX_ERR(0, 244, __pyx_L1_error);
+ __Pyx_GIVEREF(__pyx_t_1);
+ if (__Pyx_PyTuple_SET_ITEM(__pyx_t_9, 3, __pyx_t_1) != (0)) __PYX_ERR(0, 244, __pyx_L1_error);
+ __pyx_t_7 = 0;
+ __pyx_t_6 = 0;
+ __pyx_t_5 = 0;
+ __pyx_t_1 = 0;
+
+ /* "fontTools/cu2qu/cu2qu.py":245
+ * (p0, (2 * p0 + p1) / 3.0, mid1 - deriv1, mid1),
+ * (mid1, mid1 + deriv1, mid2 - deriv2, mid2),
+ * (mid2, mid2 + deriv2, (p2 + 2 * p3) / 3.0, p3), # <<<<<<<<<<<<<<
+ * )
+ *
+*/
+ __pyx_t_1 = __pyx_PyComplex_FromComplex(__pyx_v_mid2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 245, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_1);
+ __pyx_t_4 = __Pyx_c_sum_double(__pyx_v_mid2, __pyx_v_deriv2);
+ __pyx_t_5 = __pyx_PyComplex_FromComplex(__pyx_t_4); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 245, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_5);
+ __pyx_t_4 = __Pyx_c_sum_double(__pyx_v_p2, __Pyx_c_prod_double(__pyx_t_double_complex_from_parts(2, 0), __pyx_v_p3));
+ __pyx_t_3 = __pyx_t_double_complex_from_parts(3.0, 0);
+ if (unlikely(__Pyx_c_is_zero_double(__pyx_t_3))) {
+ PyErr_SetString(PyExc_ZeroDivisionError, "float division");
+ __PYX_ERR(0, 245, __pyx_L1_error)
+ }
+ __pyx_t_2 = __Pyx_c_quot_double(__pyx_t_4, __pyx_t_3);
+ __pyx_t_6 = __pyx_PyComplex_FromComplex(__pyx_t_2); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 245, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_6);
+ __pyx_t_7 = __pyx_PyComplex_FromComplex(__pyx_v_p3); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 245, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_7);
+ __pyx_t_10 = PyTuple_New(4); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 245, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_10);
+ __Pyx_GIVEREF(__pyx_t_1);
+ if (__Pyx_PyTuple_SET_ITEM(__pyx_t_10, 0, __pyx_t_1) != (0)) __PYX_ERR(0, 245, __pyx_L1_error);
+ __Pyx_GIVEREF(__pyx_t_5);
+ if (__Pyx_PyTuple_SET_ITEM(__pyx_t_10, 1, __pyx_t_5) != (0)) __PYX_ERR(0, 245, __pyx_L1_error);
+ __Pyx_GIVEREF(__pyx_t_6);
+ if (__Pyx_PyTuple_SET_ITEM(__pyx_t_10, 2, __pyx_t_6) != (0)) __PYX_ERR(0, 245, __pyx_L1_error);
+ __Pyx_GIVEREF(__pyx_t_7);
+ if (__Pyx_PyTuple_SET_ITEM(__pyx_t_10, 3, __pyx_t_7) != (0)) __PYX_ERR(0, 245, __pyx_L1_error);
+ __pyx_t_1 = 0;
+ __pyx_t_5 = 0;
+ __pyx_t_6 = 0;
+ __pyx_t_7 = 0;
+
+ /* "fontTools/cu2qu/cu2qu.py":243
+ * deriv2 = (4 * p3 - 3 * p1 - p0) * (1 / 27)
+ * return (
+ * (p0, (2 * p0 + p1) / 3.0, mid1 - deriv1, mid1), # <<<<<<<<<<<<<<
+ * (mid1, mid1 + deriv1, mid2 - deriv2, mid2),
+ * (mid2, mid2 + deriv2, (p2 + 2 * p3) / 3.0, p3),
+*/
+ __pyx_t_7 = PyTuple_New(3); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 243, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_7);
+ __Pyx_GIVEREF(__pyx_t_8);
+ if (__Pyx_PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_8) != (0)) __PYX_ERR(0, 243, __pyx_L1_error);
+ __Pyx_GIVEREF(__pyx_t_9);
+ if (__Pyx_PyTuple_SET_ITEM(__pyx_t_7, 1, __pyx_t_9) != (0)) __PYX_ERR(0, 243, __pyx_L1_error);
+ __Pyx_GIVEREF(__pyx_t_10);
+ if (__Pyx_PyTuple_SET_ITEM(__pyx_t_7, 2, __pyx_t_10) != (0)) __PYX_ERR(0, 243, __pyx_L1_error);
+ __pyx_t_8 = 0;
+ __pyx_t_9 = 0;
+ __pyx_t_10 = 0;
+ __pyx_r = __pyx_t_7;
+ __pyx_t_7 = 0;
+ goto __pyx_L0;
+
+ /* "fontTools/cu2qu/cu2qu.py":209
+ *
+ *
+ * @cython.cfunc # <<<<<<<<<<<<<<
+ * @cython.inline
+ * @cython.locals(
+*/
+
+ /* function exit code */
+ __pyx_L1_error:;
+ __Pyx_XDECREF(__pyx_t_1);
+ __Pyx_XDECREF(__pyx_t_5);
+ __Pyx_XDECREF(__pyx_t_6);
+ __Pyx_XDECREF(__pyx_t_7);
+ __Pyx_XDECREF(__pyx_t_8);
+ __Pyx_XDECREF(__pyx_t_9);
+ __Pyx_XDECREF(__pyx_t_10);
+ __Pyx_AddTraceback("fontTools.cu2qu.cu2qu.split_cubic_into_three", __pyx_clineno, __pyx_lineno, __pyx_filename);
+ __pyx_r = 0;
+ __pyx_L0:;
+ __Pyx_XGIVEREF(__pyx_r);
+ __Pyx_RefNannyFinishContext();
+ return __pyx_r;
+}
+
+/* "fontTools/cu2qu/cu2qu.py":249
+ *
+ *
+ * @cython.cfunc # <<<<<<<<<<<<<<
+ * @cython.inline
+ * @cython.returns(cython.complex)
+*/
+
+static CYTHON_INLINE __pyx_t_double_complex __pyx_f_9fontTools_5cu2qu_5cu2qu_cubic_approx_control(double __pyx_v_t, __pyx_t_double_complex __pyx_v_p0, __pyx_t_double_complex __pyx_v_p1, __pyx_t_double_complex __pyx_v_p2, __pyx_t_double_complex __pyx_v_p3) {
+ __pyx_t_double_complex __pyx_v__p1;
+ __pyx_t_double_complex __pyx_v__p2;
+ __pyx_t_double_complex __pyx_r;
+
+ /* "fontTools/cu2qu/cu2qu.py":273
+ * complex: Location of candidate control point on quadratic curve.
+ * """
+ * _p1 = p0 + (p1 - p0) * 1.5 # <<<<<<<<<<<<<<
+ * _p2 = p3 + (p2 - p3) * 1.5
+ * return _p1 + (_p2 - _p1) * t
+*/
+ __pyx_v__p1 = __Pyx_c_sum_double(__pyx_v_p0, __Pyx_c_prod_double(__Pyx_c_diff_double(__pyx_v_p1, __pyx_v_p0), __pyx_t_double_complex_from_parts(1.5, 0)));
+
+ /* "fontTools/cu2qu/cu2qu.py":274
+ * """
+ * _p1 = p0 + (p1 - p0) * 1.5
+ * _p2 = p3 + (p2 - p3) * 1.5 # <<<<<<<<<<<<<<
+ * return _p1 + (_p2 - _p1) * t
+ *
+*/
+ __pyx_v__p2 = __Pyx_c_sum_double(__pyx_v_p3, __Pyx_c_prod_double(__Pyx_c_diff_double(__pyx_v_p2, __pyx_v_p3), __pyx_t_double_complex_from_parts(1.5, 0)));
+
+ /* "fontTools/cu2qu/cu2qu.py":275
+ * _p1 = p0 + (p1 - p0) * 1.5
+ * _p2 = p3 + (p2 - p3) * 1.5
+ * return _p1 + (_p2 - _p1) * t # <<<<<<<<<<<<<<
+ *
+ *
+*/
+ __pyx_r = __Pyx_c_sum_double(__pyx_v__p1, __Pyx_c_prod_double(__Pyx_c_diff_double(__pyx_v__p2, __pyx_v__p1), __pyx_t_double_complex_from_parts(__pyx_v_t, 0)));
+ goto __pyx_L0;
+
+ /* "fontTools/cu2qu/cu2qu.py":249
+ *
+ *
+ * @cython.cfunc # <<<<<<<<<<<<<<
+ * @cython.inline
+ * @cython.returns(cython.complex)
+*/
+
+ /* function exit code */
+ __pyx_L0:;
+ return __pyx_r;
+}
+
+/* "fontTools/cu2qu/cu2qu.py":278
+ *
+ *
+ * @cython.cfunc # <<<<<<<<<<<<<<
+ * @cython.inline
+ * @cython.returns(cython.complex)
+*/
+
+static CYTHON_INLINE __pyx_t_double_complex __pyx_f_9fontTools_5cu2qu_5cu2qu_calc_intersect(__pyx_t_double_complex __pyx_v_a, __pyx_t_double_complex __pyx_v_b, __pyx_t_double_complex __pyx_v_c, __pyx_t_double_complex __pyx_v_d) {
+ __pyx_t_double_complex __pyx_v_ab;
+ __pyx_t_double_complex __pyx_v_cd;
+ __pyx_t_double_complex __pyx_v_p;
+ double __pyx_v_h;
+ __pyx_t_double_complex __pyx_r;
+ __Pyx_RefNannyDeclarations
+ PyObject *__pyx_t_1 = NULL;
+ PyObject *__pyx_t_2 = NULL;
+ PyObject *__pyx_t_3 = NULL;
+ double __pyx_t_4;
+ double __pyx_t_5;
+ int __pyx_t_6;
+ PyObject *__pyx_t_7 = NULL;
+ PyObject *__pyx_t_8 = NULL;
+ PyObject *__pyx_t_9 = NULL;
+ int __pyx_t_10;
+ int __pyx_t_11;
+ PyObject *__pyx_t_12 = NULL;
+ PyObject *__pyx_t_13 = NULL;
+ PyObject *__pyx_t_14 = NULL;
+ PyObject *__pyx_t_15 = NULL;
+ PyObject *__pyx_t_16 = NULL;
+ size_t __pyx_t_17;
+ __pyx_t_double_complex __pyx_t_18;
+ int __pyx_lineno = 0;
+ const char *__pyx_filename = NULL;
+ int __pyx_clineno = 0;
+ __Pyx_RefNannySetupContext("calc_intersect", 0);
+
+ /* "fontTools/cu2qu/cu2qu.py":296
+ * if no intersection was found.
+ * """
+ * ab = b - a # <<<<<<<<<<<<<<
+ * cd = d - c
+ * p = ab * 1j
+*/
+ __pyx_v_ab = __Pyx_c_diff_double(__pyx_v_b, __pyx_v_a);
+
+ /* "fontTools/cu2qu/cu2qu.py":297
+ * """
+ * ab = b - a
+ * cd = d - c # <<<<<<<<<<<<<<
+ * p = ab * 1j
+ * try:
+*/
+ __pyx_v_cd = __Pyx_c_diff_double(__pyx_v_d, __pyx_v_c);
+
+ /* "fontTools/cu2qu/cu2qu.py":298
+ * ab = b - a
+ * cd = d - c
+ * p = ab * 1j # <<<<<<<<<<<<<<
+ * try:
+ * h = dot(p, a - c) / dot(p, cd)
+*/
+ __pyx_v_p = __Pyx_c_prod_double(__pyx_v_ab, __pyx_t_double_complex_from_parts(0, 1.0));
+
+ /* "fontTools/cu2qu/cu2qu.py":299
+ * cd = d - c
+ * p = ab * 1j
+ * try: # <<<<<<<<<<<<<<
+ * h = dot(p, a - c) / dot(p, cd)
+ * except ZeroDivisionError:
+*/
+ {
+ __Pyx_PyThreadState_declare
+ __Pyx_PyThreadState_assign
+ __Pyx_ExceptionSave(&__pyx_t_1, &__pyx_t_2, &__pyx_t_3);
+ __Pyx_XGOTREF(__pyx_t_1);
+ __Pyx_XGOTREF(__pyx_t_2);
+ __Pyx_XGOTREF(__pyx_t_3);
+ /*try:*/ {
+
+ /* "fontTools/cu2qu/cu2qu.py":300
+ * p = ab * 1j
+ * try:
+ * h = dot(p, a - c) / dot(p, cd) # <<<<<<<<<<<<<<
+ * except ZeroDivisionError:
+ * # if 3 or 4 points are equal, we do have an intersection despite the zero-div:
+*/
+ __pyx_t_4 = __pyx_f_9fontTools_5cu2qu_5cu2qu_dot(__pyx_v_p, __Pyx_c_diff_double(__pyx_v_a, __pyx_v_c)); if (unlikely(__pyx_t_4 == ((double)-1) && PyErr_Occurred())) __PYX_ERR(0, 300, __pyx_L3_error)
+ __pyx_t_5 = __pyx_f_9fontTools_5cu2qu_5cu2qu_dot(__pyx_v_p, __pyx_v_cd); if (unlikely(__pyx_t_5 == ((double)-1) && PyErr_Occurred())) __PYX_ERR(0, 300, __pyx_L3_error)
+ if (unlikely(__pyx_t_5 == 0)) {
+ PyErr_SetString(PyExc_ZeroDivisionError, "float division");
+ __PYX_ERR(0, 300, __pyx_L3_error)
+ }
+ __pyx_v_h = (__pyx_t_4 / __pyx_t_5);
+
+ /* "fontTools/cu2qu/cu2qu.py":299
+ * cd = d - c
+ * p = ab * 1j
+ * try: # <<<<<<<<<<<<<<
+ * h = dot(p, a - c) / dot(p, cd)
+ * except ZeroDivisionError:
+*/
+ }
+ __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0;
+ __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0;
+ __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0;
+ goto __pyx_L8_try_end;
+ __pyx_L3_error:;
+
+ /* "fontTools/cu2qu/cu2qu.py":301
+ * try:
+ * h = dot(p, a - c) / dot(p, cd)
+ * except ZeroDivisionError: # <<<<<<<<<<<<<<
+ * # if 3 or 4 points are equal, we do have an intersection despite the zero-div:
+ * # return one of the off-curves so that the algorithm can attempt a one-curve
+*/
+ __pyx_t_6 = __Pyx_PyErr_ExceptionMatches(__pyx_builtin_ZeroDivisionError);
+ if (__pyx_t_6) {
+ __Pyx_AddTraceback("fontTools.cu2qu.cu2qu.calc_intersect", __pyx_clineno, __pyx_lineno, __pyx_filename);
+ if (__Pyx_GetException(&__pyx_t_7, &__pyx_t_8, &__pyx_t_9) < 0) __PYX_ERR(0, 301, __pyx_L5_except_error)
+ __Pyx_XGOTREF(__pyx_t_7);
+ __Pyx_XGOTREF(__pyx_t_8);
+ __Pyx_XGOTREF(__pyx_t_9);
+
+ /* "fontTools/cu2qu/cu2qu.py":306
+ * # solution if it's within tolerance:
+ * # https://github.com/linebender/kurbo/pull/484
+ * if b == c and (a == b or c == d): # <<<<<<<<<<<<<<
+ * return b
+ * return complex(NAN, NAN)
+*/
+ __pyx_t_11 = (__Pyx_c_eq_double(__pyx_v_b, __pyx_v_c));
+ if (__pyx_t_11) {
+ } else {
+ __pyx_t_10 = __pyx_t_11;
+ goto __pyx_L12_bool_binop_done;
+ }
+ __pyx_t_11 = (__Pyx_c_eq_double(__pyx_v_a, __pyx_v_b));
+ if (!__pyx_t_11) {
+ } else {
+ __pyx_t_10 = __pyx_t_11;
+ goto __pyx_L12_bool_binop_done;
+ }
+ __pyx_t_11 = (__Pyx_c_eq_double(__pyx_v_c, __pyx_v_d));
+ __pyx_t_10 = __pyx_t_11;
+ __pyx_L12_bool_binop_done:;
+ if (__pyx_t_10) {
+
+ /* "fontTools/cu2qu/cu2qu.py":307
+ * # https://github.com/linebender/kurbo/pull/484
+ * if b == c and (a == b or c == d):
+ * return b # <<<<<<<<<<<<<<
+ * return complex(NAN, NAN)
+ * return c + cd * h
+*/
+ __pyx_r = __pyx_v_b;
+ __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;
+ __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0;
+ __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0;
+ goto __pyx_L6_except_return;
+
+ /* "fontTools/cu2qu/cu2qu.py":306
+ * # solution if it's within tolerance:
+ * # https://github.com/linebender/kurbo/pull/484
+ * if b == c and (a == b or c == d): # <<<<<<<<<<<<<<
+ * return b
+ * return complex(NAN, NAN)
+*/
+ }
+
+ /* "fontTools/cu2qu/cu2qu.py":308
+ * if b == c and (a == b or c == d):
+ * return b
+ * return complex(NAN, NAN) # <<<<<<<<<<<<<<
+ * return c + cd * h
+ *
+*/
+ __pyx_t_13 = NULL;
+ __Pyx_INCREF((PyObject *)(&PyComplex_Type));
+ __pyx_t_14 = ((PyObject *)(&PyComplex_Type));
+ __Pyx_GetModuleGlobalName(__pyx_t_15, __pyx_mstate_global->__pyx_n_u_NAN); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 308, __pyx_L5_except_error)
+ __Pyx_GOTREF(__pyx_t_15);
+ __Pyx_GetModuleGlobalName(__pyx_t_16, __pyx_mstate_global->__pyx_n_u_NAN); if (unlikely(!__pyx_t_16)) __PYX_ERR(0, 308, __pyx_L5_except_error)
+ __Pyx_GOTREF(__pyx_t_16);
+ __pyx_t_17 = 1;
+ {
+ PyObject *__pyx_callargs[3] = {__pyx_t_13, __pyx_t_15, __pyx_t_16};
+ __pyx_t_12 = __Pyx_PyObject_FastCall(__pyx_t_14, __pyx_callargs+__pyx_t_17, (3-__pyx_t_17) | (__pyx_t_17*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
+ __Pyx_XDECREF(__pyx_t_13); __pyx_t_13 = 0;
+ __Pyx_DECREF(__pyx_t_15); __pyx_t_15 = 0;
+ __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0;
+ __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0;
+ if (unlikely(!__pyx_t_12)) __PYX_ERR(0, 308, __pyx_L5_except_error)
+ __Pyx_GOTREF(__pyx_t_12);
+ }
+ __pyx_t_18 = __Pyx_PyComplex_As___pyx_t_double_complex(__pyx_t_12); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 308, __pyx_L5_except_error)
+ __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0;
+ __pyx_r = __pyx_t_18;
+ __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;
+ __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0;
+ __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0;
+ goto __pyx_L6_except_return;
+ }
+ goto __pyx_L5_except_error;
+
+ /* "fontTools/cu2qu/cu2qu.py":299
+ * cd = d - c
+ * p = ab * 1j
+ * try: # <<<<<<<<<<<<<<
+ * h = dot(p, a - c) / dot(p, cd)
+ * except ZeroDivisionError:
+*/
+ __pyx_L5_except_error:;
+ __Pyx_XGIVEREF(__pyx_t_1);
+ __Pyx_XGIVEREF(__pyx_t_2);
+ __Pyx_XGIVEREF(__pyx_t_3);
+ __Pyx_ExceptionReset(__pyx_t_1, __pyx_t_2, __pyx_t_3);
+ goto __pyx_L1_error;
+ __pyx_L6_except_return:;
+ __Pyx_XGIVEREF(__pyx_t_1);
+ __Pyx_XGIVEREF(__pyx_t_2);
+ __Pyx_XGIVEREF(__pyx_t_3);
+ __Pyx_ExceptionReset(__pyx_t_1, __pyx_t_2, __pyx_t_3);
+ goto __pyx_L0;
+ __pyx_L8_try_end:;
+ }
+
+ /* "fontTools/cu2qu/cu2qu.py":309
+ * return b
+ * return complex(NAN, NAN)
+ * return c + cd * h # <<<<<<<<<<<<<<
+ *
+ *
+*/
+ __pyx_r = __Pyx_c_sum_double(__pyx_v_c, __Pyx_c_prod_double(__pyx_v_cd, __pyx_t_double_complex_from_parts(__pyx_v_h, 0)));
+ goto __pyx_L0;
+
+ /* "fontTools/cu2qu/cu2qu.py":278
+ *
+ *
+ * @cython.cfunc # <<<<<<<<<<<<<<
+ * @cython.inline
+ * @cython.returns(cython.complex)
+*/
+
+ /* function exit code */
+ __pyx_L1_error:;
+ __Pyx_XDECREF(__pyx_t_7);
+ __Pyx_XDECREF(__pyx_t_8);
+ __Pyx_XDECREF(__pyx_t_9);
+ __Pyx_XDECREF(__pyx_t_12);
+ __Pyx_XDECREF(__pyx_t_13);
+ __Pyx_XDECREF(__pyx_t_14);
+ __Pyx_XDECREF(__pyx_t_15);
+ __Pyx_XDECREF(__pyx_t_16);
+ __Pyx_AddTraceback("fontTools.cu2qu.cu2qu.calc_intersect", __pyx_clineno, __pyx_lineno, __pyx_filename);
+ __pyx_r = __pyx_t_double_complex_from_parts(0, 0);
+ __pyx_L0:;
+ __Pyx_RefNannyFinishContext();
+ return __pyx_r;
+}
+
+/* "fontTools/cu2qu/cu2qu.py":312
+ *
+ *
+ * @cython.cfunc # <<<<<<<<<<<<<<
+ * @cython.returns(cython.int)
+ * @cython.locals(
+*/
+
+static int __pyx_f_9fontTools_5cu2qu_5cu2qu_cubic_farthest_fit_inside(__pyx_t_double_complex __pyx_v_p0, __pyx_t_double_complex __pyx_v_p1, __pyx_t_double_complex __pyx_v_p2, __pyx_t_double_complex __pyx_v_p3, double __pyx_v_tolerance) {
+ __pyx_t_double_complex __pyx_v_mid;
+ __pyx_t_double_complex __pyx_v_deriv3;
+ int __pyx_r;
+ int __pyx_t_1;
+ int __pyx_t_2;
+ int __pyx_t_3;
+ int __pyx_t_4;
+ int __pyx_lineno = 0;
+ const char *__pyx_filename = NULL;
+ int __pyx_clineno = 0;
+
+ /* "fontTools/cu2qu/cu2qu.py":341
+ * """
+ * # First check p2 then p1, as p2 has higher error early on.
+ * if abs(p2) <= tolerance and abs(p1) <= tolerance: # <<<<<<<<<<<<<<
+ * return True
+ *
+*/
+ __pyx_t_2 = (__Pyx_c_abs_double(__pyx_v_p2) <= __pyx_v_tolerance);
+ if (__pyx_t_2) {
+ } else {
+ __pyx_t_1 = __pyx_t_2;
+ goto __pyx_L4_bool_binop_done;
+ }
+ __pyx_t_2 = (__Pyx_c_abs_double(__pyx_v_p1) <= __pyx_v_tolerance);
+ __pyx_t_1 = __pyx_t_2;
+ __pyx_L4_bool_binop_done:;
+ if (__pyx_t_1) {
+
+ /* "fontTools/cu2qu/cu2qu.py":342
+ * # First check p2 then p1, as p2 has higher error early on.
+ * if abs(p2) <= tolerance and abs(p1) <= tolerance:
+ * return True # <<<<<<<<<<<<<<
+ *
+ * # Split.
+*/
+ __pyx_r = 1;
+ goto __pyx_L0;
+
+ /* "fontTools/cu2qu/cu2qu.py":341
+ * """
+ * # First check p2 then p1, as p2 has higher error early on.
+ * if abs(p2) <= tolerance and abs(p1) <= tolerance: # <<<<<<<<<<<<<<
+ * return True
+ *
+*/
+ }
+
+ /* "fontTools/cu2qu/cu2qu.py":345
+ *
+ * # Split.
+ * mid = (p0 + 3 * (p1 + p2) + p3) * 0.125 # <<<<<<<<<<<<<<
+ * if abs(mid) > tolerance:
+ * return False
+*/
+ __pyx_v_mid = __Pyx_c_prod_double(__Pyx_c_sum_double(__Pyx_c_sum_double(__pyx_v_p0, __Pyx_c_prod_double(__pyx_t_double_complex_from_parts(3, 0), __Pyx_c_sum_double(__pyx_v_p1, __pyx_v_p2))), __pyx_v_p3), __pyx_t_double_complex_from_parts(0.125, 0));
+
+ /* "fontTools/cu2qu/cu2qu.py":346
+ * # Split.
+ * mid = (p0 + 3 * (p1 + p2) + p3) * 0.125
+ * if abs(mid) > tolerance: # <<<<<<<<<<<<<<
+ * return False
+ * deriv3 = (p3 + p2 - p1 - p0) * 0.125
+*/
+ __pyx_t_1 = (__Pyx_c_abs_double(__pyx_v_mid) > __pyx_v_tolerance);
+ if (__pyx_t_1) {
+
+ /* "fontTools/cu2qu/cu2qu.py":347
+ * mid = (p0 + 3 * (p1 + p2) + p3) * 0.125
+ * if abs(mid) > tolerance:
+ * return False # <<<<<<<<<<<<<<
+ * deriv3 = (p3 + p2 - p1 - p0) * 0.125
+ * return cubic_farthest_fit_inside(
+*/
+ __pyx_r = 0;
+ goto __pyx_L0;
+
+ /* "fontTools/cu2qu/cu2qu.py":346
+ * # Split.
+ * mid = (p0 + 3 * (p1 + p2) + p3) * 0.125
+ * if abs(mid) > tolerance: # <<<<<<<<<<<<<<
+ * return False
+ * deriv3 = (p3 + p2 - p1 - p0) * 0.125
+*/
+ }
+
+ /* "fontTools/cu2qu/cu2qu.py":348
+ * if abs(mid) > tolerance:
+ * return False
+ * deriv3 = (p3 + p2 - p1 - p0) * 0.125 # <<<<<<<<<<<<<<
+ * return cubic_farthest_fit_inside(
+ * p0, (p0 + p1) * 0.5, mid - deriv3, mid, tolerance
+*/
+ __pyx_v_deriv3 = __Pyx_c_prod_double(__Pyx_c_diff_double(__Pyx_c_diff_double(__Pyx_c_sum_double(__pyx_v_p3, __pyx_v_p2), __pyx_v_p1), __pyx_v_p0), __pyx_t_double_complex_from_parts(0.125, 0));
+
+ /* "fontTools/cu2qu/cu2qu.py":349
+ * return False
+ * deriv3 = (p3 + p2 - p1 - p0) * 0.125
+ * return cubic_farthest_fit_inside( # <<<<<<<<<<<<<<
+ * p0, (p0 + p1) * 0.5, mid - deriv3, mid, tolerance
+ * ) and cubic_farthest_fit_inside(mid, mid + deriv3, (p2 + p3) * 0.5, p3, tolerance)
+*/
+ __pyx_t_4 = __pyx_f_9fontTools_5cu2qu_5cu2qu_cubic_farthest_fit_inside(__pyx_v_p0, __Pyx_c_prod_double(__Pyx_c_sum_double(__pyx_v_p0, __pyx_v_p1), __pyx_t_double_complex_from_parts(0.5, 0)), __Pyx_c_diff_double(__pyx_v_mid, __pyx_v_deriv3), __pyx_v_mid, __pyx_v_tolerance); if (unlikely(__pyx_t_4 == ((int)-1) && PyErr_Occurred())) __PYX_ERR(0, 349, __pyx_L1_error)
+ if (__pyx_t_4) {
+ } else {
+ __pyx_t_3 = __pyx_t_4;
+ goto __pyx_L7_bool_binop_done;
+ }
+
+ /* "fontTools/cu2qu/cu2qu.py":351
+ * return cubic_farthest_fit_inside(
+ * p0, (p0 + p1) * 0.5, mid - deriv3, mid, tolerance
+ * ) and cubic_farthest_fit_inside(mid, mid + deriv3, (p2 + p3) * 0.5, p3, tolerance) # <<<<<<<<<<<<<<
+ *
+ *
+*/
+ __pyx_t_4 = __pyx_f_9fontTools_5cu2qu_5cu2qu_cubic_farthest_fit_inside(__pyx_v_mid, __Pyx_c_sum_double(__pyx_v_mid, __pyx_v_deriv3), __Pyx_c_prod_double(__Pyx_c_sum_double(__pyx_v_p2, __pyx_v_p3), __pyx_t_double_complex_from_parts(0.5, 0)), __pyx_v_p3, __pyx_v_tolerance); if (unlikely(__pyx_t_4 == ((int)-1) && PyErr_Occurred())) __PYX_ERR(0, 351, __pyx_L1_error)
+ __pyx_t_3 = __pyx_t_4;
+ __pyx_L7_bool_binop_done:;
+ __pyx_r = __pyx_t_3;
+ goto __pyx_L0;
+
+ /* "fontTools/cu2qu/cu2qu.py":312
+ *
+ *
+ * @cython.cfunc # <<<<<<<<<<<<<<
+ * @cython.returns(cython.int)
+ * @cython.locals(
+*/
+
+ /* function exit code */
+ __pyx_L1_error:;
+ __Pyx_AddTraceback("fontTools.cu2qu.cu2qu.cubic_farthest_fit_inside", __pyx_clineno, __pyx_lineno, __pyx_filename);
+ __pyx_r = -1;
+ __pyx_L0:;
+ return __pyx_r;
+}
+
+/* "fontTools/cu2qu/cu2qu.py":354
+ *
+ *
+ * @cython.cfunc # <<<<<<<<<<<<<<
+ * @cython.inline
+ * @cython.locals(tolerance=cython.double)
+*/
+
+static CYTHON_INLINE PyObject *__pyx_f_9fontTools_5cu2qu_5cu2qu_cubic_approx_quadratic(PyObject *__pyx_v_cubic, double __pyx_v_tolerance) {
+ __pyx_t_double_complex __pyx_v_q1;
+ __pyx_t_double_complex __pyx_v_c0;
+ __pyx_t_double_complex __pyx_v_c1;
+ __pyx_t_double_complex __pyx_v_c2;
+ __pyx_t_double_complex __pyx_v_c3;
+ PyObject *__pyx_r = NULL;
+ __Pyx_RefNannyDeclarations
+ PyObject *__pyx_t_1 = NULL;
+ __pyx_t_double_complex __pyx_t_2;
+ __pyx_t_double_complex __pyx_t_3;
+ __pyx_t_double_complex __pyx_t_4;
+ __pyx_t_double_complex __pyx_t_5;
+ __pyx_t_double_complex __pyx_t_6;
+ PyObject *__pyx_t_7 = NULL;
+ PyObject *__pyx_t_8 = NULL;
+ PyObject *__pyx_t_9 = NULL;
+ size_t __pyx_t_10;
+ int __pyx_t_11;
+ int __pyx_t_12;
+ int __pyx_lineno = 0;
+ const char *__pyx_filename = NULL;
+ int __pyx_clineno = 0;
+ __Pyx_RefNannySetupContext("cubic_approx_quadratic", 0);
+
+ /* "fontTools/cu2qu/cu2qu.py":378
+ * """
+ *
+ * q1 = calc_intersect(cubic[0], cubic[1], cubic[2], cubic[3]) # <<<<<<<<<<<<<<
+ * if math.isnan(q1.imag):
+ * return None
+*/
+ __pyx_t_1 = __Pyx_GetItemInt(__pyx_v_cubic, 0, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 378, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_1);
+ __pyx_t_2 = __Pyx_PyComplex_As___pyx_t_double_complex(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 378, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
+ __pyx_t_1 = __Pyx_GetItemInt(__pyx_v_cubic, 1, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 378, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_1);
+ __pyx_t_3 = __Pyx_PyComplex_As___pyx_t_double_complex(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 378, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
+ __pyx_t_1 = __Pyx_GetItemInt(__pyx_v_cubic, 2, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 378, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_1);
+ __pyx_t_4 = __Pyx_PyComplex_As___pyx_t_double_complex(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 378, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
+ __pyx_t_1 = __Pyx_GetItemInt(__pyx_v_cubic, 3, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 378, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_1);
+ __pyx_t_5 = __Pyx_PyComplex_As___pyx_t_double_complex(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 378, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
+ __pyx_t_6 = __pyx_f_9fontTools_5cu2qu_5cu2qu_calc_intersect(__pyx_t_2, __pyx_t_3, __pyx_t_4, __pyx_t_5); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 378, __pyx_L1_error)
+ __pyx_v_q1 = __pyx_t_6;
+
+ /* "fontTools/cu2qu/cu2qu.py":379
+ *
+ * q1 = calc_intersect(cubic[0], cubic[1], cubic[2], cubic[3])
+ * if math.isnan(q1.imag): # <<<<<<<<<<<<<<
+ * return None
+ * c0 = cubic[0]
+*/
+ __pyx_t_7 = NULL;
+ __Pyx_GetModuleGlobalName(__pyx_t_8, __pyx_mstate_global->__pyx_n_u_math); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 379, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_8);
+ __pyx_t_9 = __Pyx_PyObject_GetAttrStr(__pyx_t_8, __pyx_mstate_global->__pyx_n_u_isnan); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 379, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_9);
+ __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0;
+ __pyx_t_8 = PyFloat_FromDouble(__Pyx_CIMAG(__pyx_v_q1)); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 379, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_8);
+ __pyx_t_10 = 1;
+ #if CYTHON_UNPACK_METHODS
+ if (unlikely(PyMethod_Check(__pyx_t_9))) {
+ __pyx_t_7 = PyMethod_GET_SELF(__pyx_t_9);
+ assert(__pyx_t_7);
+ PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_9);
+ __Pyx_INCREF(__pyx_t_7);
+ __Pyx_INCREF(__pyx__function);
+ __Pyx_DECREF_SET(__pyx_t_9, __pyx__function);
+ __pyx_t_10 = 0;
+ }
+ #endif
+ {
+ PyObject *__pyx_callargs[2] = {__pyx_t_7, __pyx_t_8};
+ __pyx_t_1 = __Pyx_PyObject_FastCall(__pyx_t_9, __pyx_callargs+__pyx_t_10, (2-__pyx_t_10) | (__pyx_t_10*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
+ __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0;
+ __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0;
+ __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0;
+ if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 379, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_1);
+ }
+ __pyx_t_11 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely((__pyx_t_11 < 0))) __PYX_ERR(0, 379, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
+ if (__pyx_t_11) {
+
+ /* "fontTools/cu2qu/cu2qu.py":380
+ * q1 = calc_intersect(cubic[0], cubic[1], cubic[2], cubic[3])
+ * if math.isnan(q1.imag):
+ * return None # <<<<<<<<<<<<<<
+ * c0 = cubic[0]
+ * c3 = cubic[3]
+*/
+ __Pyx_XDECREF(__pyx_r);
+ __pyx_r = Py_None; __Pyx_INCREF(Py_None);
+ goto __pyx_L0;
+
+ /* "fontTools/cu2qu/cu2qu.py":379
+ *
+ * q1 = calc_intersect(cubic[0], cubic[1], cubic[2], cubic[3])
+ * if math.isnan(q1.imag): # <<<<<<<<<<<<<<
+ * return None
+ * c0 = cubic[0]
+*/
+ }
+
+ /* "fontTools/cu2qu/cu2qu.py":381
+ * if math.isnan(q1.imag):
+ * return None
+ * c0 = cubic[0] # <<<<<<<<<<<<<<
+ * c3 = cubic[3]
+ * c1 = c0 + (q1 - c0) * (2 / 3)
+*/
+ __pyx_t_1 = __Pyx_GetItemInt(__pyx_v_cubic, 0, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 381, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_1);
+ __pyx_t_6 = __Pyx_PyComplex_As___pyx_t_double_complex(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 381, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
+ __pyx_v_c0 = __pyx_t_6;
+
+ /* "fontTools/cu2qu/cu2qu.py":382
+ * return None
+ * c0 = cubic[0]
+ * c3 = cubic[3] # <<<<<<<<<<<<<<
+ * c1 = c0 + (q1 - c0) * (2 / 3)
+ * c2 = c3 + (q1 - c3) * (2 / 3)
+*/
+ __pyx_t_1 = __Pyx_GetItemInt(__pyx_v_cubic, 3, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 382, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_1);
+ __pyx_t_6 = __Pyx_PyComplex_As___pyx_t_double_complex(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 382, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
+ __pyx_v_c3 = __pyx_t_6;
+
+ /* "fontTools/cu2qu/cu2qu.py":383
+ * c0 = cubic[0]
+ * c3 = cubic[3]
+ * c1 = c0 + (q1 - c0) * (2 / 3) # <<<<<<<<<<<<<<
+ * c2 = c3 + (q1 - c3) * (2 / 3)
+ * if not cubic_farthest_fit_inside(0, c1 - cubic[1], c2 - cubic[2], 0, tolerance):
+*/
+ __pyx_v_c1 = __Pyx_c_sum_double(__pyx_v_c0, __Pyx_c_prod_double(__Pyx_c_diff_double(__pyx_v_q1, __pyx_v_c0), __pyx_t_double_complex_from_parts((2.0 / 3.0), 0)));
+
+ /* "fontTools/cu2qu/cu2qu.py":384
+ * c3 = cubic[3]
+ * c1 = c0 + (q1 - c0) * (2 / 3)
+ * c2 = c3 + (q1 - c3) * (2 / 3) # <<<<<<<<<<<<<<
+ * if not cubic_farthest_fit_inside(0, c1 - cubic[1], c2 - cubic[2], 0, tolerance):
+ * return None
+*/
+ __pyx_v_c2 = __Pyx_c_sum_double(__pyx_v_c3, __Pyx_c_prod_double(__Pyx_c_diff_double(__pyx_v_q1, __pyx_v_c3), __pyx_t_double_complex_from_parts((2.0 / 3.0), 0)));
+
+ /* "fontTools/cu2qu/cu2qu.py":385
+ * c1 = c0 + (q1 - c0) * (2 / 3)
+ * c2 = c3 + (q1 - c3) * (2 / 3)
+ * if not cubic_farthest_fit_inside(0, c1 - cubic[1], c2 - cubic[2], 0, tolerance): # <<<<<<<<<<<<<<
+ * return None
+ * return c0, q1, c3
+*/
+ __pyx_t_1 = __pyx_PyComplex_FromComplex(__pyx_v_c1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 385, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_1);
+ __pyx_t_9 = __Pyx_GetItemInt(__pyx_v_cubic, 1, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 385, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_9);
+ __pyx_t_8 = PyNumber_Subtract(__pyx_t_1, __pyx_t_9); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 385, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_8);
+ __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
+ __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0;
+ __pyx_t_6 = __Pyx_PyComplex_As___pyx_t_double_complex(__pyx_t_8); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 385, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0;
+ __pyx_t_8 = __pyx_PyComplex_FromComplex(__pyx_v_c2); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 385, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_8);
+ __pyx_t_9 = __Pyx_GetItemInt(__pyx_v_cubic, 2, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 385, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_9);
+ __pyx_t_1 = PyNumber_Subtract(__pyx_t_8, __pyx_t_9); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 385, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_1);
+ __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0;
+ __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0;
+ __pyx_t_5 = __Pyx_PyComplex_As___pyx_t_double_complex(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 385, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
+ __pyx_t_12 = __pyx_f_9fontTools_5cu2qu_5cu2qu_cubic_farthest_fit_inside(__pyx_t_double_complex_from_parts(0, 0), __pyx_t_6, __pyx_t_5, __pyx_t_double_complex_from_parts(0, 0), __pyx_v_tolerance); if (unlikely(__pyx_t_12 == ((int)-1) && PyErr_Occurred())) __PYX_ERR(0, 385, __pyx_L1_error)
+ __pyx_t_11 = (!(__pyx_t_12 != 0));
+ if (__pyx_t_11) {
+
+ /* "fontTools/cu2qu/cu2qu.py":386
+ * c2 = c3 + (q1 - c3) * (2 / 3)
+ * if not cubic_farthest_fit_inside(0, c1 - cubic[1], c2 - cubic[2], 0, tolerance):
+ * return None # <<<<<<<<<<<<<<
+ * return c0, q1, c3
+ *
+*/
+ __Pyx_XDECREF(__pyx_r);
+ __pyx_r = Py_None; __Pyx_INCREF(Py_None);
+ goto __pyx_L0;
+
+ /* "fontTools/cu2qu/cu2qu.py":385
+ * c1 = c0 + (q1 - c0) * (2 / 3)
+ * c2 = c3 + (q1 - c3) * (2 / 3)
+ * if not cubic_farthest_fit_inside(0, c1 - cubic[1], c2 - cubic[2], 0, tolerance): # <<<<<<<<<<<<<<
+ * return None
+ * return c0, q1, c3
+*/
+ }
+
+ /* "fontTools/cu2qu/cu2qu.py":387
+ * if not cubic_farthest_fit_inside(0, c1 - cubic[1], c2 - cubic[2], 0, tolerance):
+ * return None
+ * return c0, q1, c3 # <<<<<<<<<<<<<<
+ *
+ *
+*/
+ __Pyx_XDECREF(__pyx_r);
+ __pyx_t_1 = __pyx_PyComplex_FromComplex(__pyx_v_c0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 387, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_1);
+ __pyx_t_9 = __pyx_PyComplex_FromComplex(__pyx_v_q1); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 387, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_9);
+ __pyx_t_8 = __pyx_PyComplex_FromComplex(__pyx_v_c3); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 387, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_8);
+ __pyx_t_7 = PyTuple_New(3); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 387, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_7);
+ __Pyx_GIVEREF(__pyx_t_1);
+ if (__Pyx_PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_1) != (0)) __PYX_ERR(0, 387, __pyx_L1_error);
+ __Pyx_GIVEREF(__pyx_t_9);
+ if (__Pyx_PyTuple_SET_ITEM(__pyx_t_7, 1, __pyx_t_9) != (0)) __PYX_ERR(0, 387, __pyx_L1_error);
+ __Pyx_GIVEREF(__pyx_t_8);
+ if (__Pyx_PyTuple_SET_ITEM(__pyx_t_7, 2, __pyx_t_8) != (0)) __PYX_ERR(0, 387, __pyx_L1_error);
+ __pyx_t_1 = 0;
+ __pyx_t_9 = 0;
+ __pyx_t_8 = 0;
+ __pyx_r = __pyx_t_7;
+ __pyx_t_7 = 0;
+ goto __pyx_L0;
+
+ /* "fontTools/cu2qu/cu2qu.py":354
+ *
+ *
+ * @cython.cfunc # <<<<<<<<<<<<<<
+ * @cython.inline
+ * @cython.locals(tolerance=cython.double)
+*/
+
+ /* function exit code */
+ __pyx_L1_error:;
+ __Pyx_XDECREF(__pyx_t_1);
+ __Pyx_XDECREF(__pyx_t_7);
+ __Pyx_XDECREF(__pyx_t_8);
+ __Pyx_XDECREF(__pyx_t_9);
+ __Pyx_AddTraceback("fontTools.cu2qu.cu2qu.cubic_approx_quadratic", __pyx_clineno, __pyx_lineno, __pyx_filename);
+ __pyx_r = 0;
+ __pyx_L0:;
+ __Pyx_XGIVEREF(__pyx_r);
+ __Pyx_RefNannyFinishContext();
+ return __pyx_r;
+}
+
+/* "fontTools/cu2qu/cu2qu.py":390
+ *
+ *
+ * @cython.cfunc # <<<<<<<<<<<<<<
+ * @cython.locals(n=cython.int, tolerance=cython.double)
+ * @cython.locals(i=cython.int)
+*/
+
+static PyObject *__pyx_f_9fontTools_5cu2qu_5cu2qu_cubic_approx_spline(PyObject *__pyx_v_cubic, int __pyx_v_n, double __pyx_v_tolerance, int __pyx_v_all_quadratic) {
+ __pyx_t_double_complex __pyx_v_q0;
+ __pyx_t_double_complex __pyx_v_q1;
+ __pyx_t_double_complex __pyx_v_next_q1;
+ __pyx_t_double_complex __pyx_v_q2;
+ __pyx_t_double_complex __pyx_v_d1;
+ CYTHON_UNUSED __pyx_t_double_complex __pyx_v_c0;
+ __pyx_t_double_complex __pyx_v_c1;
+ __pyx_t_double_complex __pyx_v_c2;
+ __pyx_t_double_complex __pyx_v_c3;
+ int __pyx_v_i;
+ PyObject *__pyx_v_cubics = NULL;
+ PyObject *__pyx_v_next_cubic = NULL;
+ PyObject *__pyx_v_spline = NULL;
+ __pyx_t_double_complex __pyx_v_d0;
+ PyObject *__pyx_r = NULL;
+ __Pyx_RefNannyDeclarations
+ int __pyx_t_1;
+ PyObject *__pyx_t_2 = NULL;
+ int __pyx_t_3;
+ __pyx_t_double_complex __pyx_t_4;
+ __pyx_t_double_complex __pyx_t_5;
+ __pyx_t_double_complex __pyx_t_6;
+ __pyx_t_double_complex __pyx_t_7;
+ PyObject *__pyx_t_8 = NULL;
+ __pyx_t_double_complex __pyx_t_9;
+ PyObject *__pyx_t_10 = NULL;
+ long __pyx_t_11;
+ long __pyx_t_12;
+ int __pyx_t_13;
+ PyObject *__pyx_t_14 = NULL;
+ PyObject *__pyx_t_15 = NULL;
+ PyObject *(*__pyx_t_16)(PyObject *);
+ long __pyx_t_17;
+ int __pyx_t_18;
+ int __pyx_t_19;
+ int __pyx_lineno = 0;
+ const char *__pyx_filename = NULL;
+ int __pyx_clineno = 0;
+ __Pyx_RefNannySetupContext("cubic_approx_spline", 0);
+
+ /* "fontTools/cu2qu/cu2qu.py":419
+ * """
+ *
+ * if n == 1: # <<<<<<<<<<<<<<
+ * return cubic_approx_quadratic(cubic, tolerance)
+ * if n == 2 and all_quadratic == False:
+*/
+ __pyx_t_1 = (__pyx_v_n == 1);
+ if (__pyx_t_1) {
+
+ /* "fontTools/cu2qu/cu2qu.py":420
+ *
+ * if n == 1:
+ * return cubic_approx_quadratic(cubic, tolerance) # <<<<<<<<<<<<<<
+ * if n == 2 and all_quadratic == False:
+ * return cubic
+*/
+ __Pyx_XDECREF(__pyx_r);
+ __pyx_t_2 = __pyx_f_9fontTools_5cu2qu_5cu2qu_cubic_approx_quadratic(__pyx_v_cubic, __pyx_v_tolerance); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 420, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_2);
+ __pyx_r = __pyx_t_2;
+ __pyx_t_2 = 0;
+ goto __pyx_L0;
+
+ /* "fontTools/cu2qu/cu2qu.py":419
+ * """
+ *
+ * if n == 1: # <<<<<<<<<<<<<<
+ * return cubic_approx_quadratic(cubic, tolerance)
+ * if n == 2 and all_quadratic == False:
+*/
+ }
+
+ /* "fontTools/cu2qu/cu2qu.py":421
+ * if n == 1:
+ * return cubic_approx_quadratic(cubic, tolerance)
+ * if n == 2 and all_quadratic == False: # <<<<<<<<<<<<<<
+ * return cubic
+ *
+*/
+ __pyx_t_3 = (__pyx_v_n == 2);
+ if (__pyx_t_3) {
+ } else {
+ __pyx_t_1 = __pyx_t_3;
+ goto __pyx_L5_bool_binop_done;
+ }
+ __pyx_t_3 = (__pyx_v_all_quadratic == 0);
+ __pyx_t_1 = __pyx_t_3;
+ __pyx_L5_bool_binop_done:;
+ if (__pyx_t_1) {
+
+ /* "fontTools/cu2qu/cu2qu.py":422
+ * return cubic_approx_quadratic(cubic, tolerance)
+ * if n == 2 and all_quadratic == False:
+ * return cubic # <<<<<<<<<<<<<<
+ *
+ * cubics = split_cubic_into_n_iter(cubic[0], cubic[1], cubic[2], cubic[3], n)
+*/
+ __Pyx_XDECREF(__pyx_r);
+ __Pyx_INCREF(__pyx_v_cubic);
+ __pyx_r = __pyx_v_cubic;
+ goto __pyx_L0;
+
+ /* "fontTools/cu2qu/cu2qu.py":421
+ * if n == 1:
+ * return cubic_approx_quadratic(cubic, tolerance)
+ * if n == 2 and all_quadratic == False: # <<<<<<<<<<<<<<
+ * return cubic
+ *
+*/
+ }
+
+ /* "fontTools/cu2qu/cu2qu.py":424
+ * return cubic
+ *
+ * cubics = split_cubic_into_n_iter(cubic[0], cubic[1], cubic[2], cubic[3], n) # <<<<<<<<<<<<<<
+ *
+ * # calculate the spline of quadratics and check errors at the same time.
+*/
+ __pyx_t_2 = __Pyx_GetItemInt(__pyx_v_cubic, 0, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 424, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_2);
+ __pyx_t_4 = __Pyx_PyComplex_As___pyx_t_double_complex(__pyx_t_2); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 424, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+ __pyx_t_2 = __Pyx_GetItemInt(__pyx_v_cubic, 1, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 424, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_2);
+ __pyx_t_5 = __Pyx_PyComplex_As___pyx_t_double_complex(__pyx_t_2); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 424, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+ __pyx_t_2 = __Pyx_GetItemInt(__pyx_v_cubic, 2, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 424, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_2);
+ __pyx_t_6 = __Pyx_PyComplex_As___pyx_t_double_complex(__pyx_t_2); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 424, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+ __pyx_t_2 = __Pyx_GetItemInt(__pyx_v_cubic, 3, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 424, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_2);
+ __pyx_t_7 = __Pyx_PyComplex_As___pyx_t_double_complex(__pyx_t_2); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 424, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+ __pyx_t_2 = __Pyx_PyLong_From_int(__pyx_v_n); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 424, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_2);
+ __pyx_t_8 = __pyx_f_9fontTools_5cu2qu_5cu2qu_split_cubic_into_n_iter(__pyx_t_4, __pyx_t_5, __pyx_t_6, __pyx_t_7, __pyx_t_2); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 424, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_8);
+ __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+ __pyx_v_cubics = __pyx_t_8;
+ __pyx_t_8 = 0;
+
+ /* "fontTools/cu2qu/cu2qu.py":427
+ *
+ * # calculate the spline of quadratics and check errors at the same time.
+ * next_cubic = next(cubics) # <<<<<<<<<<<<<<
+ * next_q1 = cubic_approx_control(
+ * 0, next_cubic[0], next_cubic[1], next_cubic[2], next_cubic[3]
+*/
+ __pyx_t_8 = __Pyx_PyIter_Next(__pyx_v_cubics); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 427, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_8);
+ __pyx_v_next_cubic = __pyx_t_8;
+ __pyx_t_8 = 0;
+
+ /* "fontTools/cu2qu/cu2qu.py":429
+ * next_cubic = next(cubics)
+ * next_q1 = cubic_approx_control(
+ * 0, next_cubic[0], next_cubic[1], next_cubic[2], next_cubic[3] # <<<<<<<<<<<<<<
+ * )
+ * q2 = cubic[0]
+*/
+ __pyx_t_8 = __Pyx_GetItemInt(__pyx_v_next_cubic, 0, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 429, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_8);
+ __pyx_t_7 = __Pyx_PyComplex_As___pyx_t_double_complex(__pyx_t_8); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 429, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0;
+ __pyx_t_8 = __Pyx_GetItemInt(__pyx_v_next_cubic, 1, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 429, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_8);
+ __pyx_t_6 = __Pyx_PyComplex_As___pyx_t_double_complex(__pyx_t_8); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 429, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0;
+ __pyx_t_8 = __Pyx_GetItemInt(__pyx_v_next_cubic, 2, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 429, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_8);
+ __pyx_t_5 = __Pyx_PyComplex_As___pyx_t_double_complex(__pyx_t_8); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 429, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0;
+ __pyx_t_8 = __Pyx_GetItemInt(__pyx_v_next_cubic, 3, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 429, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_8);
+ __pyx_t_4 = __Pyx_PyComplex_As___pyx_t_double_complex(__pyx_t_8); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 429, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0;
+
+ /* "fontTools/cu2qu/cu2qu.py":428
+ * # calculate the spline of quadratics and check errors at the same time.
+ * next_cubic = next(cubics)
+ * next_q1 = cubic_approx_control( # <<<<<<<<<<<<<<
+ * 0, next_cubic[0], next_cubic[1], next_cubic[2], next_cubic[3]
+ * )
+*/
+ __pyx_t_9 = __pyx_f_9fontTools_5cu2qu_5cu2qu_cubic_approx_control(0.0, __pyx_t_7, __pyx_t_6, __pyx_t_5, __pyx_t_4); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 428, __pyx_L1_error)
+ __pyx_v_next_q1 = __pyx_t_9;
+
+ /* "fontTools/cu2qu/cu2qu.py":431
+ * 0, next_cubic[0], next_cubic[1], next_cubic[2], next_cubic[3]
+ * )
+ * q2 = cubic[0] # <<<<<<<<<<<<<<
+ * d1 = 0j
+ * spline = [cubic[0], next_q1]
+*/
+ __pyx_t_8 = __Pyx_GetItemInt(__pyx_v_cubic, 0, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 431, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_8);
+ __pyx_t_9 = __Pyx_PyComplex_As___pyx_t_double_complex(__pyx_t_8); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 431, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0;
+ __pyx_v_q2 = __pyx_t_9;
+
+ /* "fontTools/cu2qu/cu2qu.py":432
+ * )
+ * q2 = cubic[0]
+ * d1 = 0j # <<<<<<<<<<<<<<
+ * spline = [cubic[0], next_q1]
+ * for i in range(1, n + 1):
+*/
+ __pyx_v_d1 = __pyx_t_double_complex_from_parts(0, 0.0);
+
+ /* "fontTools/cu2qu/cu2qu.py":433
+ * q2 = cubic[0]
+ * d1 = 0j
+ * spline = [cubic[0], next_q1] # <<<<<<<<<<<<<<
+ * for i in range(1, n + 1):
+ * # Current cubic to convert
+*/
+ __pyx_t_8 = __Pyx_GetItemInt(__pyx_v_cubic, 0, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 433, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_8);
+ __pyx_t_2 = __pyx_PyComplex_FromComplex(__pyx_v_next_q1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 433, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_2);
+ __pyx_t_10 = PyList_New(2); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 433, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_10);
+ __Pyx_GIVEREF(__pyx_t_8);
+ if (__Pyx_PyList_SET_ITEM(__pyx_t_10, 0, __pyx_t_8) != (0)) __PYX_ERR(0, 433, __pyx_L1_error);
+ __Pyx_GIVEREF(__pyx_t_2);
+ if (__Pyx_PyList_SET_ITEM(__pyx_t_10, 1, __pyx_t_2) != (0)) __PYX_ERR(0, 433, __pyx_L1_error);
+ __pyx_t_8 = 0;
+ __pyx_t_2 = 0;
+ __pyx_v_spline = ((PyObject*)__pyx_t_10);
+ __pyx_t_10 = 0;
+
+ /* "fontTools/cu2qu/cu2qu.py":434
+ * d1 = 0j
+ * spline = [cubic[0], next_q1]
+ * for i in range(1, n + 1): # <<<<<<<<<<<<<<
+ * # Current cubic to convert
+ * c0, c1, c2, c3 = next_cubic
+*/
+ __pyx_t_11 = (__pyx_v_n + 1);
+ __pyx_t_12 = __pyx_t_11;
+ for (__pyx_t_13 = 1; __pyx_t_13 < __pyx_t_12; __pyx_t_13+=1) {
+ __pyx_v_i = __pyx_t_13;
+
+ /* "fontTools/cu2qu/cu2qu.py":436
+ * for i in range(1, n + 1):
+ * # Current cubic to convert
+ * c0, c1, c2, c3 = next_cubic # <<<<<<<<<<<<<<
+ *
+ * # Current quadratic approximation of current cubic
+*/
+ if ((likely(PyTuple_CheckExact(__pyx_v_next_cubic))) || (PyList_CheckExact(__pyx_v_next_cubic))) {
+ PyObject* sequence = __pyx_v_next_cubic;
+ Py_ssize_t size = __Pyx_PySequence_SIZE(sequence);
+ if (unlikely(size != 4)) {
+ if (size > 4) __Pyx_RaiseTooManyValuesError(4);
+ else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size);
+ __PYX_ERR(0, 436, __pyx_L1_error)
+ }
+ #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS
+ if (likely(PyTuple_CheckExact(sequence))) {
+ __pyx_t_10 = PyTuple_GET_ITEM(sequence, 0);
+ __Pyx_INCREF(__pyx_t_10);
+ __pyx_t_2 = PyTuple_GET_ITEM(sequence, 1);
+ __Pyx_INCREF(__pyx_t_2);
+ __pyx_t_8 = PyTuple_GET_ITEM(sequence, 2);
+ __Pyx_INCREF(__pyx_t_8);
+ __pyx_t_14 = PyTuple_GET_ITEM(sequence, 3);
+ __Pyx_INCREF(__pyx_t_14);
+ } else {
+ __pyx_t_10 = __Pyx_PyList_GetItemRef(sequence, 0);
+ if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 436, __pyx_L1_error)
+ __Pyx_XGOTREF(__pyx_t_10);
+ __pyx_t_2 = __Pyx_PyList_GetItemRef(sequence, 1);
+ if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 436, __pyx_L1_error)
+ __Pyx_XGOTREF(__pyx_t_2);
+ __pyx_t_8 = __Pyx_PyList_GetItemRef(sequence, 2);
+ if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 436, __pyx_L1_error)
+ __Pyx_XGOTREF(__pyx_t_8);
+ __pyx_t_14 = __Pyx_PyList_GetItemRef(sequence, 3);
+ if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 436, __pyx_L1_error)
+ __Pyx_XGOTREF(__pyx_t_14);
+ }
+ #else
+ {
+ Py_ssize_t i;
+ PyObject** temps[4] = {&__pyx_t_10,&__pyx_t_2,&__pyx_t_8,&__pyx_t_14};
+ for (i=0; i < 4; i++) {
+ PyObject* item = __Pyx_PySequence_ITEM(sequence, i); if (unlikely(!item)) __PYX_ERR(0, 436, __pyx_L1_error)
+ __Pyx_GOTREF(item);
+ *(temps[i]) = item;
+ }
+ }
+ #endif
+ } else {
+ Py_ssize_t index = -1;
+ PyObject** temps[4] = {&__pyx_t_10,&__pyx_t_2,&__pyx_t_8,&__pyx_t_14};
+ __pyx_t_15 = PyObject_GetIter(__pyx_v_next_cubic); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 436, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_15);
+ __pyx_t_16 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_15);
+ for (index=0; index < 4; index++) {
+ PyObject* item = __pyx_t_16(__pyx_t_15); if (unlikely(!item)) goto __pyx_L9_unpacking_failed;
+ __Pyx_GOTREF(item);
+ *(temps[index]) = item;
+ }
+ if (__Pyx_IternextUnpackEndCheck(__pyx_t_16(__pyx_t_15), 4) < 0) __PYX_ERR(0, 436, __pyx_L1_error)
+ __pyx_t_16 = NULL;
+ __Pyx_DECREF(__pyx_t_15); __pyx_t_15 = 0;
+ goto __pyx_L10_unpacking_done;
+ __pyx_L9_unpacking_failed:;
+ __Pyx_DECREF(__pyx_t_15); __pyx_t_15 = 0;
+ __pyx_t_16 = NULL;
+ if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index);
+ __PYX_ERR(0, 436, __pyx_L1_error)
+ __pyx_L10_unpacking_done:;
+ }
+ __pyx_t_9 = __Pyx_PyComplex_As___pyx_t_double_complex(__pyx_t_10); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 436, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0;
+ __pyx_t_4 = __Pyx_PyComplex_As___pyx_t_double_complex(__pyx_t_2); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 436, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+ __pyx_t_5 = __Pyx_PyComplex_As___pyx_t_double_complex(__pyx_t_8); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 436, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0;
+ __pyx_t_6 = __Pyx_PyComplex_As___pyx_t_double_complex(__pyx_t_14); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 436, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0;
+ __pyx_v_c0 = __pyx_t_9;
+ __pyx_v_c1 = __pyx_t_4;
+ __pyx_v_c2 = __pyx_t_5;
+ __pyx_v_c3 = __pyx_t_6;
+
+ /* "fontTools/cu2qu/cu2qu.py":439
+ *
+ * # Current quadratic approximation of current cubic
+ * q0 = q2 # <<<<<<<<<<<<<<
+ * q1 = next_q1
+ * if i < n:
+*/
+ __pyx_v_q0 = __pyx_v_q2;
+
+ /* "fontTools/cu2qu/cu2qu.py":440
+ * # Current quadratic approximation of current cubic
+ * q0 = q2
+ * q1 = next_q1 # <<<<<<<<<<<<<<
+ * if i < n:
+ * next_cubic = next(cubics)
+*/
+ __pyx_v_q1 = __pyx_v_next_q1;
+
+ /* "fontTools/cu2qu/cu2qu.py":441
+ * q0 = q2
+ * q1 = next_q1
+ * if i < n: # <<<<<<<<<<<<<<
+ * next_cubic = next(cubics)
+ * next_q1 = cubic_approx_control(
+*/
+ __pyx_t_1 = (__pyx_v_i < __pyx_v_n);
+ if (__pyx_t_1) {
+
+ /* "fontTools/cu2qu/cu2qu.py":442
+ * q1 = next_q1
+ * if i < n:
+ * next_cubic = next(cubics) # <<<<<<<<<<<<<<
+ * next_q1 = cubic_approx_control(
+ * i / (n - 1), next_cubic[0], next_cubic[1], next_cubic[2], next_cubic[3]
+*/
+ __pyx_t_14 = __Pyx_PyIter_Next(__pyx_v_cubics); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 442, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_14);
+ __Pyx_DECREF_SET(__pyx_v_next_cubic, __pyx_t_14);
+ __pyx_t_14 = 0;
+
+ /* "fontTools/cu2qu/cu2qu.py":444
+ * next_cubic = next(cubics)
+ * next_q1 = cubic_approx_control(
+ * i / (n - 1), next_cubic[0], next_cubic[1], next_cubic[2], next_cubic[3] # <<<<<<<<<<<<<<
+ * )
+ * spline.append(next_q1)
+*/
+ __pyx_t_17 = (__pyx_v_n - 1);
+ if (unlikely(__pyx_t_17 == 0)) {
+ PyErr_SetString(PyExc_ZeroDivisionError, "float division");
+ __PYX_ERR(0, 444, __pyx_L1_error)
+ }
+ __pyx_t_14 = __Pyx_GetItemInt(__pyx_v_next_cubic, 0, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 444, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_14);
+ __pyx_t_6 = __Pyx_PyComplex_As___pyx_t_double_complex(__pyx_t_14); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 444, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0;
+ __pyx_t_14 = __Pyx_GetItemInt(__pyx_v_next_cubic, 1, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 444, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_14);
+ __pyx_t_5 = __Pyx_PyComplex_As___pyx_t_double_complex(__pyx_t_14); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 444, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0;
+ __pyx_t_14 = __Pyx_GetItemInt(__pyx_v_next_cubic, 2, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 444, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_14);
+ __pyx_t_4 = __Pyx_PyComplex_As___pyx_t_double_complex(__pyx_t_14); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 444, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0;
+ __pyx_t_14 = __Pyx_GetItemInt(__pyx_v_next_cubic, 3, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 444, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_14);
+ __pyx_t_9 = __Pyx_PyComplex_As___pyx_t_double_complex(__pyx_t_14); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 444, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0;
+
+ /* "fontTools/cu2qu/cu2qu.py":443
+ * if i < n:
+ * next_cubic = next(cubics)
+ * next_q1 = cubic_approx_control( # <<<<<<<<<<<<<<
+ * i / (n - 1), next_cubic[0], next_cubic[1], next_cubic[2], next_cubic[3]
+ * )
+*/
+ __pyx_t_7 = __pyx_f_9fontTools_5cu2qu_5cu2qu_cubic_approx_control((((double)__pyx_v_i) / ((double)__pyx_t_17)), __pyx_t_6, __pyx_t_5, __pyx_t_4, __pyx_t_9); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 443, __pyx_L1_error)
+ __pyx_v_next_q1 = __pyx_t_7;
+
+ /* "fontTools/cu2qu/cu2qu.py":446
+ * i / (n - 1), next_cubic[0], next_cubic[1], next_cubic[2], next_cubic[3]
+ * )
+ * spline.append(next_q1) # <<<<<<<<<<<<<<
+ * q2 = (q1 + next_q1) * 0.5
+ * else:
+*/
+ __pyx_t_14 = __pyx_PyComplex_FromComplex(__pyx_v_next_q1); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 446, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_14);
+ __pyx_t_18 = __Pyx_PyList_Append(__pyx_v_spline, __pyx_t_14); if (unlikely(__pyx_t_18 == ((int)-1))) __PYX_ERR(0, 446, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0;
+
+ /* "fontTools/cu2qu/cu2qu.py":447
+ * )
+ * spline.append(next_q1)
+ * q2 = (q1 + next_q1) * 0.5 # <<<<<<<<<<<<<<
+ * else:
+ * q2 = c3
+*/
+ __pyx_v_q2 = __Pyx_c_prod_double(__Pyx_c_sum_double(__pyx_v_q1, __pyx_v_next_q1), __pyx_t_double_complex_from_parts(0.5, 0));
+
+ /* "fontTools/cu2qu/cu2qu.py":441
+ * q0 = q2
+ * q1 = next_q1
+ * if i < n: # <<<<<<<<<<<<<<
+ * next_cubic = next(cubics)
+ * next_q1 = cubic_approx_control(
+*/
+ goto __pyx_L11;
+ }
+
+ /* "fontTools/cu2qu/cu2qu.py":449
+ * q2 = (q1 + next_q1) * 0.5
+ * else:
+ * q2 = c3 # <<<<<<<<<<<<<<
+ *
+ * # End-point deltas
+*/
+ /*else*/ {
+ __pyx_v_q2 = __pyx_v_c3;
+ }
+ __pyx_L11:;
+
+ /* "fontTools/cu2qu/cu2qu.py":452
+ *
+ * # End-point deltas
+ * d0 = d1 # <<<<<<<<<<<<<<
+ * d1 = q2 - c3
+ *
+*/
+ __pyx_v_d0 = __pyx_v_d1;
+
+ /* "fontTools/cu2qu/cu2qu.py":453
+ * # End-point deltas
+ * d0 = d1
+ * d1 = q2 - c3 # <<<<<<<<<<<<<<
+ *
+ * if abs(d1) > tolerance or not cubic_farthest_fit_inside(
+*/
+ __pyx_v_d1 = __Pyx_c_diff_double(__pyx_v_q2, __pyx_v_c3);
+
+ /* "fontTools/cu2qu/cu2qu.py":455
+ * d1 = q2 - c3
+ *
+ * if abs(d1) > tolerance or not cubic_farthest_fit_inside( # <<<<<<<<<<<<<<
+ * d0,
+ * q0 + (q1 - q0) * (2 / 3) - c1,
+*/
+ __pyx_t_3 = (__Pyx_c_abs_double(__pyx_v_d1) > __pyx_v_tolerance);
+ if (!__pyx_t_3) {
+ } else {
+ __pyx_t_1 = __pyx_t_3;
+ goto __pyx_L13_bool_binop_done;
+ }
+
+ /* "fontTools/cu2qu/cu2qu.py":460
+ * q2 + (q1 - q2) * (2 / 3) - c2,
+ * d1,
+ * tolerance, # <<<<<<<<<<<<<<
+ * ):
+ * return None
+*/
+ __pyx_t_19 = __pyx_f_9fontTools_5cu2qu_5cu2qu_cubic_farthest_fit_inside(__pyx_v_d0, __Pyx_c_diff_double(__Pyx_c_sum_double(__pyx_v_q0, __Pyx_c_prod_double(__Pyx_c_diff_double(__pyx_v_q1, __pyx_v_q0), __pyx_t_double_complex_from_parts((2.0 / 3.0), 0))), __pyx_v_c1), __Pyx_c_diff_double(__Pyx_c_sum_double(__pyx_v_q2, __Pyx_c_prod_double(__Pyx_c_diff_double(__pyx_v_q1, __pyx_v_q2), __pyx_t_double_complex_from_parts((2.0 / 3.0), 0))), __pyx_v_c2), __pyx_v_d1, __pyx_v_tolerance); if (unlikely(__pyx_t_19 == ((int)-1) && PyErr_Occurred())) __PYX_ERR(0, 455, __pyx_L1_error)
+
+ /* "fontTools/cu2qu/cu2qu.py":455
+ * d1 = q2 - c3
+ *
+ * if abs(d1) > tolerance or not cubic_farthest_fit_inside( # <<<<<<<<<<<<<<
+ * d0,
+ * q0 + (q1 - q0) * (2 / 3) - c1,
+*/
+ __pyx_t_3 = (!(__pyx_t_19 != 0));
+ __pyx_t_1 = __pyx_t_3;
+ __pyx_L13_bool_binop_done:;
+ if (__pyx_t_1) {
+
+ /* "fontTools/cu2qu/cu2qu.py":462
+ * tolerance,
+ * ):
+ * return None # <<<<<<<<<<<<<<
+ * spline.append(cubic[3])
+ *
+*/
+ __Pyx_XDECREF(__pyx_r);
+ __pyx_r = Py_None; __Pyx_INCREF(Py_None);
+ goto __pyx_L0;
+
+ /* "fontTools/cu2qu/cu2qu.py":455
+ * d1 = q2 - c3
+ *
+ * if abs(d1) > tolerance or not cubic_farthest_fit_inside( # <<<<<<<<<<<<<<
+ * d0,
+ * q0 + (q1 - q0) * (2 / 3) - c1,
+*/
+ }
+ }
+
+ /* "fontTools/cu2qu/cu2qu.py":463
+ * ):
+ * return None
+ * spline.append(cubic[3]) # <<<<<<<<<<<<<<
+ *
+ * return spline
+*/
+ __pyx_t_14 = __Pyx_GetItemInt(__pyx_v_cubic, 3, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 463, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_14);
+ __pyx_t_18 = __Pyx_PyList_Append(__pyx_v_spline, __pyx_t_14); if (unlikely(__pyx_t_18 == ((int)-1))) __PYX_ERR(0, 463, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0;
+
+ /* "fontTools/cu2qu/cu2qu.py":465
+ * spline.append(cubic[3])
+ *
+ * return spline # <<<<<<<<<<<<<<
+ *
+ *
+*/
+ __Pyx_XDECREF(__pyx_r);
+ __Pyx_INCREF(__pyx_v_spline);
+ __pyx_r = __pyx_v_spline;
+ goto __pyx_L0;
+
+ /* "fontTools/cu2qu/cu2qu.py":390
+ *
+ *
+ * @cython.cfunc # <<<<<<<<<<<<<<
+ * @cython.locals(n=cython.int, tolerance=cython.double)
+ * @cython.locals(i=cython.int)
+*/
+
+ /* function exit code */
+ __pyx_L1_error:;
+ __Pyx_XDECREF(__pyx_t_2);
+ __Pyx_XDECREF(__pyx_t_8);
+ __Pyx_XDECREF(__pyx_t_10);
+ __Pyx_XDECREF(__pyx_t_14);
+ __Pyx_XDECREF(__pyx_t_15);
+ __Pyx_AddTraceback("fontTools.cu2qu.cu2qu.cubic_approx_spline", __pyx_clineno, __pyx_lineno, __pyx_filename);
+ __pyx_r = 0;
+ __pyx_L0:;
+ __Pyx_XDECREF(__pyx_v_cubics);
+ __Pyx_XDECREF(__pyx_v_next_cubic);
+ __Pyx_XDECREF(__pyx_v_spline);
+ __Pyx_XGIVEREF(__pyx_r);
+ __Pyx_RefNannyFinishContext();
+ return __pyx_r;
+}
+
+/* "fontTools/cu2qu/cu2qu.py":468
+ *
+ *
+ * @cython.locals(max_err=cython.double) # <<<<<<<<<<<<<<
+ * @cython.locals(n=cython.int)
+ * @cython.locals(all_quadratic=cython.int)
+*/
+
+/* Python wrapper */
+static PyObject *__pyx_pw_9fontTools_5cu2qu_5cu2qu_4curve_to_quadratic(PyObject *__pyx_self,
+#if CYTHON_METH_FASTCALL
+PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds
+#else
+PyObject *__pyx_args, PyObject *__pyx_kwds
+#endif
+); /*proto*/
+PyDoc_STRVAR(__pyx_doc_9fontTools_5cu2qu_5cu2qu_3curve_to_quadratic, "curve_to_quadratic(curve, double max_err, int all_quadratic=True)\n\nApproximate a cubic Bezier curve with a spline of n quadratics.\n\nArgs:\n cubic (sequence): Four 2D tuples representing control points of\n the cubic Bezier curve.\n max_err (double): Permitted deviation from the original curve.\n all_quadratic (bool): If True (default) returned value is a\n quadratic spline. If False, it's either a single quadratic\n curve or a single cubic curve.\n\nReturns:\n If all_quadratic is True: A list of 2D tuples, representing\n control points of the quadratic spline if it fits within the\n given tolerance, or ``None`` if no suitable spline could be\n calculated.\n\n If all_quadratic is False: Either a quadratic curve (if length\n of output is 3), or a cubic curve (if length of output is 4).");
+static PyMethodDef __pyx_mdef_9fontTools_5cu2qu_5cu2qu_4curve_to_quadratic = {"curve_to_quadratic", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_9fontTools_5cu2qu_5cu2qu_4curve_to_quadratic, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_9fontTools_5cu2qu_5cu2qu_3curve_to_quadratic};
+static PyObject *__pyx_pw_9fontTools_5cu2qu_5cu2qu_4curve_to_quadratic(PyObject *__pyx_self,
+#if CYTHON_METH_FASTCALL
+PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds
+#else
+PyObject *__pyx_args, PyObject *__pyx_kwds
+#endif
+) {
+ PyObject *__pyx_v_curve = 0;
+ double __pyx_v_max_err;
+ int __pyx_v_all_quadratic;
+ #if !CYTHON_METH_FASTCALL
+ CYTHON_UNUSED Py_ssize_t __pyx_nargs;
+ #endif
+ CYTHON_UNUSED PyObject *const *__pyx_kwvalues;
+ PyObject* values[3] = {0,0,0};
+ int __pyx_lineno = 0;
+ const char *__pyx_filename = NULL;
+ int __pyx_clineno = 0;
+ PyObject *__pyx_r = 0;
+ __Pyx_RefNannyDeclarations
+ __Pyx_RefNannySetupContext("curve_to_quadratic (wrapper)", 0);
+ #if !CYTHON_METH_FASTCALL
+ #if CYTHON_ASSUME_SAFE_SIZE
+ __pyx_nargs = PyTuple_GET_SIZE(__pyx_args);
+ #else
+ __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL;
+ #endif
+ #endif
+ __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs);
+ {
+ PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_curve,&__pyx_mstate_global->__pyx_n_u_max_err,&__pyx_mstate_global->__pyx_n_u_all_quadratic,0};
+ const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0;
+ if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 468, __pyx_L3_error)
+ if (__pyx_kwds_len > 0) {
+ switch (__pyx_nargs) {
+ case 3:
+ values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2);
+ if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 468, __pyx_L3_error)
+ CYTHON_FALLTHROUGH;
+ case 2:
+ values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1);
+ if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 468, __pyx_L3_error)
+ CYTHON_FALLTHROUGH;
+ case 1:
+ values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0);
+ if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 468, __pyx_L3_error)
+ CYTHON_FALLTHROUGH;
+ case 0: break;
+ default: goto __pyx_L5_argtuple_error;
+ }
+ const Py_ssize_t kwd_pos_args = __pyx_nargs;
+ if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "curve_to_quadratic", 0) < 0) __PYX_ERR(0, 468, __pyx_L3_error)
+ for (Py_ssize_t i = __pyx_nargs; i < 2; i++) {
+ if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("curve_to_quadratic", 0, 2, 3, i); __PYX_ERR(0, 468, __pyx_L3_error) }
+ }
+ } else {
+ switch (__pyx_nargs) {
+ case 3:
+ values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2);
+ if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 468, __pyx_L3_error)
+ CYTHON_FALLTHROUGH;
+ case 2:
+ values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1);
+ if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 468, __pyx_L3_error)
+ values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0);
+ if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 468, __pyx_L3_error)
+ break;
+ default: goto __pyx_L5_argtuple_error;
+ }
+ }
+ __pyx_v_curve = values[0];
+ __pyx_v_max_err = __Pyx_PyFloat_AsDouble(values[1]); if (unlikely((__pyx_v_max_err == (double)-1) && PyErr_Occurred())) __PYX_ERR(0, 471, __pyx_L3_error)
+ if (values[2]) {
+ __pyx_v_all_quadratic = __Pyx_PyLong_As_int(values[2]); if (unlikely((__pyx_v_all_quadratic == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 471, __pyx_L3_error)
+ } else {
+
+ /* "fontTools/cu2qu/cu2qu.py":471
+ * @cython.locals(n=cython.int)
+ * @cython.locals(all_quadratic=cython.int)
+ * def curve_to_quadratic(curve, max_err, all_quadratic=True): # <<<<<<<<<<<<<<
+ * """Approximate a cubic Bezier curve with a spline of n quadratics.
+ *
+*/
+ __pyx_v_all_quadratic = ((int)((int)1));
+ }
+ }
+ goto __pyx_L6_skip;
+ __pyx_L5_argtuple_error:;
+ __Pyx_RaiseArgtupleInvalid("curve_to_quadratic", 0, 2, 3, __pyx_nargs); __PYX_ERR(0, 468, __pyx_L3_error)
+ __pyx_L6_skip:;
+ goto __pyx_L4_argument_unpacking_done;
+ __pyx_L3_error:;
+ for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) {
+ Py_XDECREF(values[__pyx_temp]);
+ }
+ __Pyx_AddTraceback("fontTools.cu2qu.cu2qu.curve_to_quadratic", __pyx_clineno, __pyx_lineno, __pyx_filename);
+ __Pyx_RefNannyFinishContext();
+ return NULL;
+ __pyx_L4_argument_unpacking_done:;
+ __pyx_r = __pyx_pf_9fontTools_5cu2qu_5cu2qu_3curve_to_quadratic(__pyx_self, __pyx_v_curve, __pyx_v_max_err, __pyx_v_all_quadratic);
+
+ /* "fontTools/cu2qu/cu2qu.py":468
+ *
+ *
+ * @cython.locals(max_err=cython.double) # <<<<<<<<<<<<<<
+ * @cython.locals(n=cython.int)
+ * @cython.locals(all_quadratic=cython.int)
+*/
+
+ /* function exit code */
+ for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) {
+ Py_XDECREF(values[__pyx_temp]);
+ }
+ __Pyx_RefNannyFinishContext();
+ return __pyx_r;
+}
+
+static PyObject *__pyx_pf_9fontTools_5cu2qu_5cu2qu_3curve_to_quadratic(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_curve, double __pyx_v_max_err, int __pyx_v_all_quadratic) {
+ int __pyx_v_n;
+ PyObject *__pyx_v_spline = NULL;
+ PyObject *__pyx_7genexpr__pyx_v_p = NULL;
+ PyObject *__pyx_8genexpr1__pyx_v_s = NULL;
+ PyObject *__pyx_r = NULL;
+ __Pyx_RefNannyDeclarations
+ PyObject *__pyx_t_1 = NULL;
+ PyObject *__pyx_t_2 = NULL;
+ Py_ssize_t __pyx_t_3;
+ PyObject *(*__pyx_t_4)(PyObject *);
+ PyObject *__pyx_t_5 = NULL;
+ PyObject *__pyx_t_6 = NULL;
+ Py_ssize_t __pyx_t_7;
+ int __pyx_t_8;
+ int __pyx_t_9;
+ Py_ssize_t __pyx_t_10;
+ PyObject *__pyx_t_11 = NULL;
+ size_t __pyx_t_12;
+ int __pyx_lineno = 0;
+ const char *__pyx_filename = NULL;
+ int __pyx_clineno = 0;
+ __Pyx_RefNannySetupContext("curve_to_quadratic", 0);
+ __Pyx_INCREF(__pyx_v_curve);
+
+ /* "fontTools/cu2qu/cu2qu.py":492
+ * """
+ *
+ * curve = [complex(*p) for p in curve] # <<<<<<<<<<<<<<
+ *
+ * for n in range(1, MAX_N + 1):
+*/
+ { /* enter inner scope */
+ __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 492, __pyx_L5_error)
+ __Pyx_GOTREF(__pyx_t_1);
+ if (likely(PyList_CheckExact(__pyx_v_curve)) || PyTuple_CheckExact(__pyx_v_curve)) {
+ __pyx_t_2 = __pyx_v_curve; __Pyx_INCREF(__pyx_t_2);
+ __pyx_t_3 = 0;
+ __pyx_t_4 = NULL;
+ } else {
+ __pyx_t_3 = -1; __pyx_t_2 = PyObject_GetIter(__pyx_v_curve); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 492, __pyx_L5_error)
+ __Pyx_GOTREF(__pyx_t_2);
+ __pyx_t_4 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_2); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 492, __pyx_L5_error)
+ }
+ for (;;) {
+ if (likely(!__pyx_t_4)) {
+ if (likely(PyList_CheckExact(__pyx_t_2))) {
+ {
+ Py_ssize_t __pyx_temp = __Pyx_PyList_GET_SIZE(__pyx_t_2);
+ #if !CYTHON_ASSUME_SAFE_SIZE
+ if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 492, __pyx_L5_error)
+ #endif
+ if (__pyx_t_3 >= __pyx_temp) break;
+ }
+ __pyx_t_5 = __Pyx_PyList_GetItemRef(__pyx_t_2, __pyx_t_3);
+ ++__pyx_t_3;
+ } else {
+ {
+ Py_ssize_t __pyx_temp = __Pyx_PyTuple_GET_SIZE(__pyx_t_2);
+ #if !CYTHON_ASSUME_SAFE_SIZE
+ if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 492, __pyx_L5_error)
+ #endif
+ if (__pyx_t_3 >= __pyx_temp) break;
+ }
+ #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS
+ __pyx_t_5 = __Pyx_NewRef(PyTuple_GET_ITEM(__pyx_t_2, __pyx_t_3));
+ #else
+ __pyx_t_5 = __Pyx_PySequence_ITEM(__pyx_t_2, __pyx_t_3);
+ #endif
+ ++__pyx_t_3;
+ }
+ if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 492, __pyx_L5_error)
+ } else {
+ __pyx_t_5 = __pyx_t_4(__pyx_t_2);
+ if (unlikely(!__pyx_t_5)) {
+ PyObject* exc_type = PyErr_Occurred();
+ if (exc_type) {
+ if (unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) __PYX_ERR(0, 492, __pyx_L5_error)
+ PyErr_Clear();
+ }
+ break;
+ }
+ }
+ __Pyx_GOTREF(__pyx_t_5);
+ __Pyx_XDECREF_SET(__pyx_7genexpr__pyx_v_p, __pyx_t_5);
+ __pyx_t_5 = 0;
+ __pyx_t_5 = __Pyx_PySequence_Tuple(__pyx_7genexpr__pyx_v_p); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 492, __pyx_L5_error)
+ __Pyx_GOTREF(__pyx_t_5);
+ __pyx_t_6 = __Pyx_PyObject_Call(((PyObject *)(&PyComplex_Type)), __pyx_t_5, NULL); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 492, __pyx_L5_error)
+ __Pyx_GOTREF(__pyx_t_6);
+ __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
+ if (unlikely(__Pyx_ListComp_Append(__pyx_t_1, (PyObject*)__pyx_t_6))) __PYX_ERR(0, 492, __pyx_L5_error)
+ __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
+ }
+ __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+ __Pyx_XDECREF(__pyx_7genexpr__pyx_v_p); __pyx_7genexpr__pyx_v_p = 0;
+ goto __pyx_L9_exit_scope;
+ __pyx_L5_error:;
+ __Pyx_XDECREF(__pyx_7genexpr__pyx_v_p); __pyx_7genexpr__pyx_v_p = 0;
+ goto __pyx_L1_error;
+ __pyx_L9_exit_scope:;
+ } /* exit inner scope */
+ __Pyx_DECREF_SET(__pyx_v_curve, __pyx_t_1);
+ __pyx_t_1 = 0;
+
+ /* "fontTools/cu2qu/cu2qu.py":494
+ * curve = [complex(*p) for p in curve]
+ *
+ * for n in range(1, MAX_N + 1): # <<<<<<<<<<<<<<
+ * spline = cubic_approx_spline(curve, n, max_err, all_quadratic)
+ * if spline is not None:
+*/
+ __Pyx_GetModuleGlobalName(__pyx_t_1, __pyx_mstate_global->__pyx_n_u_MAX_N); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 494, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_1);
+ __pyx_t_2 = __Pyx_PyLong_AddObjC(__pyx_t_1, __pyx_mstate_global->__pyx_int_1, 1, 0, 0); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 494, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_2);
+ __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
+ __pyx_t_3 = __Pyx_PyIndex_AsSsize_t(__pyx_t_2); if (unlikely((__pyx_t_3 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(0, 494, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+ __pyx_t_7 = __pyx_t_3;
+ for (__pyx_t_8 = 1; __pyx_t_8 < __pyx_t_7; __pyx_t_8+=1) {
+ __pyx_v_n = __pyx_t_8;
+
+ /* "fontTools/cu2qu/cu2qu.py":495
+ *
+ * for n in range(1, MAX_N + 1):
+ * spline = cubic_approx_spline(curve, n, max_err, all_quadratic) # <<<<<<<<<<<<<<
+ * if spline is not None:
+ * # done. go home
+*/
+ __pyx_t_2 = __pyx_f_9fontTools_5cu2qu_5cu2qu_cubic_approx_spline(__pyx_v_curve, __pyx_v_n, __pyx_v_max_err, __pyx_v_all_quadratic); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 495, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_2);
+ __Pyx_XDECREF_SET(__pyx_v_spline, __pyx_t_2);
+ __pyx_t_2 = 0;
+
+ /* "fontTools/cu2qu/cu2qu.py":496
+ * for n in range(1, MAX_N + 1):
+ * spline = cubic_approx_spline(curve, n, max_err, all_quadratic)
+ * if spline is not None: # <<<<<<<<<<<<<<
+ * # done. go home
+ * return [(s.real, s.imag) for s in spline]
+*/
+ __pyx_t_9 = (__pyx_v_spline != Py_None);
+ if (__pyx_t_9) {
+
+ /* "fontTools/cu2qu/cu2qu.py":498
+ * if spline is not None:
+ * # done. go home
+ * return [(s.real, s.imag) for s in spline] # <<<<<<<<<<<<<<
+ *
+ * raise ApproxNotFoundError(curve)
+*/
+ __Pyx_XDECREF(__pyx_r);
+ { /* enter inner scope */
+ __pyx_t_2 = PyList_New(0); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 498, __pyx_L15_error)
+ __Pyx_GOTREF(__pyx_t_2);
+ if (likely(PyList_CheckExact(__pyx_v_spline)) || PyTuple_CheckExact(__pyx_v_spline)) {
+ __pyx_t_1 = __pyx_v_spline; __Pyx_INCREF(__pyx_t_1);
+ __pyx_t_10 = 0;
+ __pyx_t_4 = NULL;
+ } else {
+ __pyx_t_10 = -1; __pyx_t_1 = PyObject_GetIter(__pyx_v_spline); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 498, __pyx_L15_error)
+ __Pyx_GOTREF(__pyx_t_1);
+ __pyx_t_4 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 498, __pyx_L15_error)
+ }
+ for (;;) {
+ if (likely(!__pyx_t_4)) {
+ if (likely(PyList_CheckExact(__pyx_t_1))) {
+ {
+ Py_ssize_t __pyx_temp = __Pyx_PyList_GET_SIZE(__pyx_t_1);
+ #if !CYTHON_ASSUME_SAFE_SIZE
+ if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 498, __pyx_L15_error)
+ #endif
+ if (__pyx_t_10 >= __pyx_temp) break;
+ }
+ __pyx_t_6 = __Pyx_PyList_GetItemRef(__pyx_t_1, __pyx_t_10);
+ ++__pyx_t_10;
+ } else {
+ {
+ Py_ssize_t __pyx_temp = __Pyx_PyTuple_GET_SIZE(__pyx_t_1);
+ #if !CYTHON_ASSUME_SAFE_SIZE
+ if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 498, __pyx_L15_error)
+ #endif
+ if (__pyx_t_10 >= __pyx_temp) break;
+ }
+ #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS
+ __pyx_t_6 = __Pyx_NewRef(PyTuple_GET_ITEM(__pyx_t_1, __pyx_t_10));
+ #else
+ __pyx_t_6 = __Pyx_PySequence_ITEM(__pyx_t_1, __pyx_t_10);
+ #endif
+ ++__pyx_t_10;
+ }
+ if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 498, __pyx_L15_error)
+ } else {
+ __pyx_t_6 = __pyx_t_4(__pyx_t_1);
+ if (unlikely(!__pyx_t_6)) {
+ PyObject* exc_type = PyErr_Occurred();
+ if (exc_type) {
+ if (unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) __PYX_ERR(0, 498, __pyx_L15_error)
+ PyErr_Clear();
+ }
+ break;
+ }
+ }
+ __Pyx_GOTREF(__pyx_t_6);
+ __Pyx_XDECREF_SET(__pyx_8genexpr1__pyx_v_s, __pyx_t_6);
+ __pyx_t_6 = 0;
+ __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_8genexpr1__pyx_v_s, __pyx_mstate_global->__pyx_n_u_real); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 498, __pyx_L15_error)
+ __Pyx_GOTREF(__pyx_t_6);
+ __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_8genexpr1__pyx_v_s, __pyx_mstate_global->__pyx_n_u_imag); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 498, __pyx_L15_error)
+ __Pyx_GOTREF(__pyx_t_5);
+ __pyx_t_11 = PyTuple_New(2); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 498, __pyx_L15_error)
+ __Pyx_GOTREF(__pyx_t_11);
+ __Pyx_GIVEREF(__pyx_t_6);
+ if (__Pyx_PyTuple_SET_ITEM(__pyx_t_11, 0, __pyx_t_6) != (0)) __PYX_ERR(0, 498, __pyx_L15_error);
+ __Pyx_GIVEREF(__pyx_t_5);
+ if (__Pyx_PyTuple_SET_ITEM(__pyx_t_11, 1, __pyx_t_5) != (0)) __PYX_ERR(0, 498, __pyx_L15_error);
+ __pyx_t_6 = 0;
+ __pyx_t_5 = 0;
+ if (unlikely(__Pyx_ListComp_Append(__pyx_t_2, (PyObject*)__pyx_t_11))) __PYX_ERR(0, 498, __pyx_L15_error)
+ __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0;
+ }
+ __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
+ __Pyx_XDECREF(__pyx_8genexpr1__pyx_v_s); __pyx_8genexpr1__pyx_v_s = 0;
+ goto __pyx_L19_exit_scope;
+ __pyx_L15_error:;
+ __Pyx_XDECREF(__pyx_8genexpr1__pyx_v_s); __pyx_8genexpr1__pyx_v_s = 0;
+ goto __pyx_L1_error;
+ __pyx_L19_exit_scope:;
+ } /* exit inner scope */
+ __pyx_r = __pyx_t_2;
+ __pyx_t_2 = 0;
+ goto __pyx_L0;
+
+ /* "fontTools/cu2qu/cu2qu.py":496
+ * for n in range(1, MAX_N + 1):
+ * spline = cubic_approx_spline(curve, n, max_err, all_quadratic)
+ * if spline is not None: # <<<<<<<<<<<<<<
+ * # done. go home
+ * return [(s.real, s.imag) for s in spline]
+*/
+ }
+ }
+
+ /* "fontTools/cu2qu/cu2qu.py":500
+ * return [(s.real, s.imag) for s in spline]
+ *
+ * raise ApproxNotFoundError(curve) # <<<<<<<<<<<<<<
+ *
+ *
+*/
+ __pyx_t_1 = NULL;
+ __Pyx_GetModuleGlobalName(__pyx_t_11, __pyx_mstate_global->__pyx_n_u_ApproxNotFoundError); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 500, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_11);
+ __pyx_t_12 = 1;
+ #if CYTHON_UNPACK_METHODS
+ if (unlikely(PyMethod_Check(__pyx_t_11))) {
+ __pyx_t_1 = PyMethod_GET_SELF(__pyx_t_11);
+ assert(__pyx_t_1);
+ PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_11);
+ __Pyx_INCREF(__pyx_t_1);
+ __Pyx_INCREF(__pyx__function);
+ __Pyx_DECREF_SET(__pyx_t_11, __pyx__function);
+ __pyx_t_12 = 0;
+ }
+ #endif
+ {
+ PyObject *__pyx_callargs[2] = {__pyx_t_1, __pyx_v_curve};
+ __pyx_t_2 = __Pyx_PyObject_FastCall(__pyx_t_11, __pyx_callargs+__pyx_t_12, (2-__pyx_t_12) | (__pyx_t_12*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
+ __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0;
+ __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0;
+ if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 500, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_2);
+ }
+ __Pyx_Raise(__pyx_t_2, 0, 0, 0);
+ __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+ __PYX_ERR(0, 500, __pyx_L1_error)
+
+ /* "fontTools/cu2qu/cu2qu.py":468
+ *
+ *
+ * @cython.locals(max_err=cython.double) # <<<<<<<<<<<<<<
+ * @cython.locals(n=cython.int)
+ * @cython.locals(all_quadratic=cython.int)
+*/
+
+ /* function exit code */
+ __pyx_L1_error:;
+ __Pyx_XDECREF(__pyx_t_1);
+ __Pyx_XDECREF(__pyx_t_2);
+ __Pyx_XDECREF(__pyx_t_5);
+ __Pyx_XDECREF(__pyx_t_6);
+ __Pyx_XDECREF(__pyx_t_11);
+ __Pyx_AddTraceback("fontTools.cu2qu.cu2qu.curve_to_quadratic", __pyx_clineno, __pyx_lineno, __pyx_filename);
+ __pyx_r = NULL;
+ __pyx_L0:;
+ __Pyx_XDECREF(__pyx_v_spline);
+ __Pyx_XDECREF(__pyx_7genexpr__pyx_v_p);
+ __Pyx_XDECREF(__pyx_8genexpr1__pyx_v_s);
+ __Pyx_XDECREF(__pyx_v_curve);
+ __Pyx_XGIVEREF(__pyx_r);
+ __Pyx_RefNannyFinishContext();
+ return __pyx_r;
+}
+
+/* "fontTools/cu2qu/cu2qu.py":503
+ *
+ *
+ * @cython.locals(l=cython.int, last_i=cython.int, i=cython.int) # <<<<<<<<<<<<<<
+ * @cython.locals(all_quadratic=cython.int)
+ * def curves_to_quadratic(curves, max_errors, all_quadratic=True):
+*/
+
+/* Python wrapper */
+static PyObject *__pyx_pw_9fontTools_5cu2qu_5cu2qu_6curves_to_quadratic(PyObject *__pyx_self,
+#if CYTHON_METH_FASTCALL
+PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds
+#else
+PyObject *__pyx_args, PyObject *__pyx_kwds
+#endif
+); /*proto*/
+PyDoc_STRVAR(__pyx_doc_9fontTools_5cu2qu_5cu2qu_5curves_to_quadratic, "curves_to_quadratic(curves, max_errors, int all_quadratic=True)\n\nReturn quadratic Bezier splines approximating the input cubic Beziers.\n\nArgs:\n curves: A sequence of *n* curves, each curve being a sequence of four\n 2D tuples.\n max_errors: A sequence of *n* floats representing the maximum permissible\n deviation from each of the cubic Bezier curves.\n all_quadratic (bool): If True (default) returned values are a\n quadratic spline. If False, they are either a single quadratic\n curve or a single cubic curve.\n\nExample::\n\n >>> curves_to_quadratic( [\n ... [ (50,50), (100,100), (150,100), (200,50) ],\n ... [ (75,50), (120,100), (150,75), (200,60) ]\n ... ], [1,1] )\n [[(50.0, 50.0), (75.0, 75.0), (125.0, 91.66666666666666), (175.0, 75.0), (200.0, 50.0)], [(75.0, 50.0), (97.5, 75.0), (135.41666666666666, 82.08333333333333), (175.0, 67.5), (200.0, 60.0)]]\n\nThe returned splines have \"implied oncurve points\" suitable for use in\nTrueType ``glif`` outlines - i.e. in the first spline returned above,\nthe first quadratic segment runs from (50,50) to\n( (75 + 125)/2 , (120 + 91.666..)/2 ) = (100, 83.333...).\n\nReturns:\n If all_quadratic is True, a list of splines, each spline being a list\n of 2D tuples.\n\n If all_quadratic is False, a list of curves, each curve being a quadratic\n (length 3), or cubic (length 4).\n\nRaises:\n fontTools.cu2qu.Errors.ApproxNotFoundError: if no suitable approximation\n can be found for all curves with the given parameters.");
+static PyMethodDef __pyx_mdef_9fontTools_5cu2qu_5cu2qu_6curves_to_quadratic = {"curves_to_quadratic", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_9fontTools_5cu2qu_5cu2qu_6curves_to_quadratic, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_9fontTools_5cu2qu_5cu2qu_5curves_to_quadratic};
+static PyObject *__pyx_pw_9fontTools_5cu2qu_5cu2qu_6curves_to_quadratic(PyObject *__pyx_self,
+#if CYTHON_METH_FASTCALL
+PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds
+#else
+PyObject *__pyx_args, PyObject *__pyx_kwds
+#endif
+) {
+ PyObject *__pyx_v_curves = 0;
+ PyObject *__pyx_v_max_errors = 0;
+ int __pyx_v_all_quadratic;
+ #if !CYTHON_METH_FASTCALL
+ CYTHON_UNUSED Py_ssize_t __pyx_nargs;
+ #endif
+ CYTHON_UNUSED PyObject *const *__pyx_kwvalues;
+ PyObject* values[3] = {0,0,0};
+ int __pyx_lineno = 0;
+ const char *__pyx_filename = NULL;
+ int __pyx_clineno = 0;
+ PyObject *__pyx_r = 0;
+ __Pyx_RefNannyDeclarations
+ __Pyx_RefNannySetupContext("curves_to_quadratic (wrapper)", 0);
+ #if !CYTHON_METH_FASTCALL
+ #if CYTHON_ASSUME_SAFE_SIZE
+ __pyx_nargs = PyTuple_GET_SIZE(__pyx_args);
+ #else
+ __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL;
+ #endif
+ #endif
+ __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs);
+ {
+ PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_curves,&__pyx_mstate_global->__pyx_n_u_max_errors,&__pyx_mstate_global->__pyx_n_u_all_quadratic,0};
+ const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0;
+ if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 503, __pyx_L3_error)
+ if (__pyx_kwds_len > 0) {
+ switch (__pyx_nargs) {
+ case 3:
+ values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2);
+ if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 503, __pyx_L3_error)
+ CYTHON_FALLTHROUGH;
+ case 2:
+ values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1);
+ if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 503, __pyx_L3_error)
+ CYTHON_FALLTHROUGH;
+ case 1:
+ values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0);
+ if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 503, __pyx_L3_error)
+ CYTHON_FALLTHROUGH;
+ case 0: break;
+ default: goto __pyx_L5_argtuple_error;
+ }
+ const Py_ssize_t kwd_pos_args = __pyx_nargs;
+ if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "curves_to_quadratic", 0) < 0) __PYX_ERR(0, 503, __pyx_L3_error)
+ for (Py_ssize_t i = __pyx_nargs; i < 2; i++) {
+ if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("curves_to_quadratic", 0, 2, 3, i); __PYX_ERR(0, 503, __pyx_L3_error) }
+ }
+ } else {
+ switch (__pyx_nargs) {
+ case 3:
+ values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2);
+ if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 503, __pyx_L3_error)
+ CYTHON_FALLTHROUGH;
+ case 2:
+ values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1);
+ if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 503, __pyx_L3_error)
+ values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0);
+ if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 503, __pyx_L3_error)
+ break;
+ default: goto __pyx_L5_argtuple_error;
+ }
+ }
+ __pyx_v_curves = values[0];
+ __pyx_v_max_errors = values[1];
+ if (values[2]) {
+ __pyx_v_all_quadratic = __Pyx_PyLong_As_int(values[2]); if (unlikely((__pyx_v_all_quadratic == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 505, __pyx_L3_error)
+ } else {
+
+ /* "fontTools/cu2qu/cu2qu.py":505
+ * @cython.locals(l=cython.int, last_i=cython.int, i=cython.int)
+ * @cython.locals(all_quadratic=cython.int)
+ * def curves_to_quadratic(curves, max_errors, all_quadratic=True): # <<<<<<<<<<<<<<
+ * """Return quadratic Bezier splines approximating the input cubic Beziers.
+ *
+*/
+ __pyx_v_all_quadratic = ((int)((int)1));
+ }
+ }
+ goto __pyx_L6_skip;
+ __pyx_L5_argtuple_error:;
+ __Pyx_RaiseArgtupleInvalid("curves_to_quadratic", 0, 2, 3, __pyx_nargs); __PYX_ERR(0, 503, __pyx_L3_error)
+ __pyx_L6_skip:;
+ goto __pyx_L4_argument_unpacking_done;
+ __pyx_L3_error:;
+ for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) {
+ Py_XDECREF(values[__pyx_temp]);
+ }
+ __Pyx_AddTraceback("fontTools.cu2qu.cu2qu.curves_to_quadratic", __pyx_clineno, __pyx_lineno, __pyx_filename);
+ __Pyx_RefNannyFinishContext();
+ return NULL;
+ __pyx_L4_argument_unpacking_done:;
+ __pyx_r = __pyx_pf_9fontTools_5cu2qu_5cu2qu_5curves_to_quadratic(__pyx_self, __pyx_v_curves, __pyx_v_max_errors, __pyx_v_all_quadratic);
+
+ /* "fontTools/cu2qu/cu2qu.py":503
+ *
+ *
+ * @cython.locals(l=cython.int, last_i=cython.int, i=cython.int) # <<<<<<<<<<<<<<
+ * @cython.locals(all_quadratic=cython.int)
+ * def curves_to_quadratic(curves, max_errors, all_quadratic=True):
+*/
+
+ /* function exit code */
+ for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) {
+ Py_XDECREF(values[__pyx_temp]);
+ }
+ __Pyx_RefNannyFinishContext();
+ return __pyx_r;
+}
+
+static PyObject *__pyx_pf_9fontTools_5cu2qu_5cu2qu_5curves_to_quadratic(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_curves, PyObject *__pyx_v_max_errors, int __pyx_v_all_quadratic) {
+ int __pyx_v_l;
+ int __pyx_v_last_i;
+ int __pyx_v_i;
+ PyObject *__pyx_v_splines = NULL;
+ PyObject *__pyx_v_n = NULL;
+ PyObject *__pyx_v_spline = NULL;
+ PyObject *__pyx_8genexpr2__pyx_v_curve = NULL;
+ PyObject *__pyx_8genexpr3__pyx_v_p = NULL;
+ PyObject *__pyx_8genexpr4__pyx_v_spline = NULL;
+ PyObject *__pyx_8genexpr5__pyx_v_s = NULL;
+ PyObject *__pyx_r = NULL;
+ __Pyx_RefNannyDeclarations
+ PyObject *__pyx_t_1 = NULL;
+ PyObject *__pyx_t_2 = NULL;
+ Py_ssize_t __pyx_t_3;
+ PyObject *(*__pyx_t_4)(PyObject *);
+ PyObject *__pyx_t_5 = NULL;
+ PyObject *__pyx_t_6 = NULL;
+ Py_ssize_t __pyx_t_7;
+ PyObject *(*__pyx_t_8)(PyObject *);
+ PyObject *__pyx_t_9 = NULL;
+ PyObject *__pyx_t_10 = NULL;
+ int __pyx_t_11;
+ int __pyx_t_12;
+ double __pyx_t_13;
+ long __pyx_t_14;
+ PyObject *__pyx_t_15 = NULL;
+ size_t __pyx_t_16;
+ int __pyx_lineno = 0;
+ const char *__pyx_filename = NULL;
+ int __pyx_clineno = 0;
+ __Pyx_RefNannySetupContext("curves_to_quadratic", 0);
+ __Pyx_INCREF(__pyx_v_curves);
+
+ /* "fontTools/cu2qu/cu2qu.py":542
+ * """
+ *
+ * curves = [[complex(*p) for p in curve] for curve in curves] # <<<<<<<<<<<<<<
+ * assert len(max_errors) == len(curves)
+ *
+*/
+ { /* enter inner scope */
+ __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 542, __pyx_L5_error)
+ __Pyx_GOTREF(__pyx_t_1);
+ if (likely(PyList_CheckExact(__pyx_v_curves)) || PyTuple_CheckExact(__pyx_v_curves)) {
+ __pyx_t_2 = __pyx_v_curves; __Pyx_INCREF(__pyx_t_2);
+ __pyx_t_3 = 0;
+ __pyx_t_4 = NULL;
+ } else {
+ __pyx_t_3 = -1; __pyx_t_2 = PyObject_GetIter(__pyx_v_curves); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 542, __pyx_L5_error)
+ __Pyx_GOTREF(__pyx_t_2);
+ __pyx_t_4 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_2); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 542, __pyx_L5_error)
+ }
+ for (;;) {
+ if (likely(!__pyx_t_4)) {
+ if (likely(PyList_CheckExact(__pyx_t_2))) {
+ {
+ Py_ssize_t __pyx_temp = __Pyx_PyList_GET_SIZE(__pyx_t_2);
+ #if !CYTHON_ASSUME_SAFE_SIZE
+ if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 542, __pyx_L5_error)
+ #endif
+ if (__pyx_t_3 >= __pyx_temp) break;
+ }
+ __pyx_t_5 = __Pyx_PyList_GetItemRef(__pyx_t_2, __pyx_t_3);
+ ++__pyx_t_3;
+ } else {
+ {
+ Py_ssize_t __pyx_temp = __Pyx_PyTuple_GET_SIZE(__pyx_t_2);
+ #if !CYTHON_ASSUME_SAFE_SIZE
+ if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 542, __pyx_L5_error)
+ #endif
+ if (__pyx_t_3 >= __pyx_temp) break;
+ }
+ #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS
+ __pyx_t_5 = __Pyx_NewRef(PyTuple_GET_ITEM(__pyx_t_2, __pyx_t_3));
+ #else
+ __pyx_t_5 = __Pyx_PySequence_ITEM(__pyx_t_2, __pyx_t_3);
+ #endif
+ ++__pyx_t_3;
+ }
+ if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 542, __pyx_L5_error)
+ } else {
+ __pyx_t_5 = __pyx_t_4(__pyx_t_2);
+ if (unlikely(!__pyx_t_5)) {
+ PyObject* exc_type = PyErr_Occurred();
+ if (exc_type) {
+ if (unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) __PYX_ERR(0, 542, __pyx_L5_error)
+ PyErr_Clear();
+ }
+ break;
+ }
+ }
+ __Pyx_GOTREF(__pyx_t_5);
+ __Pyx_XDECREF_SET(__pyx_8genexpr2__pyx_v_curve, __pyx_t_5);
+ __pyx_t_5 = 0;
+ { /* enter inner scope */
+ __pyx_t_5 = PyList_New(0); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 542, __pyx_L10_error)
+ __Pyx_GOTREF(__pyx_t_5);
+ if (likely(PyList_CheckExact(__pyx_8genexpr2__pyx_v_curve)) || PyTuple_CheckExact(__pyx_8genexpr2__pyx_v_curve)) {
+ __pyx_t_6 = __pyx_8genexpr2__pyx_v_curve; __Pyx_INCREF(__pyx_t_6);
+ __pyx_t_7 = 0;
+ __pyx_t_8 = NULL;
+ } else {
+ __pyx_t_7 = -1; __pyx_t_6 = PyObject_GetIter(__pyx_8genexpr2__pyx_v_curve); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 542, __pyx_L10_error)
+ __Pyx_GOTREF(__pyx_t_6);
+ __pyx_t_8 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_6); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 542, __pyx_L10_error)
+ }
+ for (;;) {
+ if (likely(!__pyx_t_8)) {
+ if (likely(PyList_CheckExact(__pyx_t_6))) {
+ {
+ Py_ssize_t __pyx_temp = __Pyx_PyList_GET_SIZE(__pyx_t_6);
+ #if !CYTHON_ASSUME_SAFE_SIZE
+ if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 542, __pyx_L10_error)
+ #endif
+ if (__pyx_t_7 >= __pyx_temp) break;
+ }
+ __pyx_t_9 = __Pyx_PyList_GetItemRef(__pyx_t_6, __pyx_t_7);
+ ++__pyx_t_7;
+ } else {
+ {
+ Py_ssize_t __pyx_temp = __Pyx_PyTuple_GET_SIZE(__pyx_t_6);
+ #if !CYTHON_ASSUME_SAFE_SIZE
+ if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 542, __pyx_L10_error)
+ #endif
+ if (__pyx_t_7 >= __pyx_temp) break;
+ }
+ #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS
+ __pyx_t_9 = __Pyx_NewRef(PyTuple_GET_ITEM(__pyx_t_6, __pyx_t_7));
+ #else
+ __pyx_t_9 = __Pyx_PySequence_ITEM(__pyx_t_6, __pyx_t_7);
+ #endif
+ ++__pyx_t_7;
+ }
+ if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 542, __pyx_L10_error)
+ } else {
+ __pyx_t_9 = __pyx_t_8(__pyx_t_6);
+ if (unlikely(!__pyx_t_9)) {
+ PyObject* exc_type = PyErr_Occurred();
+ if (exc_type) {
+ if (unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) __PYX_ERR(0, 542, __pyx_L10_error)
+ PyErr_Clear();
+ }
+ break;
+ }
+ }
+ __Pyx_GOTREF(__pyx_t_9);
+ __Pyx_XDECREF_SET(__pyx_8genexpr3__pyx_v_p, __pyx_t_9);
+ __pyx_t_9 = 0;
+ __pyx_t_9 = __Pyx_PySequence_Tuple(__pyx_8genexpr3__pyx_v_p); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 542, __pyx_L10_error)
+ __Pyx_GOTREF(__pyx_t_9);
+ __pyx_t_10 = __Pyx_PyObject_Call(((PyObject *)(&PyComplex_Type)), __pyx_t_9, NULL); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 542, __pyx_L10_error)
+ __Pyx_GOTREF(__pyx_t_10);
+ __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0;
+ if (unlikely(__Pyx_ListComp_Append(__pyx_t_5, (PyObject*)__pyx_t_10))) __PYX_ERR(0, 542, __pyx_L10_error)
+ __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0;
+ }
+ __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
+ __Pyx_XDECREF(__pyx_8genexpr3__pyx_v_p); __pyx_8genexpr3__pyx_v_p = 0;
+ goto __pyx_L14_exit_scope;
+ __pyx_L10_error:;
+ __Pyx_XDECREF(__pyx_8genexpr3__pyx_v_p); __pyx_8genexpr3__pyx_v_p = 0;
+ goto __pyx_L5_error;
+ __pyx_L14_exit_scope:;
+ } /* exit inner scope */
+ if (unlikely(__Pyx_ListComp_Append(__pyx_t_1, (PyObject*)__pyx_t_5))) __PYX_ERR(0, 542, __pyx_L5_error)
+ __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
+ }
+ __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+ __Pyx_XDECREF(__pyx_8genexpr2__pyx_v_curve); __pyx_8genexpr2__pyx_v_curve = 0;
+ goto __pyx_L16_exit_scope;
+ __pyx_L5_error:;
+ __Pyx_XDECREF(__pyx_8genexpr2__pyx_v_curve); __pyx_8genexpr2__pyx_v_curve = 0;
+ goto __pyx_L1_error;
+ __pyx_L16_exit_scope:;
+ } /* exit inner scope */
+ __Pyx_DECREF_SET(__pyx_v_curves, __pyx_t_1);
+ __pyx_t_1 = 0;
+
+ /* "fontTools/cu2qu/cu2qu.py":543
+ *
+ * curves = [[complex(*p) for p in curve] for curve in curves]
+ * assert len(max_errors) == len(curves) # <<<<<<<<<<<<<<
+ *
+ * l = len(curves)
+*/
+ #ifndef CYTHON_WITHOUT_ASSERTIONS
+ if (unlikely(__pyx_assertions_enabled())) {
+ __pyx_t_3 = PyObject_Length(__pyx_v_max_errors); if (unlikely(__pyx_t_3 == ((Py_ssize_t)-1))) __PYX_ERR(0, 543, __pyx_L1_error)
+ __pyx_t_7 = PyObject_Length(__pyx_v_curves); if (unlikely(__pyx_t_7 == ((Py_ssize_t)-1))) __PYX_ERR(0, 543, __pyx_L1_error)
+ __pyx_t_11 = (__pyx_t_3 == __pyx_t_7);
+ if (unlikely(!__pyx_t_11)) {
+ __Pyx_Raise(__pyx_builtin_AssertionError, 0, 0, 0);
+ __PYX_ERR(0, 543, __pyx_L1_error)
+ }
+ }
+ #else
+ if ((1)); else __PYX_ERR(0, 543, __pyx_L1_error)
+ #endif
+
+ /* "fontTools/cu2qu/cu2qu.py":545
+ * assert len(max_errors) == len(curves)
+ *
+ * l = len(curves) # <<<<<<<<<<<<<<
+ * splines = [None] * l
+ * last_i = i = 0
+*/
+ __pyx_t_7 = PyObject_Length(__pyx_v_curves); if (unlikely(__pyx_t_7 == ((Py_ssize_t)-1))) __PYX_ERR(0, 545, __pyx_L1_error)
+ __pyx_v_l = __pyx_t_7;
+
+ /* "fontTools/cu2qu/cu2qu.py":546
+ *
+ * l = len(curves)
+ * splines = [None] * l # <<<<<<<<<<<<<<
+ * last_i = i = 0
+ * n = 1
+*/
+ __pyx_t_1 = PyList_New(1 * ((__pyx_v_l<0) ? 0:__pyx_v_l)); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 546, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_1);
+ { Py_ssize_t __pyx_temp;
+ for (__pyx_temp=0; __pyx_temp < __pyx_v_l; __pyx_temp++) {
+ __Pyx_INCREF(Py_None);
+ __Pyx_GIVEREF(Py_None);
+ if (__Pyx_PyList_SET_ITEM(__pyx_t_1, __pyx_temp, Py_None) != (0)) __PYX_ERR(0, 546, __pyx_L1_error);
+ }
+ }
+ __pyx_v_splines = ((PyObject*)__pyx_t_1);
+ __pyx_t_1 = 0;
+
+ /* "fontTools/cu2qu/cu2qu.py":547
+ * l = len(curves)
+ * splines = [None] * l
+ * last_i = i = 0 # <<<<<<<<<<<<<<
+ * n = 1
+ * while True:
+*/
+ __pyx_v_last_i = 0;
+ __pyx_v_i = 0;
+
+ /* "fontTools/cu2qu/cu2qu.py":548
+ * splines = [None] * l
+ * last_i = i = 0
+ * n = 1 # <<<<<<<<<<<<<<
+ * while True:
+ * spline = cubic_approx_spline(curves[i], n, max_errors[i], all_quadratic)
+*/
+ __Pyx_INCREF(__pyx_mstate_global->__pyx_int_1);
+ __pyx_v_n = __pyx_mstate_global->__pyx_int_1;
+
+ /* "fontTools/cu2qu/cu2qu.py":549
+ * last_i = i = 0
+ * n = 1
+ * while True: # <<<<<<<<<<<<<<
+ * spline = cubic_approx_spline(curves[i], n, max_errors[i], all_quadratic)
+ * if spline is None:
+*/
+ while (1) {
+
+ /* "fontTools/cu2qu/cu2qu.py":550
+ * n = 1
+ * while True:
+ * spline = cubic_approx_spline(curves[i], n, max_errors[i], all_quadratic) # <<<<<<<<<<<<<<
+ * if spline is None:
+ * if n == MAX_N:
+*/
+ __pyx_t_1 = __Pyx_GetItemInt(__pyx_v_curves, __pyx_v_i, int, 1, __Pyx_PyLong_From_int, 0, 1, 1, 1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 550, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_1);
+ __pyx_t_12 = __Pyx_PyLong_As_int(__pyx_v_n); if (unlikely((__pyx_t_12 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 550, __pyx_L1_error)
+ __pyx_t_2 = __Pyx_GetItemInt(__pyx_v_max_errors, __pyx_v_i, int, 1, __Pyx_PyLong_From_int, 0, 1, 1, 1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 550, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_2);
+ __pyx_t_13 = __Pyx_PyFloat_AsDouble(__pyx_t_2); if (unlikely((__pyx_t_13 == (double)-1) && PyErr_Occurred())) __PYX_ERR(0, 550, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+ __pyx_t_2 = __pyx_f_9fontTools_5cu2qu_5cu2qu_cubic_approx_spline(__pyx_t_1, __pyx_t_12, __pyx_t_13, __pyx_v_all_quadratic); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 550, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_2);
+ __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
+ __Pyx_XDECREF_SET(__pyx_v_spline, __pyx_t_2);
+ __pyx_t_2 = 0;
+
+ /* "fontTools/cu2qu/cu2qu.py":551
+ * while True:
+ * spline = cubic_approx_spline(curves[i], n, max_errors[i], all_quadratic)
+ * if spline is None: # <<<<<<<<<<<<<<
+ * if n == MAX_N:
+ * break
+*/
+ __pyx_t_11 = (__pyx_v_spline == Py_None);
+ if (__pyx_t_11) {
+
+ /* "fontTools/cu2qu/cu2qu.py":552
+ * spline = cubic_approx_spline(curves[i], n, max_errors[i], all_quadratic)
+ * if spline is None:
+ * if n == MAX_N: # <<<<<<<<<<<<<<
+ * break
+ * n += 1
+*/
+ __Pyx_GetModuleGlobalName(__pyx_t_2, __pyx_mstate_global->__pyx_n_u_MAX_N); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 552, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_2);
+ __pyx_t_1 = PyObject_RichCompare(__pyx_v_n, __pyx_t_2, Py_EQ); __Pyx_XGOTREF(__pyx_t_1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 552, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+ __pyx_t_11 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely((__pyx_t_11 < 0))) __PYX_ERR(0, 552, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
+ if (__pyx_t_11) {
+
+ /* "fontTools/cu2qu/cu2qu.py":553
+ * if spline is None:
+ * if n == MAX_N:
+ * break # <<<<<<<<<<<<<<
+ * n += 1
+ * last_i = i
+*/
+ goto __pyx_L18_break;
+
+ /* "fontTools/cu2qu/cu2qu.py":552
+ * spline = cubic_approx_spline(curves[i], n, max_errors[i], all_quadratic)
+ * if spline is None:
+ * if n == MAX_N: # <<<<<<<<<<<<<<
+ * break
+ * n += 1
+*/
+ }
+
+ /* "fontTools/cu2qu/cu2qu.py":554
+ * if n == MAX_N:
+ * break
+ * n += 1 # <<<<<<<<<<<<<<
+ * last_i = i
+ * continue
+*/
+ __pyx_t_1 = __Pyx_PyLong_AddObjC(__pyx_v_n, __pyx_mstate_global->__pyx_int_1, 1, 1, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 554, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_1);
+ __Pyx_DECREF_SET(__pyx_v_n, __pyx_t_1);
+ __pyx_t_1 = 0;
+
+ /* "fontTools/cu2qu/cu2qu.py":555
+ * break
+ * n += 1
+ * last_i = i # <<<<<<<<<<<<<<
+ * continue
+ * splines[i] = spline
+*/
+ __pyx_v_last_i = __pyx_v_i;
+
+ /* "fontTools/cu2qu/cu2qu.py":556
+ * n += 1
+ * last_i = i
+ * continue # <<<<<<<<<<<<<<
+ * splines[i] = spline
+ * i = (i + 1) % l
+*/
+ goto __pyx_L17_continue;
+
+ /* "fontTools/cu2qu/cu2qu.py":551
+ * while True:
+ * spline = cubic_approx_spline(curves[i], n, max_errors[i], all_quadratic)
+ * if spline is None: # <<<<<<<<<<<<<<
+ * if n == MAX_N:
+ * break
+*/
+ }
+
+ /* "fontTools/cu2qu/cu2qu.py":557
+ * last_i = i
+ * continue
+ * splines[i] = spline # <<<<<<<<<<<<<<
+ * i = (i + 1) % l
+ * if i == last_i:
+*/
+ if (unlikely((__Pyx_SetItemInt(__pyx_v_splines, __pyx_v_i, __pyx_v_spline, int, 1, __Pyx_PyLong_From_int, 1, 1, 1, 1) < 0))) __PYX_ERR(0, 557, __pyx_L1_error)
+
+ /* "fontTools/cu2qu/cu2qu.py":558
+ * continue
+ * splines[i] = spline
+ * i = (i + 1) % l # <<<<<<<<<<<<<<
+ * if i == last_i:
+ * # done. go home
+*/
+ __pyx_t_14 = (__pyx_v_i + 1);
+ if (unlikely(__pyx_v_l == 0)) {
+ PyErr_SetString(PyExc_ZeroDivisionError, "integer division or modulo by zero");
+ __PYX_ERR(0, 558, __pyx_L1_error)
+ }
+ __pyx_v_i = __Pyx_mod_long(__pyx_t_14, __pyx_v_l, 0);
+
+ /* "fontTools/cu2qu/cu2qu.py":559
+ * splines[i] = spline
+ * i = (i + 1) % l
+ * if i == last_i: # <<<<<<<<<<<<<<
+ * # done. go home
+ * return [[(s.real, s.imag) for s in spline] for spline in splines]
+*/
+ __pyx_t_11 = (__pyx_v_i == __pyx_v_last_i);
+ if (__pyx_t_11) {
+
+ /* "fontTools/cu2qu/cu2qu.py":561
+ * if i == last_i:
+ * # done. go home
+ * return [[(s.real, s.imag) for s in spline] for spline in splines] # <<<<<<<<<<<<<<
+ *
+ * raise ApproxNotFoundError(curves)
+*/
+ __Pyx_XDECREF(__pyx_r);
+ { /* enter inner scope */
+ __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 561, __pyx_L24_error)
+ __Pyx_GOTREF(__pyx_t_1);
+ __pyx_t_2 = __pyx_v_splines; __Pyx_INCREF(__pyx_t_2);
+ __pyx_t_7 = 0;
+ for (;;) {
+ {
+ Py_ssize_t __pyx_temp = __Pyx_PyList_GET_SIZE(__pyx_t_2);
+ #if !CYTHON_ASSUME_SAFE_SIZE
+ if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 561, __pyx_L24_error)
+ #endif
+ if (__pyx_t_7 >= __pyx_temp) break;
+ }
+ __pyx_t_5 = __Pyx_PyList_GetItemRef(__pyx_t_2, __pyx_t_7);
+ ++__pyx_t_7;
+ if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 561, __pyx_L24_error)
+ __Pyx_GOTREF(__pyx_t_5);
+ __Pyx_XDECREF_SET(__pyx_8genexpr4__pyx_v_spline, __pyx_t_5);
+ __pyx_t_5 = 0;
+ { /* enter inner scope */
+ __pyx_t_5 = PyList_New(0); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 561, __pyx_L29_error)
+ __Pyx_GOTREF(__pyx_t_5);
+ if (likely(PyList_CheckExact(__pyx_8genexpr4__pyx_v_spline)) || PyTuple_CheckExact(__pyx_8genexpr4__pyx_v_spline)) {
+ __pyx_t_6 = __pyx_8genexpr4__pyx_v_spline; __Pyx_INCREF(__pyx_t_6);
+ __pyx_t_3 = 0;
+ __pyx_t_4 = NULL;
+ } else {
+ __pyx_t_3 = -1; __pyx_t_6 = PyObject_GetIter(__pyx_8genexpr4__pyx_v_spline); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 561, __pyx_L29_error)
+ __Pyx_GOTREF(__pyx_t_6);
+ __pyx_t_4 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_6); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 561, __pyx_L29_error)
+ }
+ for (;;) {
+ if (likely(!__pyx_t_4)) {
+ if (likely(PyList_CheckExact(__pyx_t_6))) {
+ {
+ Py_ssize_t __pyx_temp = __Pyx_PyList_GET_SIZE(__pyx_t_6);
+ #if !CYTHON_ASSUME_SAFE_SIZE
+ if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 561, __pyx_L29_error)
+ #endif
+ if (__pyx_t_3 >= __pyx_temp) break;
+ }
+ __pyx_t_10 = __Pyx_PyList_GetItemRef(__pyx_t_6, __pyx_t_3);
+ ++__pyx_t_3;
+ } else {
+ {
+ Py_ssize_t __pyx_temp = __Pyx_PyTuple_GET_SIZE(__pyx_t_6);
+ #if !CYTHON_ASSUME_SAFE_SIZE
+ if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 561, __pyx_L29_error)
+ #endif
+ if (__pyx_t_3 >= __pyx_temp) break;
+ }
+ #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS
+ __pyx_t_10 = __Pyx_NewRef(PyTuple_GET_ITEM(__pyx_t_6, __pyx_t_3));
+ #else
+ __pyx_t_10 = __Pyx_PySequence_ITEM(__pyx_t_6, __pyx_t_3);
+ #endif
+ ++__pyx_t_3;
+ }
+ if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 561, __pyx_L29_error)
+ } else {
+ __pyx_t_10 = __pyx_t_4(__pyx_t_6);
+ if (unlikely(!__pyx_t_10)) {
+ PyObject* exc_type = PyErr_Occurred();
+ if (exc_type) {
+ if (unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) __PYX_ERR(0, 561, __pyx_L29_error)
+ PyErr_Clear();
+ }
+ break;
+ }
+ }
+ __Pyx_GOTREF(__pyx_t_10);
+ __Pyx_XDECREF_SET(__pyx_8genexpr5__pyx_v_s, __pyx_t_10);
+ __pyx_t_10 = 0;
+ __pyx_t_10 = __Pyx_PyObject_GetAttrStr(__pyx_8genexpr5__pyx_v_s, __pyx_mstate_global->__pyx_n_u_real); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 561, __pyx_L29_error)
+ __Pyx_GOTREF(__pyx_t_10);
+ __pyx_t_9 = __Pyx_PyObject_GetAttrStr(__pyx_8genexpr5__pyx_v_s, __pyx_mstate_global->__pyx_n_u_imag); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 561, __pyx_L29_error)
+ __Pyx_GOTREF(__pyx_t_9);
+ __pyx_t_15 = PyTuple_New(2); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 561, __pyx_L29_error)
+ __Pyx_GOTREF(__pyx_t_15);
+ __Pyx_GIVEREF(__pyx_t_10);
+ if (__Pyx_PyTuple_SET_ITEM(__pyx_t_15, 0, __pyx_t_10) != (0)) __PYX_ERR(0, 561, __pyx_L29_error);
+ __Pyx_GIVEREF(__pyx_t_9);
+ if (__Pyx_PyTuple_SET_ITEM(__pyx_t_15, 1, __pyx_t_9) != (0)) __PYX_ERR(0, 561, __pyx_L29_error);
+ __pyx_t_10 = 0;
+ __pyx_t_9 = 0;
+ if (unlikely(__Pyx_ListComp_Append(__pyx_t_5, (PyObject*)__pyx_t_15))) __PYX_ERR(0, 561, __pyx_L29_error)
+ __Pyx_DECREF(__pyx_t_15); __pyx_t_15 = 0;
+ }
+ __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
+ __Pyx_XDECREF(__pyx_8genexpr5__pyx_v_s); __pyx_8genexpr5__pyx_v_s = 0;
+ goto __pyx_L33_exit_scope;
+ __pyx_L29_error:;
+ __Pyx_XDECREF(__pyx_8genexpr5__pyx_v_s); __pyx_8genexpr5__pyx_v_s = 0;
+ goto __pyx_L24_error;
+ __pyx_L33_exit_scope:;
+ } /* exit inner scope */
+ if (unlikely(__Pyx_ListComp_Append(__pyx_t_1, (PyObject*)__pyx_t_5))) __PYX_ERR(0, 561, __pyx_L24_error)
+ __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
+ }
+ __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+ __Pyx_XDECREF(__pyx_8genexpr4__pyx_v_spline); __pyx_8genexpr4__pyx_v_spline = 0;
+ goto __pyx_L35_exit_scope;
+ __pyx_L24_error:;
+ __Pyx_XDECREF(__pyx_8genexpr4__pyx_v_spline); __pyx_8genexpr4__pyx_v_spline = 0;
+ goto __pyx_L1_error;
+ __pyx_L35_exit_scope:;
+ } /* exit inner scope */
+ __pyx_r = __pyx_t_1;
+ __pyx_t_1 = 0;
+ goto __pyx_L0;
+
+ /* "fontTools/cu2qu/cu2qu.py":559
+ * splines[i] = spline
+ * i = (i + 1) % l
+ * if i == last_i: # <<<<<<<<<<<<<<
+ * # done. go home
+ * return [[(s.real, s.imag) for s in spline] for spline in splines]
+*/
+ }
+ __pyx_L17_continue:;
+ }
+ __pyx_L18_break:;
+
+ /* "fontTools/cu2qu/cu2qu.py":563
+ * return [[(s.real, s.imag) for s in spline] for spline in splines]
+ *
+ * raise ApproxNotFoundError(curves) # <<<<<<<<<<<<<<
+*/
+ __pyx_t_2 = NULL;
+ __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_ApproxNotFoundError); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 563, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_5);
+ __pyx_t_16 = 1;
+ #if CYTHON_UNPACK_METHODS
+ if (unlikely(PyMethod_Check(__pyx_t_5))) {
+ __pyx_t_2 = PyMethod_GET_SELF(__pyx_t_5);
+ assert(__pyx_t_2);
+ PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5);
+ __Pyx_INCREF(__pyx_t_2);
+ __Pyx_INCREF(__pyx__function);
+ __Pyx_DECREF_SET(__pyx_t_5, __pyx__function);
+ __pyx_t_16 = 0;
+ }
+ #endif
+ {
+ PyObject *__pyx_callargs[2] = {__pyx_t_2, __pyx_v_curves};
+ __pyx_t_1 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_16, (2-__pyx_t_16) | (__pyx_t_16*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET));
+ __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0;
+ __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
+ if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 563, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_1);
+ }
+ __Pyx_Raise(__pyx_t_1, 0, 0, 0);
+ __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
+ __PYX_ERR(0, 563, __pyx_L1_error)
+
+ /* "fontTools/cu2qu/cu2qu.py":503
+ *
+ *
+ * @cython.locals(l=cython.int, last_i=cython.int, i=cython.int) # <<<<<<<<<<<<<<
+ * @cython.locals(all_quadratic=cython.int)
+ * def curves_to_quadratic(curves, max_errors, all_quadratic=True):
+*/
+
+ /* function exit code */
+ __pyx_L1_error:;
+ __Pyx_XDECREF(__pyx_t_1);
+ __Pyx_XDECREF(__pyx_t_2);
+ __Pyx_XDECREF(__pyx_t_5);
+ __Pyx_XDECREF(__pyx_t_6);
+ __Pyx_XDECREF(__pyx_t_9);
+ __Pyx_XDECREF(__pyx_t_10);
+ __Pyx_XDECREF(__pyx_t_15);
+ __Pyx_AddTraceback("fontTools.cu2qu.cu2qu.curves_to_quadratic", __pyx_clineno, __pyx_lineno, __pyx_filename);
+ __pyx_r = NULL;
+ __pyx_L0:;
+ __Pyx_XDECREF(__pyx_v_splines);
+ __Pyx_XDECREF(__pyx_v_n);
+ __Pyx_XDECREF(__pyx_v_spline);
+ __Pyx_XDECREF(__pyx_8genexpr2__pyx_v_curve);
+ __Pyx_XDECREF(__pyx_8genexpr3__pyx_v_p);
+ __Pyx_XDECREF(__pyx_8genexpr4__pyx_v_spline);
+ __Pyx_XDECREF(__pyx_8genexpr5__pyx_v_s);
+ __Pyx_XDECREF(__pyx_v_curves);
+ __Pyx_XGIVEREF(__pyx_r);
+ __Pyx_RefNannyFinishContext();
+ return __pyx_r;
+}
+/* #### Code section: module_exttypes ### */
+
+static PyObject *__pyx_tp_new_9fontTools_5cu2qu_5cu2qu___pyx_scope_struct___split_cubic_into_n_gen(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) {
+ PyObject *o;
+ #if CYTHON_COMPILING_IN_LIMITED_API
+ allocfunc alloc_func = (allocfunc)PyType_GetSlot(t, Py_tp_alloc);
+ o = alloc_func(t, 0);
+ #else
+ #if CYTHON_USE_FREELISTS
+ if (likely((int)(__pyx_mstate_global->__pyx_freecount_9fontTools_5cu2qu_5cu2qu___pyx_scope_struct___split_cubic_into_n_gen > 0) & (int)(t->tp_basicsize == sizeof(struct __pyx_obj_9fontTools_5cu2qu_5cu2qu___pyx_scope_struct___split_cubic_into_n_gen)))) {
+ o = (PyObject*)__pyx_mstate_global->__pyx_freelist_9fontTools_5cu2qu_5cu2qu___pyx_scope_struct___split_cubic_into_n_gen[--__pyx_mstate_global->__pyx_freecount_9fontTools_5cu2qu_5cu2qu___pyx_scope_struct___split_cubic_into_n_gen];
+ memset(o, 0, sizeof(struct __pyx_obj_9fontTools_5cu2qu_5cu2qu___pyx_scope_struct___split_cubic_into_n_gen));
+ (void) PyObject_INIT(o, t);
+ } else
+ #endif
+ {
+ o = (*t->tp_alloc)(t, 0);
+ if (unlikely(!o)) return 0;
+ }
+ #endif
+ return o;
+}
+
+static void __pyx_tp_dealloc_9fontTools_5cu2qu_5cu2qu___pyx_scope_struct___split_cubic_into_n_gen(PyObject *o) {
+ #if CYTHON_USE_TP_FINALIZE
+ if (unlikely((PY_VERSION_HEX >= 0x03080000 || __Pyx_PyType_HasFeature(Py_TYPE(o), Py_TPFLAGS_HAVE_FINALIZE)) && __Pyx_PyObject_GetSlot(o, tp_finalize, destructor)) && (!PyType_IS_GC(Py_TYPE(o)) || !__Pyx_PyObject_GC_IsFinalized(o))) {
+ if (__Pyx_PyObject_GetSlot(o, tp_dealloc, destructor) == __pyx_tp_dealloc_9fontTools_5cu2qu_5cu2qu___pyx_scope_struct___split_cubic_into_n_gen) {
+ if (PyObject_CallFinalizerFromDealloc(o)) return;
+ }
+ }
+ #endif
+ #if CYTHON_USE_FREELISTS
+ if (((int)(__pyx_mstate_global->__pyx_freecount_9fontTools_5cu2qu_5cu2qu___pyx_scope_struct___split_cubic_into_n_gen < 8) & (int)(Py_TYPE(o)->tp_basicsize == sizeof(struct __pyx_obj_9fontTools_5cu2qu_5cu2qu___pyx_scope_struct___split_cubic_into_n_gen)))) {
+ __pyx_mstate_global->__pyx_freelist_9fontTools_5cu2qu_5cu2qu___pyx_scope_struct___split_cubic_into_n_gen[__pyx_mstate_global->__pyx_freecount_9fontTools_5cu2qu_5cu2qu___pyx_scope_struct___split_cubic_into_n_gen++] = ((struct __pyx_obj_9fontTools_5cu2qu_5cu2qu___pyx_scope_struct___split_cubic_into_n_gen *)o);
+ } else
+ #endif
+ {
+ #if CYTHON_USE_TYPE_SLOTS
+ (*Py_TYPE(o)->tp_free)(o);
+ #else
+ {
+ freefunc tp_free = (freefunc)PyType_GetSlot(Py_TYPE(o), Py_tp_free);
+ if (tp_free) tp_free(o);
+ }
+ #endif
+ }
+}
+#if CYTHON_USE_TYPE_SPECS
+static PyType_Slot __pyx_type_9fontTools_5cu2qu_5cu2qu___pyx_scope_struct___split_cubic_into_n_gen_slots[] = {
+ {Py_tp_dealloc, (void *)__pyx_tp_dealloc_9fontTools_5cu2qu_5cu2qu___pyx_scope_struct___split_cubic_into_n_gen},
+ {Py_tp_new, (void *)__pyx_tp_new_9fontTools_5cu2qu_5cu2qu___pyx_scope_struct___split_cubic_into_n_gen},
+ {0, 0},
+};
+static PyType_Spec __pyx_type_9fontTools_5cu2qu_5cu2qu___pyx_scope_struct___split_cubic_into_n_gen_spec = {
+ "fontTools.cu2qu.cu2qu.__pyx_scope_struct___split_cubic_into_n_gen",
+ sizeof(struct __pyx_obj_9fontTools_5cu2qu_5cu2qu___pyx_scope_struct___split_cubic_into_n_gen),
+ 0,
+ Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_FINALIZE,
+ __pyx_type_9fontTools_5cu2qu_5cu2qu___pyx_scope_struct___split_cubic_into_n_gen_slots,
+};
+#else
+
+static PyTypeObject __pyx_type_9fontTools_5cu2qu_5cu2qu___pyx_scope_struct___split_cubic_into_n_gen = {
+ PyVarObject_HEAD_INIT(0, 0)
+ "fontTools.cu2qu.cu2qu.""__pyx_scope_struct___split_cubic_into_n_gen", /*tp_name*/
+ sizeof(struct __pyx_obj_9fontTools_5cu2qu_5cu2qu___pyx_scope_struct___split_cubic_into_n_gen), /*tp_basicsize*/
+ 0, /*tp_itemsize*/
+ __pyx_tp_dealloc_9fontTools_5cu2qu_5cu2qu___pyx_scope_struct___split_cubic_into_n_gen, /*tp_dealloc*/
+ #if PY_VERSION_HEX < 0x030800b4
+ 0, /*tp_print*/
+ #endif
+ #if PY_VERSION_HEX >= 0x030800b4
+ 0, /*tp_vectorcall_offset*/
+ #endif
+ 0, /*tp_getattr*/
+ 0, /*tp_setattr*/
+ 0, /*tp_as_async*/
+ 0, /*tp_repr*/
+ 0, /*tp_as_number*/
+ 0, /*tp_as_sequence*/
+ 0, /*tp_as_mapping*/
+ 0, /*tp_hash*/
+ 0, /*tp_call*/
+ 0, /*tp_str*/
+ 0, /*tp_getattro*/
+ 0, /*tp_setattro*/
+ 0, /*tp_as_buffer*/
+ Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_FINALIZE, /*tp_flags*/
+ 0, /*tp_doc*/
+ 0, /*tp_traverse*/
+ 0, /*tp_clear*/
+ 0, /*tp_richcompare*/
+ 0, /*tp_weaklistoffset*/
+ 0, /*tp_iter*/
+ 0, /*tp_iternext*/
+ 0, /*tp_methods*/
+ 0, /*tp_members*/
+ 0, /*tp_getset*/
+ 0, /*tp_base*/
+ 0, /*tp_dict*/
+ 0, /*tp_descr_get*/
+ 0, /*tp_descr_set*/
+ #if !CYTHON_USE_TYPE_SPECS
+ 0, /*tp_dictoffset*/
+ #endif
+ 0, /*tp_init*/
+ 0, /*tp_alloc*/
+ __pyx_tp_new_9fontTools_5cu2qu_5cu2qu___pyx_scope_struct___split_cubic_into_n_gen, /*tp_new*/
+ 0, /*tp_free*/
+ 0, /*tp_is_gc*/
+ 0, /*tp_bases*/
+ 0, /*tp_mro*/
+ 0, /*tp_cache*/
+ 0, /*tp_subclasses*/
+ 0, /*tp_weaklist*/
+ 0, /*tp_del*/
+ 0, /*tp_version_tag*/
+ #if CYTHON_USE_TP_FINALIZE
+ 0, /*tp_finalize*/
+ #else
+ NULL, /*tp_finalize*/
+ #endif
+ #if PY_VERSION_HEX >= 0x030800b1 && (!CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800)
+ 0, /*tp_vectorcall*/
+ #endif
+ #if __PYX_NEED_TP_PRINT_SLOT == 1
+ 0, /*tp_print*/
+ #endif
+ #if PY_VERSION_HEX >= 0x030C0000
+ 0, /*tp_watched*/
+ #endif
+ #if PY_VERSION_HEX >= 0x030d00A4
+ 0, /*tp_versions_used*/
+ #endif
+ #if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 && PY_VERSION_HEX < 0x030a0000
+ 0, /*tp_pypy_flags*/
+ #endif
+};
+#endif
+
+static PyMethodDef __pyx_methods[] = {
+ {0, 0, 0, 0}
+};
+/* #### Code section: initfunc_declarations ### */
+static CYTHON_SMALL_CODE int __Pyx_InitCachedBuiltins(__pyx_mstatetype *__pyx_mstate); /*proto*/
+static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(__pyx_mstatetype *__pyx_mstate); /*proto*/
+static CYTHON_SMALL_CODE int __Pyx_InitGlobals(void); /*proto*/
+static CYTHON_SMALL_CODE int __Pyx_InitConstants(__pyx_mstatetype *__pyx_mstate); /*proto*/
+static CYTHON_SMALL_CODE int __Pyx_modinit_global_init_code(__pyx_mstatetype *__pyx_mstate); /*proto*/
+static CYTHON_SMALL_CODE int __Pyx_modinit_variable_export_code(__pyx_mstatetype *__pyx_mstate); /*proto*/
+static CYTHON_SMALL_CODE int __Pyx_modinit_function_export_code(__pyx_mstatetype *__pyx_mstate); /*proto*/
+static CYTHON_SMALL_CODE int __Pyx_modinit_type_init_code(__pyx_mstatetype *__pyx_mstate); /*proto*/
+static CYTHON_SMALL_CODE int __Pyx_modinit_type_import_code(__pyx_mstatetype *__pyx_mstate); /*proto*/
+static CYTHON_SMALL_CODE int __Pyx_modinit_variable_import_code(__pyx_mstatetype *__pyx_mstate); /*proto*/
+static CYTHON_SMALL_CODE int __Pyx_modinit_function_import_code(__pyx_mstatetype *__pyx_mstate); /*proto*/
+static CYTHON_SMALL_CODE int __Pyx_CreateCodeObjects(__pyx_mstatetype *__pyx_mstate); /*proto*/
+/* #### Code section: init_module ### */
+
+static int __Pyx_modinit_global_init_code(__pyx_mstatetype *__pyx_mstate) {
+ __Pyx_RefNannyDeclarations
+ CYTHON_UNUSED_VAR(__pyx_mstate);
+ __Pyx_RefNannySetupContext("__Pyx_modinit_global_init_code", 0);
+ /*--- Global init code ---*/
+ __Pyx_RefNannyFinishContext();
+ return 0;
+}
+
+static int __Pyx_modinit_variable_export_code(__pyx_mstatetype *__pyx_mstate) {
+ __Pyx_RefNannyDeclarations
+ CYTHON_UNUSED_VAR(__pyx_mstate);
+ __Pyx_RefNannySetupContext("__Pyx_modinit_variable_export_code", 0);
+ /*--- Variable export code ---*/
+ __Pyx_RefNannyFinishContext();
+ return 0;
+}
+
+static int __Pyx_modinit_function_export_code(__pyx_mstatetype *__pyx_mstate) {
+ __Pyx_RefNannyDeclarations
+ CYTHON_UNUSED_VAR(__pyx_mstate);
+ __Pyx_RefNannySetupContext("__Pyx_modinit_function_export_code", 0);
+ /*--- Function export code ---*/
+ __Pyx_RefNannyFinishContext();
+ return 0;
+}
+
+static int __Pyx_modinit_type_init_code(__pyx_mstatetype *__pyx_mstate) {
+ __Pyx_RefNannyDeclarations
+ CYTHON_UNUSED_VAR(__pyx_mstate);
+ int __pyx_lineno = 0;
+ const char *__pyx_filename = NULL;
+ int __pyx_clineno = 0;
+ __Pyx_RefNannySetupContext("__Pyx_modinit_type_init_code", 0);
+ /*--- Type init code ---*/
+ #if CYTHON_USE_TYPE_SPECS
+ __pyx_mstate->__pyx_ptype_9fontTools_5cu2qu_5cu2qu___pyx_scope_struct___split_cubic_into_n_gen = (PyTypeObject *) __Pyx_PyType_FromModuleAndSpec(__pyx_m, &__pyx_type_9fontTools_5cu2qu_5cu2qu___pyx_scope_struct___split_cubic_into_n_gen_spec, NULL); if (unlikely(!__pyx_mstate->__pyx_ptype_9fontTools_5cu2qu_5cu2qu___pyx_scope_struct___split_cubic_into_n_gen)) __PYX_ERR(0, 150, __pyx_L1_error)
+ if (__Pyx_fix_up_extension_type_from_spec(&__pyx_type_9fontTools_5cu2qu_5cu2qu___pyx_scope_struct___split_cubic_into_n_gen_spec, __pyx_mstate->__pyx_ptype_9fontTools_5cu2qu_5cu2qu___pyx_scope_struct___split_cubic_into_n_gen) < 0) __PYX_ERR(0, 150, __pyx_L1_error)
+ #else
+ __pyx_mstate->__pyx_ptype_9fontTools_5cu2qu_5cu2qu___pyx_scope_struct___split_cubic_into_n_gen = &__pyx_type_9fontTools_5cu2qu_5cu2qu___pyx_scope_struct___split_cubic_into_n_gen;
+ #endif
+ #if !CYTHON_COMPILING_IN_LIMITED_API
+ #endif
+ #if !CYTHON_USE_TYPE_SPECS
+ if (__Pyx_PyType_Ready(__pyx_mstate->__pyx_ptype_9fontTools_5cu2qu_5cu2qu___pyx_scope_struct___split_cubic_into_n_gen) < 0) __PYX_ERR(0, 150, __pyx_L1_error)
+ #endif
+ #if !CYTHON_COMPILING_IN_LIMITED_API
+ if ((CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP) && likely(!__pyx_mstate->__pyx_ptype_9fontTools_5cu2qu_5cu2qu___pyx_scope_struct___split_cubic_into_n_gen->tp_dictoffset && __pyx_mstate->__pyx_ptype_9fontTools_5cu2qu_5cu2qu___pyx_scope_struct___split_cubic_into_n_gen->tp_getattro == PyObject_GenericGetAttr)) {
+ __pyx_mstate->__pyx_ptype_9fontTools_5cu2qu_5cu2qu___pyx_scope_struct___split_cubic_into_n_gen->tp_getattro = PyObject_GenericGetAttr;
+ }
+ #endif
+ __Pyx_RefNannyFinishContext();
+ return 0;
+ __pyx_L1_error:;
+ __Pyx_RefNannyFinishContext();
+ return -1;
+}
+
+static int __Pyx_modinit_type_import_code(__pyx_mstatetype *__pyx_mstate) {
+ __Pyx_RefNannyDeclarations
+ CYTHON_UNUSED_VAR(__pyx_mstate);
+ __Pyx_RefNannySetupContext("__Pyx_modinit_type_import_code", 0);
+ /*--- Type import code ---*/
+ __Pyx_RefNannyFinishContext();
+ return 0;
+}
+
+static int __Pyx_modinit_variable_import_code(__pyx_mstatetype *__pyx_mstate) {
+ __Pyx_RefNannyDeclarations
+ CYTHON_UNUSED_VAR(__pyx_mstate);
+ __Pyx_RefNannySetupContext("__Pyx_modinit_variable_import_code", 0);
+ /*--- Variable import code ---*/
+ __Pyx_RefNannyFinishContext();
+ return 0;
+}
+
+static int __Pyx_modinit_function_import_code(__pyx_mstatetype *__pyx_mstate) {
+ __Pyx_RefNannyDeclarations
+ CYTHON_UNUSED_VAR(__pyx_mstate);
+ __Pyx_RefNannySetupContext("__Pyx_modinit_function_import_code", 0);
+ /*--- Function import code ---*/
+ __Pyx_RefNannyFinishContext();
+ return 0;
+}
+
+#if CYTHON_PEP489_MULTI_PHASE_INIT
+static PyObject* __pyx_pymod_create(PyObject *spec, PyModuleDef *def); /*proto*/
+static int __pyx_pymod_exec_cu2qu(PyObject* module); /*proto*/
+static PyModuleDef_Slot __pyx_moduledef_slots[] = {
+ {Py_mod_create, (void*)__pyx_pymod_create},
+ {Py_mod_exec, (void*)__pyx_pymod_exec_cu2qu},
+ #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING
+ {Py_mod_gil, Py_MOD_GIL_USED},
+ #endif
+ #if PY_VERSION_HEX >= 0x030C0000 && CYTHON_USE_MODULE_STATE
+ {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED},
+ #endif
+ {0, NULL}
+};
+#endif
+
+#ifdef __cplusplus
+namespace {
+ struct PyModuleDef __pyx_moduledef =
+ #else
+ static struct PyModuleDef __pyx_moduledef =
+ #endif
+ {
+ PyModuleDef_HEAD_INIT,
+ "cu2qu",
+ 0, /* m_doc */
+ #if CYTHON_USE_MODULE_STATE
+ sizeof(__pyx_mstatetype), /* m_size */
+ #else
+ (CYTHON_PEP489_MULTI_PHASE_INIT) ? 0 : -1, /* m_size */
+ #endif
+ __pyx_methods /* m_methods */,
+ #if CYTHON_PEP489_MULTI_PHASE_INIT
+ __pyx_moduledef_slots, /* m_slots */
+ #else
+ NULL, /* m_reload */
+ #endif
+ #if CYTHON_USE_MODULE_STATE
+ __pyx_m_traverse, /* m_traverse */
+ __pyx_m_clear, /* m_clear */
+ NULL /* m_free */
+ #else
+ NULL, /* m_traverse */
+ NULL, /* m_clear */
+ NULL /* m_free */
+ #endif
+ };
+ #ifdef __cplusplus
+} /* anonymous namespace */
+#endif
+
+/* PyModInitFuncType */
+#ifndef CYTHON_NO_PYINIT_EXPORT
+ #define __Pyx_PyMODINIT_FUNC PyMODINIT_FUNC
+#else
+ #ifdef __cplusplus
+ #define __Pyx_PyMODINIT_FUNC extern "C" PyObject *
+ #else
+ #define __Pyx_PyMODINIT_FUNC PyObject *
+ #endif
+#endif
+
+__Pyx_PyMODINIT_FUNC PyInit_cu2qu(void) CYTHON_SMALL_CODE; /*proto*/
+__Pyx_PyMODINIT_FUNC PyInit_cu2qu(void)
+#if CYTHON_PEP489_MULTI_PHASE_INIT
+{
+ return PyModuleDef_Init(&__pyx_moduledef);
+}
+/* ModuleCreationPEP489 */
+#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x03090000
+static PY_INT64_T __Pyx_GetCurrentInterpreterId(void) {
+ {
+ PyObject *module = PyImport_ImportModule("_interpreters"); // 3.13+ I think
+ if (!module) {
+ PyErr_Clear(); // just try the 3.8-3.12 version
+ module = PyImport_ImportModule("_xxsubinterpreters");
+ if (!module) goto bad;
+ }
+ PyObject *current = PyObject_CallMethod(module, "get_current", NULL);
+ Py_DECREF(module);
+ if (!current) goto bad;
+ if (PyTuple_Check(current)) {
+ PyObject *new_current = PySequence_GetItem(current, 0);
+ Py_DECREF(current);
+ current = new_current;
+ if (!new_current) goto bad;
+ }
+ long long as_c_int = PyLong_AsLongLong(current);
+ Py_DECREF(current);
+ return as_c_int;
+ }
+ bad:
+ PySys_WriteStderr("__Pyx_GetCurrentInterpreterId failed. Try setting the C define CYTHON_PEP489_MULTI_PHASE_INIT=0\n");
+ return -1;
+}
+#endif
+#if !CYTHON_USE_MODULE_STATE
+static CYTHON_SMALL_CODE int __Pyx_check_single_interpreter(void) {
+ static PY_INT64_T main_interpreter_id = -1;
+#if CYTHON_COMPILING_IN_GRAAL
+ PY_INT64_T current_id = PyInterpreterState_GetIDFromThreadState(PyThreadState_Get());
+#elif CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX >= 0x03090000
+ PY_INT64_T current_id = PyInterpreterState_GetID(PyInterpreterState_Get());
+#elif CYTHON_COMPILING_IN_LIMITED_API
+ PY_INT64_T current_id = __Pyx_GetCurrentInterpreterId();
+#else
+ PY_INT64_T current_id = PyInterpreterState_GetID(PyThreadState_Get()->interp);
+#endif
+ if (unlikely(current_id == -1)) {
+ return -1;
+ }
+ if (main_interpreter_id == -1) {
+ main_interpreter_id = current_id;
+ return 0;
+ } else if (unlikely(main_interpreter_id != current_id)) {
+ PyErr_SetString(
+ PyExc_ImportError,
+ "Interpreter change detected - this module can only be loaded into one interpreter per process.");
+ return -1;
+ }
+ return 0;
+}
+#endif
+static CYTHON_SMALL_CODE int __Pyx_copy_spec_to_module(PyObject *spec, PyObject *moddict, const char* from_name, const char* to_name, int allow_none)
+{
+ PyObject *value = PyObject_GetAttrString(spec, from_name);
+ int result = 0;
+ if (likely(value)) {
+ if (allow_none || value != Py_None) {
+ result = PyDict_SetItemString(moddict, to_name, value);
+ }
+ Py_DECREF(value);
+ } else if (PyErr_ExceptionMatches(PyExc_AttributeError)) {
+ PyErr_Clear();
+ } else {
+ result = -1;
+ }
+ return result;
+}
+static CYTHON_SMALL_CODE PyObject* __pyx_pymod_create(PyObject *spec, PyModuleDef *def) {
+ PyObject *module = NULL, *moddict, *modname;
+ CYTHON_UNUSED_VAR(def);
+ #if !CYTHON_USE_MODULE_STATE
+ if (__Pyx_check_single_interpreter())
+ return NULL;
+ #endif
+ if (__pyx_m)
+ return __Pyx_NewRef(__pyx_m);
+ modname = PyObject_GetAttrString(spec, "name");
+ if (unlikely(!modname)) goto bad;
+ module = PyModule_NewObject(modname);
+ Py_DECREF(modname);
+ if (unlikely(!module)) goto bad;
+ moddict = PyModule_GetDict(module);
+ if (unlikely(!moddict)) goto bad;
+ if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "loader", "__loader__", 1) < 0)) goto bad;
+ if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "origin", "__file__", 1) < 0)) goto bad;
+ if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "parent", "__package__", 1) < 0)) goto bad;
+ if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "submodule_search_locations", "__path__", 0) < 0)) goto bad;
+ return module;
+bad:
+ Py_XDECREF(module);
+ return NULL;
+}
+
+
+static CYTHON_SMALL_CODE int __pyx_pymod_exec_cu2qu(PyObject *__pyx_pyinit_module)
+#endif
+{
+ int stringtab_initialized = 0;
+ #if CYTHON_USE_MODULE_STATE
+ int pystate_addmodule_run = 0;
+ #endif
+ __pyx_mstatetype *__pyx_mstate = NULL;
+ PyObject *__pyx_t_1 = NULL;
+ PyObject *__pyx_t_2 = NULL;
+ PyObject *__pyx_t_3 = NULL;
+ PyObject *__pyx_t_4 = NULL;
+ PyObject *__pyx_t_5 = NULL;
+ double __pyx_t_6;
+ int __pyx_lineno = 0;
+ const char *__pyx_filename = NULL;
+ int __pyx_clineno = 0;
+ __Pyx_RefNannyDeclarations
+ #if CYTHON_PEP489_MULTI_PHASE_INIT
+ if (__pyx_m) {
+ if (__pyx_m == __pyx_pyinit_module) return 0;
+ PyErr_SetString(PyExc_RuntimeError, "Module 'cu2qu' has already been imported. Re-initialisation is not supported.");
+ return -1;
+ }
+ #else
+ if (__pyx_m) return __Pyx_NewRef(__pyx_m);
+ #endif
+ /*--- Module creation code ---*/
+ #if CYTHON_PEP489_MULTI_PHASE_INIT
+ __pyx_t_1 = __pyx_pyinit_module;
+ Py_INCREF(__pyx_t_1);
+ #else
+ __pyx_t_1 = PyModule_Create(&__pyx_moduledef); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1, __pyx_L1_error)
+ #endif
+ #if CYTHON_USE_MODULE_STATE
+ {
+ int add_module_result = __Pyx_State_AddModule(__pyx_t_1, &__pyx_moduledef);
+ __pyx_t_1 = 0; /* transfer ownership from __pyx_t_1 to "cu2qu" pseudovariable */
+ if (unlikely((add_module_result < 0))) __PYX_ERR(0, 1, __pyx_L1_error)
+ pystate_addmodule_run = 1;
+ }
+ #else
+ __pyx_m = __pyx_t_1;
+ #endif
+ #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING
+ PyUnstable_Module_SetGIL(__pyx_m, Py_MOD_GIL_USED);
+ #endif
+ __pyx_mstate = __pyx_mstate_global;
+ CYTHON_UNUSED_VAR(__pyx_t_1);
+ __pyx_mstate->__pyx_d = PyModule_GetDict(__pyx_m); if (unlikely(!__pyx_mstate->__pyx_d)) __PYX_ERR(0, 1, __pyx_L1_error)
+ Py_INCREF(__pyx_mstate->__pyx_d);
+ __pyx_mstate->__pyx_b = __Pyx_PyImport_AddModuleRef(__Pyx_BUILTIN_MODULE_NAME); if (unlikely(!__pyx_mstate->__pyx_b)) __PYX_ERR(0, 1, __pyx_L1_error)
+ __pyx_mstate->__pyx_cython_runtime = __Pyx_PyImport_AddModuleRef("cython_runtime"); if (unlikely(!__pyx_mstate->__pyx_cython_runtime)) __PYX_ERR(0, 1, __pyx_L1_error)
+ if (PyObject_SetAttrString(__pyx_m, "__builtins__", __pyx_mstate->__pyx_b) < 0) __PYX_ERR(0, 1, __pyx_L1_error)
+ /* ImportRefnannyAPI */
+ #if CYTHON_REFNANNY
+__Pyx_RefNanny = __Pyx_RefNannyImportAPI("refnanny");
+if (!__Pyx_RefNanny) {
+ PyErr_Clear();
+ __Pyx_RefNanny = __Pyx_RefNannyImportAPI("Cython.Runtime.refnanny");
+ if (!__Pyx_RefNanny)
+ Py_FatalError("failed to import 'refnanny' module");
+}
+#endif
+
+__Pyx_RefNannySetupContext("PyInit_cu2qu", 0);
+ if (__Pyx_check_binary_version(__PYX_LIMITED_VERSION_HEX, __Pyx_get_runtime_version(), CYTHON_COMPILING_IN_LIMITED_API) < 0) __PYX_ERR(0, 1, __pyx_L1_error)
+ #ifdef __Pxy_PyFrame_Initialize_Offsets
+ __Pxy_PyFrame_Initialize_Offsets();
+ #endif
+ __pyx_mstate->__pyx_empty_tuple = PyTuple_New(0); if (unlikely(!__pyx_mstate->__pyx_empty_tuple)) __PYX_ERR(0, 1, __pyx_L1_error)
+ __pyx_mstate->__pyx_empty_bytes = PyBytes_FromStringAndSize("", 0); if (unlikely(!__pyx_mstate->__pyx_empty_bytes)) __PYX_ERR(0, 1, __pyx_L1_error)
+ __pyx_mstate->__pyx_empty_unicode = PyUnicode_FromStringAndSize("", 0); if (unlikely(!__pyx_mstate->__pyx_empty_unicode)) __PYX_ERR(0, 1, __pyx_L1_error)
+ /*--- Initialize various global constants etc. ---*/
+ if (__Pyx_InitConstants(__pyx_mstate) < 0) __PYX_ERR(0, 1, __pyx_L1_error)
+ stringtab_initialized = 1;
+ if (__Pyx_InitGlobals() < 0) __PYX_ERR(0, 1, __pyx_L1_error)
+ #if 0 || defined(__Pyx_CyFunction_USED) || defined(__Pyx_FusedFunction_USED) || defined(__Pyx_Coroutine_USED) || defined(__Pyx_Generator_USED) || defined(__Pyx_AsyncGen_USED)
+ if (__pyx_CommonTypesMetaclass_init(__pyx_m) < 0) __PYX_ERR(0, 1, __pyx_L1_error)
+ #endif
+ #ifdef __Pyx_CyFunction_USED
+ if (__pyx_CyFunction_init(__pyx_m) < 0) __PYX_ERR(0, 1, __pyx_L1_error)
+ #endif
+ #ifdef __Pyx_FusedFunction_USED
+ if (__pyx_FusedFunction_init(__pyx_m) < 0) __PYX_ERR(0, 1, __pyx_L1_error)
+ #endif
+ #ifdef __Pyx_Coroutine_USED
+ if (__pyx_Coroutine_init(__pyx_m) < 0) __PYX_ERR(0, 1, __pyx_L1_error)
+ #endif
+ #ifdef __Pyx_Generator_USED
+ if (__pyx_Generator_init(__pyx_m) < 0) __PYX_ERR(0, 1, __pyx_L1_error)
+ #endif
+ #ifdef __Pyx_AsyncGen_USED
+ if (__pyx_AsyncGen_init(__pyx_m) < 0) __PYX_ERR(0, 1, __pyx_L1_error)
+ #endif
+ /*--- Library function declarations ---*/
+ if (__pyx_module_is_main_fontTools__cu2qu__cu2qu) {
+ if (PyObject_SetAttr(__pyx_m, __pyx_mstate_global->__pyx_n_u_name, __pyx_mstate_global->__pyx_n_u_main) < 0) __PYX_ERR(0, 1, __pyx_L1_error)
+ }
+ {
+ PyObject *modules = PyImport_GetModuleDict(); if (unlikely(!modules)) __PYX_ERR(0, 1, __pyx_L1_error)
+ if (!PyDict_GetItemString(modules, "fontTools.cu2qu.cu2qu")) {
+ if (unlikely((PyDict_SetItemString(modules, "fontTools.cu2qu.cu2qu", __pyx_m) < 0))) __PYX_ERR(0, 1, __pyx_L1_error)
+ }
+ }
+ /*--- Builtin init code ---*/
+ if (__Pyx_InitCachedBuiltins(__pyx_mstate) < 0) __PYX_ERR(0, 1, __pyx_L1_error)
+ /*--- Constants init code ---*/
+ if (__Pyx_InitCachedConstants(__pyx_mstate) < 0) __PYX_ERR(0, 1, __pyx_L1_error)
+ if (__Pyx_CreateCodeObjects(__pyx_mstate) < 0) __PYX_ERR(0, 1, __pyx_L1_error)
+ /*--- Global type/function init code ---*/
+ (void)__Pyx_modinit_global_init_code(__pyx_mstate);
+ (void)__Pyx_modinit_variable_export_code(__pyx_mstate);
+ (void)__Pyx_modinit_function_export_code(__pyx_mstate);
+ if (unlikely((__Pyx_modinit_type_init_code(__pyx_mstate) < 0))) __PYX_ERR(0, 1, __pyx_L1_error)
+ (void)__Pyx_modinit_type_import_code(__pyx_mstate);
+ (void)__Pyx_modinit_variable_import_code(__pyx_mstate);
+ (void)__Pyx_modinit_function_import_code(__pyx_mstate);
+ /*--- Execution code ---*/
+
+ /* "fontTools/cu2qu/cu2qu.py":18
+ * # limitations under the License.
+ *
+ * try: # <<<<<<<<<<<<<<
+ * import cython
+ * except (AttributeError, ImportError):
+*/
+ {
+ (void)__pyx_t_1; (void)__pyx_t_2; (void)__pyx_t_3; /* mark used */
+ /*try:*/ {
+
+ /* "fontTools/cu2qu/cu2qu.py":19
+ *
+ * try:
+ * import cython # <<<<<<<<<<<<<<
+ * except (AttributeError, ImportError):
+ * # if cython not installed, use mock module with no-op decorators and types
+*/
+ }
+ }
+
+ /* "fontTools/cu2qu/cu2qu.py":23
+ * # if cython not installed, use mock module with no-op decorators and types
+ * from fontTools.misc import cython
+ * COMPILED = cython.compiled # <<<<<<<<<<<<<<
+ *
+ * import math
+*/
+ if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_COMPILED, Py_True) < 0) __PYX_ERR(0, 23, __pyx_L1_error)
+
+ /* "fontTools/cu2qu/cu2qu.py":25
+ * COMPILED = cython.compiled
+ *
+ * import math # <<<<<<<<<<<<<<
+ *
+ * from .errors import Error as Cu2QuError, ApproxNotFoundError
+*/
+ __pyx_t_4 = __Pyx_ImportDottedModule(__pyx_mstate_global->__pyx_n_u_math, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 25, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_4);
+ if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_math, __pyx_t_4) < 0) __PYX_ERR(0, 25, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
+
+ /* "fontTools/cu2qu/cu2qu.py":27
+ * import math
+ *
+ * from .errors import Error as Cu2QuError, ApproxNotFoundError # <<<<<<<<<<<<<<
+ *
+ *
+*/
+ __pyx_t_4 = __Pyx_PyList_Pack(2, __pyx_mstate_global->__pyx_n_u_Error, __pyx_mstate_global->__pyx_n_u_ApproxNotFoundError); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 27, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_4);
+ __pyx_t_5 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_errors, __pyx_t_4, 1); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 27, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_5);
+ __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
+ __pyx_t_4 = __Pyx_ImportFrom(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_Error); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 27, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_4);
+ if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_Cu2QuError, __pyx_t_4) < 0) __PYX_ERR(0, 27, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
+ __pyx_t_4 = __Pyx_ImportFrom(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_ApproxNotFoundError); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 27, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_4);
+ if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_ApproxNotFoundError, __pyx_t_4) < 0) __PYX_ERR(0, 27, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
+ __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
+
+ /* "fontTools/cu2qu/cu2qu.py":30
+ *
+ *
+ * __all__ = ["curve_to_quadratic", "curves_to_quadratic"] # <<<<<<<<<<<<<<
+ *
+ * MAX_N = 100
+*/
+ __pyx_t_5 = __Pyx_PyList_Pack(2, __pyx_mstate_global->__pyx_n_u_curve_to_quadratic, __pyx_mstate_global->__pyx_n_u_curves_to_quadratic); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 30, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_5);
+ if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_all, __pyx_t_5) < 0) __PYX_ERR(0, 30, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
+
+ /* "fontTools/cu2qu/cu2qu.py":32
+ * __all__ = ["curve_to_quadratic", "curves_to_quadratic"]
+ *
+ * MAX_N = 100 # <<<<<<<<<<<<<<
+ *
+ * NAN = float("NaN")
+*/
+ if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_MAX_N, __pyx_mstate_global->__pyx_int_100) < 0) __PYX_ERR(0, 32, __pyx_L1_error)
+
+ /* "fontTools/cu2qu/cu2qu.py":34
+ * MAX_N = 100
+ *
+ * NAN = float("NaN") # <<<<<<<<<<<<<<
+ *
+ *
+*/
+ __pyx_t_6 = __Pyx_PyUnicode_AsDouble(__pyx_mstate_global->__pyx_n_u_NaN); if (unlikely(__pyx_t_6 == ((double)((double)-1)) && PyErr_Occurred())) __PYX_ERR(0, 34, __pyx_L1_error)
+ __pyx_t_5 = PyFloat_FromDouble(__pyx_t_6); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 34, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_5);
+ if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_NAN, __pyx_t_5) < 0) __PYX_ERR(0, 34, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
+
+ /* "fontTools/cu2qu/cu2qu.py":150
+ *
+ *
+ * @cython.locals( # <<<<<<<<<<<<<<
+ * p0=cython.complex,
+ * p1=cython.complex,
+*/
+ __pyx_t_5 = __Pyx_CyFunction_New(&__pyx_mdef_9fontTools_5cu2qu_5cu2qu_1_split_cubic_into_n_gen, 0, __pyx_mstate_global->__pyx_n_u_split_cubic_into_n_gen, NULL, __pyx_mstate_global->__pyx_n_u_fontTools_cu2qu_cu2qu, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[0])); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 150, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_5);
+ if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_split_cubic_into_n_gen, __pyx_t_5) < 0) __PYX_ERR(0, 150, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
+
+ /* "fontTools/cu2qu/cu2qu.py":471
+ * @cython.locals(n=cython.int)
+ * @cython.locals(all_quadratic=cython.int)
+ * def curve_to_quadratic(curve, max_err, all_quadratic=True): # <<<<<<<<<<<<<<
+ * """Approximate a cubic Bezier curve with a spline of n quadratics.
+ *
+*/
+ __pyx_t_5 = __Pyx_PyBool_FromLong(((int)1)); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 471, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_5);
+
+ /* "fontTools/cu2qu/cu2qu.py":468
+ *
+ *
+ * @cython.locals(max_err=cython.double) # <<<<<<<<<<<<<<
+ * @cython.locals(n=cython.int)
+ * @cython.locals(all_quadratic=cython.int)
+*/
+ __pyx_t_4 = PyTuple_Pack(1, __pyx_t_5); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 468, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_4);
+ __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
+ __pyx_t_5 = __Pyx_CyFunction_New(&__pyx_mdef_9fontTools_5cu2qu_5cu2qu_4curve_to_quadratic, 0, __pyx_mstate_global->__pyx_n_u_curve_to_quadratic, NULL, __pyx_mstate_global->__pyx_n_u_fontTools_cu2qu_cu2qu, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[1])); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 468, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_5);
+ __Pyx_CyFunction_SetDefaultsTuple(__pyx_t_5, __pyx_t_4);
+ __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
+ if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_curve_to_quadratic, __pyx_t_5) < 0) __PYX_ERR(0, 468, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
+
+ /* "fontTools/cu2qu/cu2qu.py":505
+ * @cython.locals(l=cython.int, last_i=cython.int, i=cython.int)
+ * @cython.locals(all_quadratic=cython.int)
+ * def curves_to_quadratic(curves, max_errors, all_quadratic=True): # <<<<<<<<<<<<<<
+ * """Return quadratic Bezier splines approximating the input cubic Beziers.
+ *
+*/
+ __pyx_t_5 = __Pyx_PyBool_FromLong(((int)1)); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 505, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_5);
+
+ /* "fontTools/cu2qu/cu2qu.py":503
+ *
+ *
+ * @cython.locals(l=cython.int, last_i=cython.int, i=cython.int) # <<<<<<<<<<<<<<
+ * @cython.locals(all_quadratic=cython.int)
+ * def curves_to_quadratic(curves, max_errors, all_quadratic=True):
+*/
+ __pyx_t_4 = PyTuple_Pack(1, __pyx_t_5); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 503, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_4);
+ __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
+ __pyx_t_5 = __Pyx_CyFunction_New(&__pyx_mdef_9fontTools_5cu2qu_5cu2qu_6curves_to_quadratic, 0, __pyx_mstate_global->__pyx_n_u_curves_to_quadratic, NULL, __pyx_mstate_global->__pyx_n_u_fontTools_cu2qu_cu2qu, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[2])); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 503, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_5);
+ __Pyx_CyFunction_SetDefaultsTuple(__pyx_t_5, __pyx_t_4);
+ __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
+ if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_curves_to_quadratic, __pyx_t_5) < 0) __PYX_ERR(0, 503, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
+
+ /* "fontTools/cu2qu/cu2qu.py":1
+ * # cython: language_level=3 # <<<<<<<<<<<<<<
+ * # distutils: define_macros=CYTHON_TRACE_NOGIL=1
+ *
+*/
+ __pyx_t_5 = __Pyx_PyDict_NewPresized(1); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 1, __pyx_L1_error)
+ __Pyx_GOTREF(__pyx_t_5);
+ if (PyDict_SetItem(__pyx_t_5, __pyx_mstate_global->__pyx_kp_u_curves_to_quadratic_line_503, __pyx_mstate_global->__pyx_kp_u_Return_quadratic_Bezier_splines) < 0) __PYX_ERR(0, 1, __pyx_L1_error)
+ if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_test, __pyx_t_5) < 0) __PYX_ERR(0, 1, __pyx_L1_error)
+ __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
+
+ /*--- Wrapped vars code ---*/
+
+ goto __pyx_L0;
+ __pyx_L1_error:;
+ __Pyx_XDECREF(__pyx_t_4);
+ __Pyx_XDECREF(__pyx_t_5);
+ if (__pyx_m) {
+ if (__pyx_mstate->__pyx_d && stringtab_initialized) {
+ __Pyx_AddTraceback("init fontTools.cu2qu.cu2qu", __pyx_clineno, __pyx_lineno, __pyx_filename);
+ }
+ #if !CYTHON_USE_MODULE_STATE
+ Py_CLEAR(__pyx_m);
+ #else
+ Py_DECREF(__pyx_m);
+ if (pystate_addmodule_run) {
+ PyObject *tp, *value, *tb;
+ PyErr_Fetch(&tp, &value, &tb);
+ PyState_RemoveModule(&__pyx_moduledef);
+ PyErr_Restore(tp, value, tb);
+ }
+ #endif
+ } else if (!PyErr_Occurred()) {
+ PyErr_SetString(PyExc_ImportError, "init fontTools.cu2qu.cu2qu");
+ }
+ __pyx_L0:;
+ __Pyx_RefNannyFinishContext();
+ #if CYTHON_PEP489_MULTI_PHASE_INIT
+ return (__pyx_m != NULL) ? 0 : -1;
+ #else
+ return __pyx_m;
+ #endif
+}
+/* #### Code section: pystring_table ### */
+
+typedef struct {
+ const char *s;
+#if 1602 <= 65535
+ const unsigned short n;
+#elif 1602 / 2 < INT_MAX
+ const unsigned int n;
+#elif 1602 / 2 < LONG_MAX
+ const unsigned long n;
+#else
+ const Py_ssize_t n;
+#endif
+#if 1 <= 31
+ const unsigned int encoding : 5;
+#elif 1 <= 255
+ const unsigned char encoding;
+#elif 1 <= 65535
+ const unsigned short encoding;
+#else
+ const Py_ssize_t encoding;
+#endif
+ const unsigned int is_unicode : 1;
+ const unsigned int intern : 1;
+} __Pyx_StringTabEntry;
+static const char * const __pyx_string_tab_encodings[] = { 0 };
+static const __Pyx_StringTabEntry __pyx_string_tab[] = {
+ {__pyx_k_, sizeof(__pyx_k_), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_ */
+ {__pyx_k_ApproxNotFoundError, sizeof(__pyx_k_ApproxNotFoundError), 0, 1, 1}, /* PyObject cname: __pyx_n_u_ApproxNotFoundError */
+ {__pyx_k_AssertionError, sizeof(__pyx_k_AssertionError), 0, 1, 1}, /* PyObject cname: __pyx_n_u_AssertionError */
+ {__pyx_k_AttributeError, sizeof(__pyx_k_AttributeError), 0, 1, 1}, /* PyObject cname: __pyx_n_u_AttributeError */
+ {__pyx_k_COMPILED, sizeof(__pyx_k_COMPILED), 0, 1, 1}, /* PyObject cname: __pyx_n_u_COMPILED */
+ {__pyx_k_Cu2QuError, sizeof(__pyx_k_Cu2QuError), 0, 1, 1}, /* PyObject cname: __pyx_n_u_Cu2QuError */
+ {__pyx_k_Error, sizeof(__pyx_k_Error), 0, 1, 1}, /* PyObject cname: __pyx_n_u_Error */
+ {__pyx_k_ImportError, sizeof(__pyx_k_ImportError), 0, 1, 1}, /* PyObject cname: __pyx_n_u_ImportError */
+ {__pyx_k_Lib_fontTools_cu2qu_cu2qu_py, sizeof(__pyx_k_Lib_fontTools_cu2qu_cu2qu_py), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Lib_fontTools_cu2qu_cu2qu_py */
+ {__pyx_k_MAX_N, sizeof(__pyx_k_MAX_N), 0, 1, 1}, /* PyObject cname: __pyx_n_u_MAX_N */
+ {__pyx_k_NAN, sizeof(__pyx_k_NAN), 0, 1, 1}, /* PyObject cname: __pyx_n_u_NAN */
+ {__pyx_k_NaN, sizeof(__pyx_k_NaN), 0, 1, 1}, /* PyObject cname: __pyx_n_u_NaN */
+ {__pyx_k_Return_quadratic_Bezier_splines, sizeof(__pyx_k_Return_quadratic_Bezier_splines), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Return_quadratic_Bezier_splines */
+ {__pyx_k_ZeroDivisionError, sizeof(__pyx_k_ZeroDivisionError), 0, 1, 1}, /* PyObject cname: __pyx_n_u_ZeroDivisionError */
+ {__pyx_k__2, sizeof(__pyx_k__2), 0, 1, 0}, /* PyObject cname: __pyx_kp_u__2 */
+ {__pyx_k_a, sizeof(__pyx_k_a), 0, 1, 1}, /* PyObject cname: __pyx_n_u_a */
+ {__pyx_k_a1, sizeof(__pyx_k_a1), 0, 1, 1}, /* PyObject cname: __pyx_n_u_a1 */
+ {__pyx_k_all, sizeof(__pyx_k_all), 0, 1, 1}, /* PyObject cname: __pyx_n_u_all */
+ {__pyx_k_all_quadratic, sizeof(__pyx_k_all_quadratic), 0, 1, 1}, /* PyObject cname: __pyx_n_u_all_quadratic */
+ {__pyx_k_asyncio_coroutines, sizeof(__pyx_k_asyncio_coroutines), 0, 1, 1}, /* PyObject cname: __pyx_n_u_asyncio_coroutines */
+ {__pyx_k_b, sizeof(__pyx_k_b), 0, 1, 1}, /* PyObject cname: __pyx_n_u_b */
+ {__pyx_k_b1, sizeof(__pyx_k_b1), 0, 1, 1}, /* PyObject cname: __pyx_n_u_b1 */
+ {__pyx_k_c, sizeof(__pyx_k_c), 0, 1, 1}, /* PyObject cname: __pyx_n_u_c */
+ {__pyx_k_c1, sizeof(__pyx_k_c1), 0, 1, 1}, /* PyObject cname: __pyx_n_u_c1 */
+ {__pyx_k_cline_in_traceback, sizeof(__pyx_k_cline_in_traceback), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cline_in_traceback */
+ {__pyx_k_close, sizeof(__pyx_k_close), 0, 1, 1}, /* PyObject cname: __pyx_n_u_close */
+ {__pyx_k_curve, sizeof(__pyx_k_curve), 0, 1, 1}, /* PyObject cname: __pyx_n_u_curve */
+ {__pyx_k_curve_to_quadratic, sizeof(__pyx_k_curve_to_quadratic), 0, 1, 1}, /* PyObject cname: __pyx_n_u_curve_to_quadratic */
+ {__pyx_k_curves, sizeof(__pyx_k_curves), 0, 1, 1}, /* PyObject cname: __pyx_n_u_curves */
+ {__pyx_k_curves_to_quadratic, sizeof(__pyx_k_curves_to_quadratic), 0, 1, 1}, /* PyObject cname: __pyx_n_u_curves_to_quadratic */
+ {__pyx_k_curves_to_quadratic_line_503, sizeof(__pyx_k_curves_to_quadratic_line_503), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_curves_to_quadratic_line_503 */
+ {__pyx_k_d, sizeof(__pyx_k_d), 0, 1, 1}, /* PyObject cname: __pyx_n_u_d */
+ {__pyx_k_d1, sizeof(__pyx_k_d1), 0, 1, 1}, /* PyObject cname: __pyx_n_u_d1 */
+ {__pyx_k_delta_2, sizeof(__pyx_k_delta_2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_delta_2 */
+ {__pyx_k_delta_3, sizeof(__pyx_k_delta_3), 0, 1, 1}, /* PyObject cname: __pyx_n_u_delta_3 */
+ {__pyx_k_disable, sizeof(__pyx_k_disable), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_disable */
+ {__pyx_k_dt, sizeof(__pyx_k_dt), 0, 1, 1}, /* PyObject cname: __pyx_n_u_dt */
+ {__pyx_k_enable, sizeof(__pyx_k_enable), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_enable */
+ {__pyx_k_errors, sizeof(__pyx_k_errors), 0, 1, 1}, /* PyObject cname: __pyx_n_u_errors */
+ {__pyx_k_fontTools_cu2qu_cu2qu, sizeof(__pyx_k_fontTools_cu2qu_cu2qu), 0, 1, 1}, /* PyObject cname: __pyx_n_u_fontTools_cu2qu_cu2qu */
+ {__pyx_k_func, sizeof(__pyx_k_func), 0, 1, 1}, /* PyObject cname: __pyx_n_u_func */
+ {__pyx_k_gc, sizeof(__pyx_k_gc), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_gc */
+ {__pyx_k_i, sizeof(__pyx_k_i), 0, 1, 1}, /* PyObject cname: __pyx_n_u_i */
+ {__pyx_k_imag, sizeof(__pyx_k_imag), 0, 1, 1}, /* PyObject cname: __pyx_n_u_imag */
+ {__pyx_k_initializing, sizeof(__pyx_k_initializing), 0, 1, 1}, /* PyObject cname: __pyx_n_u_initializing */
+ {__pyx_k_is_coroutine, sizeof(__pyx_k_is_coroutine), 0, 1, 1}, /* PyObject cname: __pyx_n_u_is_coroutine */
+ {__pyx_k_isenabled, sizeof(__pyx_k_isenabled), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_isenabled */
+ {__pyx_k_isnan, sizeof(__pyx_k_isnan), 0, 1, 1}, /* PyObject cname: __pyx_n_u_isnan */
+ {__pyx_k_l, sizeof(__pyx_k_l), 0, 1, 1}, /* PyObject cname: __pyx_n_u_l */
+ {__pyx_k_last_i, sizeof(__pyx_k_last_i), 0, 1, 1}, /* PyObject cname: __pyx_n_u_last_i */
+ {__pyx_k_main, sizeof(__pyx_k_main), 0, 1, 1}, /* PyObject cname: __pyx_n_u_main */
+ {__pyx_k_math, sizeof(__pyx_k_math), 0, 1, 1}, /* PyObject cname: __pyx_n_u_math */
+ {__pyx_k_max_err, sizeof(__pyx_k_max_err), 0, 1, 1}, /* PyObject cname: __pyx_n_u_max_err */
+ {__pyx_k_max_errors, sizeof(__pyx_k_max_errors), 0, 1, 1}, /* PyObject cname: __pyx_n_u_max_errors */
+ {__pyx_k_module, sizeof(__pyx_k_module), 0, 1, 1}, /* PyObject cname: __pyx_n_u_module */
+ {__pyx_k_n, sizeof(__pyx_k_n), 0, 1, 1}, /* PyObject cname: __pyx_n_u_n */
+ {__pyx_k_name, sizeof(__pyx_k_name), 0, 1, 1}, /* PyObject cname: __pyx_n_u_name */
+ {__pyx_k_next, sizeof(__pyx_k_next), 0, 1, 1}, /* PyObject cname: __pyx_n_u_next */
+ {__pyx_k_p, sizeof(__pyx_k_p), 0, 1, 1}, /* PyObject cname: __pyx_n_u_p */
+ {__pyx_k_p0, sizeof(__pyx_k_p0), 0, 1, 1}, /* PyObject cname: __pyx_n_u_p0 */
+ {__pyx_k_p1, sizeof(__pyx_k_p1), 0, 1, 1}, /* PyObject cname: __pyx_n_u_p1 */
+ {__pyx_k_p2, sizeof(__pyx_k_p2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_p2 */
+ {__pyx_k_p3, sizeof(__pyx_k_p3), 0, 1, 1}, /* PyObject cname: __pyx_n_u_p3 */
+ {__pyx_k_pop, sizeof(__pyx_k_pop), 0, 1, 1}, /* PyObject cname: __pyx_n_u_pop */
+ {__pyx_k_qualname, sizeof(__pyx_k_qualname), 0, 1, 1}, /* PyObject cname: __pyx_n_u_qualname */
+ {__pyx_k_range, sizeof(__pyx_k_range), 0, 1, 1}, /* PyObject cname: __pyx_n_u_range */
+ {__pyx_k_real, sizeof(__pyx_k_real), 0, 1, 1}, /* PyObject cname: __pyx_n_u_real */
+ {__pyx_k_s, sizeof(__pyx_k_s), 0, 1, 1}, /* PyObject cname: __pyx_n_u_s */
+ {__pyx_k_send, sizeof(__pyx_k_send), 0, 1, 1}, /* PyObject cname: __pyx_n_u_send */
+ {__pyx_k_set_name, sizeof(__pyx_k_set_name), 0, 1, 1}, /* PyObject cname: __pyx_n_u_set_name */
+ {__pyx_k_spec, sizeof(__pyx_k_spec), 0, 1, 1}, /* PyObject cname: __pyx_n_u_spec */
+ {__pyx_k_spline, sizeof(__pyx_k_spline), 0, 1, 1}, /* PyObject cname: __pyx_n_u_spline */
+ {__pyx_k_splines, sizeof(__pyx_k_splines), 0, 1, 1}, /* PyObject cname: __pyx_n_u_splines */
+ {__pyx_k_split_cubic_into_n_gen, sizeof(__pyx_k_split_cubic_into_n_gen), 0, 1, 1}, /* PyObject cname: __pyx_n_u_split_cubic_into_n_gen */
+ {__pyx_k_t1, sizeof(__pyx_k_t1), 0, 1, 1}, /* PyObject cname: __pyx_n_u_t1 */
+ {__pyx_k_t1_2, sizeof(__pyx_k_t1_2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_t1_2 */
+ {__pyx_k_test, sizeof(__pyx_k_test), 0, 1, 1}, /* PyObject cname: __pyx_n_u_test */
+ {__pyx_k_throw, sizeof(__pyx_k_throw), 0, 1, 1}, /* PyObject cname: __pyx_n_u_throw */
+ {__pyx_k_value, sizeof(__pyx_k_value), 0, 1, 1}, /* PyObject cname: __pyx_n_u_value */
+ {0, 0, 0, 0, 0}
+};
+/* InitStrings.proto */
+static int __Pyx_InitStrings(__Pyx_StringTabEntry const *t, PyObject **target, const char* const* encoding_names);
+
+/* #### Code section: cached_builtins ### */
+
+static int __Pyx_InitCachedBuiltins(__pyx_mstatetype *__pyx_mstate) {
+ CYTHON_UNUSED_VAR(__pyx_mstate);
+ __pyx_builtin_AttributeError = __Pyx_GetBuiltinName(__pyx_mstate->__pyx_n_u_AttributeError); if (!__pyx_builtin_AttributeError) __PYX_ERR(0, 20, __pyx_L1_error)
+ __pyx_builtin_ImportError = __Pyx_GetBuiltinName(__pyx_mstate->__pyx_n_u_ImportError); if (!__pyx_builtin_ImportError) __PYX_ERR(0, 20, __pyx_L1_error)
+ __pyx_builtin_range = __Pyx_GetBuiltinName(__pyx_mstate->__pyx_n_u_range); if (!__pyx_builtin_range) __PYX_ERR(0, 169, __pyx_L1_error)
+ __pyx_builtin_ZeroDivisionError = __Pyx_GetBuiltinName(__pyx_mstate->__pyx_n_u_ZeroDivisionError); if (!__pyx_builtin_ZeroDivisionError) __PYX_ERR(0, 301, __pyx_L1_error)
+ __pyx_builtin_AssertionError = __Pyx_GetBuiltinName(__pyx_mstate->__pyx_n_u_AssertionError); if (!__pyx_builtin_AssertionError) __PYX_ERR(0, 543, __pyx_L1_error)
+ return 0;
+ __pyx_L1_error:;
+ return -1;
+}
+/* #### Code section: cached_constants ### */
+
+static int __Pyx_InitCachedConstants(__pyx_mstatetype *__pyx_mstate) {
+ __Pyx_RefNannyDeclarations
+ CYTHON_UNUSED_VAR(__pyx_mstate);
+ __Pyx_RefNannySetupContext("__Pyx_InitCachedConstants", 0);
+ __Pyx_RefNannyFinishContext();
+ return 0;
+}
+/* #### Code section: init_constants ### */
+
+static int __Pyx_InitConstants(__pyx_mstatetype *__pyx_mstate) {
+ CYTHON_UNUSED_VAR(__pyx_mstate);
+ __pyx_mstate->__pyx_umethod_PyDict_Type_pop.type = (PyObject*)&PyDict_Type;
+ __pyx_mstate->__pyx_umethod_PyDict_Type_pop.method_name = &__pyx_mstate->__pyx_n_u_pop;
+ if (__Pyx_InitStrings(__pyx_string_tab, __pyx_mstate->__pyx_string_tab, __pyx_string_tab_encodings) < 0) __PYX_ERR(0, 1, __pyx_L1_error);
+ __pyx_mstate->__pyx_int_1 = PyLong_FromLong(1); if (unlikely(!__pyx_mstate->__pyx_int_1)) __PYX_ERR(0, 1, __pyx_L1_error)
+ __pyx_mstate->__pyx_int_2 = PyLong_FromLong(2); if (unlikely(!__pyx_mstate->__pyx_int_2)) __PYX_ERR(0, 1, __pyx_L1_error)
+ __pyx_mstate->__pyx_int_3 = PyLong_FromLong(3); if (unlikely(!__pyx_mstate->__pyx_int_3)) __PYX_ERR(0, 1, __pyx_L1_error)
+ __pyx_mstate->__pyx_int_4 = PyLong_FromLong(4); if (unlikely(!__pyx_mstate->__pyx_int_4)) __PYX_ERR(0, 1, __pyx_L1_error)
+ __pyx_mstate->__pyx_int_6 = PyLong_FromLong(6); if (unlikely(!__pyx_mstate->__pyx_int_6)) __PYX_ERR(0, 1, __pyx_L1_error)
+ __pyx_mstate->__pyx_int_100 = PyLong_FromLong(100); if (unlikely(!__pyx_mstate->__pyx_int_100)) __PYX_ERR(0, 1, __pyx_L1_error)
+ return 0;
+ __pyx_L1_error:;
+ return -1;
+}
+/* #### Code section: init_codeobjects ### */
+\
+ typedef struct {
+ unsigned int argcount : 3;
+ unsigned int num_posonly_args : 1;
+ unsigned int num_kwonly_args : 1;
+ unsigned int nlocals : 5;
+ unsigned int flags : 10;
+ unsigned int first_line : 9;
+ unsigned int line_table_length : 13;
+ } __Pyx_PyCode_New_function_description;
+/* NewCodeObj.proto */
+static PyObject* __Pyx_PyCode_New(
+ const __Pyx_PyCode_New_function_description descr,
+ PyObject * const *varnames,
+ PyObject *filename,
+ PyObject *funcname,
+ const char *line_table,
+ PyObject *tuple_dedup_map
+);
+
+
+static int __Pyx_CreateCodeObjects(__pyx_mstatetype *__pyx_mstate) {
+ PyObject* tuple_dedup_map = PyDict_New();
+ if (unlikely(!tuple_dedup_map)) return -1;
+ {
+ const __Pyx_PyCode_New_function_description descr = {5, 0, 0, 19, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS|CO_GENERATOR), 150, 2};
+ PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_p0, __pyx_mstate->__pyx_n_u_p1, __pyx_mstate->__pyx_n_u_p2, __pyx_mstate->__pyx_n_u_p3, __pyx_mstate->__pyx_n_u_n, __pyx_mstate->__pyx_n_u_a1, __pyx_mstate->__pyx_n_u_b1, __pyx_mstate->__pyx_n_u_c1, __pyx_mstate->__pyx_n_u_d1, __pyx_mstate->__pyx_n_u_dt, __pyx_mstate->__pyx_n_u_delta_2, __pyx_mstate->__pyx_n_u_delta_3, __pyx_mstate->__pyx_n_u_i, __pyx_mstate->__pyx_n_u_a, __pyx_mstate->__pyx_n_u_b, __pyx_mstate->__pyx_n_u_c, __pyx_mstate->__pyx_n_u_d, __pyx_mstate->__pyx_n_u_t1, __pyx_mstate->__pyx_n_u_t1_2};
+ __pyx_mstate_global->__pyx_codeobj_tab[0] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_Lib_fontTools_cu2qu_cu2qu_py, __pyx_mstate->__pyx_n_u_split_cubic_into_n_gen, __pyx_k__3, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[0])) goto bad;
+ }
+ {
+ const __Pyx_PyCode_New_function_description descr = {3, 0, 0, 7, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 468, 97};
+ PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_curve, __pyx_mstate->__pyx_n_u_max_err, __pyx_mstate->__pyx_n_u_all_quadratic, __pyx_mstate->__pyx_n_u_n, __pyx_mstate->__pyx_n_u_spline, __pyx_mstate->__pyx_n_u_p, __pyx_mstate->__pyx_n_u_s};
+ __pyx_mstate_global->__pyx_codeobj_tab[1] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_Lib_fontTools_cu2qu_cu2qu_py, __pyx_mstate->__pyx_n_u_curve_to_quadratic, __pyx_k_AWBc_U_U_3fBa_AWCy_7_2QgQgT_a_Q, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[1])) goto bad;
+ }
+ {
+ const __Pyx_PyCode_New_function_description descr = {3, 0, 0, 13, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 503, 211};
+ PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_curves, __pyx_mstate->__pyx_n_u_max_errors, __pyx_mstate->__pyx_n_u_all_quadratic, __pyx_mstate->__pyx_n_u_l, __pyx_mstate->__pyx_n_u_last_i, __pyx_mstate->__pyx_n_u_i, __pyx_mstate->__pyx_n_u_splines, __pyx_mstate->__pyx_n_u_n, __pyx_mstate->__pyx_n_u_spline, __pyx_mstate->__pyx_n_u_curve, __pyx_mstate->__pyx_n_u_p, __pyx_mstate->__pyx_n_u_spline, __pyx_mstate->__pyx_n_u_s};
+ __pyx_mstate_global->__pyx_codeobj_tab[2] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_Lib_fontTools_cu2qu_cu2qu_py, __pyx_mstate->__pyx_n_u_curves_to_quadratic, __pyx_k_J_Qawb_4uG4y_3a_3c_1A_avRq_T_AV, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[2])) goto bad;
+ }
+ Py_DECREF(tuple_dedup_map);
+ return 0;
+ bad:
+ Py_DECREF(tuple_dedup_map);
+ return -1;
+}
+/* #### Code section: init_globals ### */
+
+static int __Pyx_InitGlobals(void) {
+ /* PythonCompatibility.init */
+ if (likely(__Pyx_init_co_variables() == 0)); else
+
+if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 1, __pyx_L1_error)
+
+ /* AssertionsEnabled.init */
+ if (likely(__Pyx_init_assertions_enabled() == 0)); else
+
+if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 1, __pyx_L1_error)
+
+ /* CachedMethodType.init */
+ #if CYTHON_COMPILING_IN_LIMITED_API
+{
+ PyObject *typesModule=NULL;
+ typesModule = PyImport_ImportModule("types");
+ if (typesModule) {
+ __pyx_mstate_global->__Pyx_CachedMethodType = PyObject_GetAttrString(typesModule, "MethodType");
+ Py_DECREF(typesModule);
+ }
+} // error handling follows
+#endif
+
+if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 1, __pyx_L1_error)
+
+ return 0;
+ __pyx_L1_error:;
+ return -1;
+}
+/* #### Code section: cleanup_globals ### */
+/* #### Code section: cleanup_module ### */
+/* #### Code section: main_method ### */
+/* #### Code section: utility_code_pragmas ### */
+#ifdef _MSC_VER
+#pragma warning( push )
+/* Warning 4127: conditional expression is constant
+ * Cython uses constant conditional expressions to allow in inline functions to be optimized at
+ * compile-time, so this warning is not useful
+ */
+#pragma warning( disable : 4127 )
+#endif
+
+
+
+/* #### Code section: utility_code_def ### */
+
+/* --- Runtime support code --- */
+/* Refnanny */
+#if CYTHON_REFNANNY
+static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname) {
+ PyObject *m = NULL, *p = NULL;
+ void *r = NULL;
+ m = PyImport_ImportModule(modname);
+ if (!m) goto end;
+ p = PyObject_GetAttrString(m, "RefNannyAPI");
+ if (!p) goto end;
+ r = PyLong_AsVoidPtr(p);
+end:
+ Py_XDECREF(p);
+ Py_XDECREF(m);
+ return (__Pyx_RefNannyAPIStruct *)r;
+}
+#endif
+
+/* PyErrExceptionMatches */
+#if CYTHON_FAST_THREAD_STATE
+static int __Pyx_PyErr_ExceptionMatchesTuple(PyObject *exc_type, PyObject *tuple) {
+ Py_ssize_t i, n;
+ n = PyTuple_GET_SIZE(tuple);
+ for (i=0; i= 0x030C00A6
+ PyObject *current_exception = tstate->current_exception;
+ if (unlikely(!current_exception)) return 0;
+ exc_type = (PyObject*) Py_TYPE(current_exception);
+ if (exc_type == err) return 1;
+#else
+ exc_type = tstate->curexc_type;
+ if (exc_type == err) return 1;
+ if (unlikely(!exc_type)) return 0;
+#endif
+ #if CYTHON_AVOID_BORROWED_REFS
+ Py_INCREF(exc_type);
+ #endif
+ if (unlikely(PyTuple_Check(err))) {
+ result = __Pyx_PyErr_ExceptionMatchesTuple(exc_type, err);
+ } else {
+ result = __Pyx_PyErr_GivenExceptionMatches(exc_type, err);
+ }
+ #if CYTHON_AVOID_BORROWED_REFS
+ Py_DECREF(exc_type);
+ #endif
+ return result;
+}
+#endif
+
+/* PyErrFetchRestore */
+#if CYTHON_FAST_THREAD_STATE
+static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) {
+#if PY_VERSION_HEX >= 0x030C00A6
+ PyObject *tmp_value;
+ assert(type == NULL || (value != NULL && type == (PyObject*) Py_TYPE(value)));
+ if (value) {
+ #if CYTHON_COMPILING_IN_CPYTHON
+ if (unlikely(((PyBaseExceptionObject*) value)->traceback != tb))
+ #endif
+ PyException_SetTraceback(value, tb);
+ }
+ tmp_value = tstate->current_exception;
+ tstate->current_exception = value;
+ Py_XDECREF(tmp_value);
+ Py_XDECREF(type);
+ Py_XDECREF(tb);
+#else
+ PyObject *tmp_type, *tmp_value, *tmp_tb;
+ tmp_type = tstate->curexc_type;
+ tmp_value = tstate->curexc_value;
+ tmp_tb = tstate->curexc_traceback;
+ tstate->curexc_type = type;
+ tstate->curexc_value = value;
+ tstate->curexc_traceback = tb;
+ Py_XDECREF(tmp_type);
+ Py_XDECREF(tmp_value);
+ Py_XDECREF(tmp_tb);
+#endif
+}
+static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) {
+#if PY_VERSION_HEX >= 0x030C00A6
+ PyObject* exc_value;
+ exc_value = tstate->current_exception;
+ tstate->current_exception = 0;
+ *value = exc_value;
+ *type = NULL;
+ *tb = NULL;
+ if (exc_value) {
+ *type = (PyObject*) Py_TYPE(exc_value);
+ Py_INCREF(*type);
+ #if CYTHON_COMPILING_IN_CPYTHON
+ *tb = ((PyBaseExceptionObject*) exc_value)->traceback;
+ Py_XINCREF(*tb);
+ #else
+ *tb = PyException_GetTraceback(exc_value);
+ #endif
+ }
+#else
+ *type = tstate->curexc_type;
+ *value = tstate->curexc_value;
+ *tb = tstate->curexc_traceback;
+ tstate->curexc_type = 0;
+ tstate->curexc_value = 0;
+ tstate->curexc_traceback = 0;
+#endif
+}
+#endif
+
+/* PyObjectGetAttrStr */
+#if CYTHON_USE_TYPE_SLOTS
+static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject* attr_name) {
+ PyTypeObject* tp = Py_TYPE(obj);
+ if (likely(tp->tp_getattro))
+ return tp->tp_getattro(obj, attr_name);
+ return PyObject_GetAttr(obj, attr_name);
+}
+#endif
+
+/* PyObjectGetAttrStrNoError */
+#if __PYX_LIMITED_VERSION_HEX < 0x030d0000
+static void __Pyx_PyObject_GetAttrStr_ClearAttributeError(void) {
+ __Pyx_PyThreadState_declare
+ __Pyx_PyThreadState_assign
+ if (likely(__Pyx_PyErr_ExceptionMatches(PyExc_AttributeError)))
+ __Pyx_PyErr_Clear();
+}
+#endif
+static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStrNoError(PyObject* obj, PyObject* attr_name) {
+ PyObject *result;
+#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000
+ (void) PyObject_GetOptionalAttr(obj, attr_name, &result);
+ return result;
+#else
+#if CYTHON_COMPILING_IN_CPYTHON && CYTHON_USE_TYPE_SLOTS
+ PyTypeObject* tp = Py_TYPE(obj);
+ if (likely(tp->tp_getattro == PyObject_GenericGetAttr)) {
+ return _PyObject_GenericGetAttrWithDict(obj, attr_name, NULL, 1);
+ }
+#endif
+ result = __Pyx_PyObject_GetAttrStr(obj, attr_name);
+ if (unlikely(!result)) {
+ __Pyx_PyObject_GetAttrStr_ClearAttributeError();
+ }
+ return result;
+#endif
+}
+
+/* GetBuiltinName */
+static PyObject *__Pyx_GetBuiltinName(PyObject *name) {
+ PyObject* result = __Pyx_PyObject_GetAttrStrNoError(__pyx_mstate_global->__pyx_b, name);
+ if (unlikely(!result) && !PyErr_Occurred()) {
+ PyErr_Format(PyExc_NameError,
+ "name '%U' is not defined", name);
+ }
+ return result;
+}
+
+/* PyFunctionFastCall */
+#if CYTHON_FAST_PYCALL && !CYTHON_VECTORCALL
+static PyObject* __Pyx_PyFunction_FastCallNoKw(PyCodeObject *co, PyObject *const *args, Py_ssize_t na,
+ PyObject *globals) {
+ PyFrameObject *f;
+ PyThreadState *tstate = __Pyx_PyThreadState_Current;
+ PyObject **fastlocals;
+ Py_ssize_t i;
+ PyObject *result;
+ assert(globals != NULL);
+ /* XXX Perhaps we should create a specialized
+ PyFrame_New() that doesn't take locals, but does
+ take builtins without sanity checking them.
+ */
+ assert(tstate != NULL);
+ f = PyFrame_New(tstate, co, globals, NULL);
+ if (f == NULL) {
+ return NULL;
+ }
+ fastlocals = __Pyx_PyFrame_GetLocalsplus(f);
+ for (i = 0; i < na; i++) {
+ Py_INCREF(*args);
+ fastlocals[i] = *args++;
+ }
+ result = PyEval_EvalFrameEx(f,0);
+ ++tstate->recursion_depth;
+ Py_DECREF(f);
+ --tstate->recursion_depth;
+ return result;
+}
+static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject *const *args, Py_ssize_t nargs, PyObject *kwargs) {
+ PyCodeObject *co = (PyCodeObject *)PyFunction_GET_CODE(func);
+ PyObject *globals = PyFunction_GET_GLOBALS(func);
+ PyObject *argdefs = PyFunction_GET_DEFAULTS(func);
+ PyObject *closure;
+ PyObject *kwdefs;
+ PyObject *kwtuple, **k;
+ PyObject **d;
+ Py_ssize_t nd;
+ Py_ssize_t nk;
+ PyObject *result;
+ assert(kwargs == NULL || PyDict_Check(kwargs));
+ nk = kwargs ? PyDict_Size(kwargs) : 0;
+ if (unlikely(Py_EnterRecursiveCall(" while calling a Python object"))) {
+ return NULL;
+ }
+ if (
+ co->co_kwonlyargcount == 0 &&
+ likely(kwargs == NULL || nk == 0) &&
+ co->co_flags == (CO_OPTIMIZED | CO_NEWLOCALS | CO_NOFREE)) {
+ if (argdefs == NULL && co->co_argcount == nargs) {
+ result = __Pyx_PyFunction_FastCallNoKw(co, args, nargs, globals);
+ goto done;
+ }
+ else if (nargs == 0 && argdefs != NULL
+ && co->co_argcount == Py_SIZE(argdefs)) {
+ /* function called with no arguments, but all parameters have
+ a default value: use default values as arguments .*/
+ args = &PyTuple_GET_ITEM(argdefs, 0);
+ result =__Pyx_PyFunction_FastCallNoKw(co, args, Py_SIZE(argdefs), globals);
+ goto done;
+ }
+ }
+ if (kwargs != NULL) {
+ Py_ssize_t pos, i;
+ kwtuple = PyTuple_New(2 * nk);
+ if (kwtuple == NULL) {
+ result = NULL;
+ goto done;
+ }
+ k = &PyTuple_GET_ITEM(kwtuple, 0);
+ pos = i = 0;
+ while (PyDict_Next(kwargs, &pos, &k[i], &k[i+1])) {
+ Py_INCREF(k[i]);
+ Py_INCREF(k[i+1]);
+ i += 2;
+ }
+ nk = i / 2;
+ }
+ else {
+ kwtuple = NULL;
+ k = NULL;
+ }
+ closure = PyFunction_GET_CLOSURE(func);
+ kwdefs = PyFunction_GET_KW_DEFAULTS(func);
+ if (argdefs != NULL) {
+ d = &PyTuple_GET_ITEM(argdefs, 0);
+ nd = Py_SIZE(argdefs);
+ }
+ else {
+ d = NULL;
+ nd = 0;
+ }
+ result = PyEval_EvalCodeEx((PyObject*)co, globals, (PyObject *)NULL,
+ args, (int)nargs,
+ k, (int)nk,
+ d, (int)nd, kwdefs, closure);
+ Py_XDECREF(kwtuple);
+done:
+ Py_LeaveRecursiveCall();
+ return result;
+}
+#endif
+
+/* PyObjectCall */
+#if CYTHON_COMPILING_IN_CPYTHON
+static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw) {
+ PyObject *result;
+ ternaryfunc call = Py_TYPE(func)->tp_call;
+ if (unlikely(!call))
+ return PyObject_Call(func, arg, kw);
+ if (unlikely(Py_EnterRecursiveCall(" while calling a Python object")))
+ return NULL;
+ result = (*call)(func, arg, kw);
+ Py_LeaveRecursiveCall();
+ if (unlikely(!result) && unlikely(!PyErr_Occurred())) {
+ PyErr_SetString(
+ PyExc_SystemError,
+ "NULL result without error in PyObject_Call");
+ }
+ return result;
+}
+#endif
+
+/* PyObjectCallMethO */
+#if CYTHON_COMPILING_IN_CPYTHON
+static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg) {
+ PyObject *self, *result;
+ PyCFunction cfunc;
+ cfunc = __Pyx_CyOrPyCFunction_GET_FUNCTION(func);
+ self = __Pyx_CyOrPyCFunction_GET_SELF(func);
+ if (unlikely(Py_EnterRecursiveCall(" while calling a Python object")))
+ return NULL;
+ result = cfunc(self, arg);
+ Py_LeaveRecursiveCall();
+ if (unlikely(!result) && unlikely(!PyErr_Occurred())) {
+ PyErr_SetString(
+ PyExc_SystemError,
+ "NULL result without error in PyObject_Call");
+ }
+ return result;
+}
+#endif
+
+/* PyObjectFastCall */
+#if PY_VERSION_HEX < 0x03090000 || CYTHON_COMPILING_IN_LIMITED_API
+static PyObject* __Pyx_PyObject_FastCall_fallback(PyObject *func, PyObject * const*args, size_t nargs, PyObject *kwargs) {
+ PyObject *argstuple;
+ PyObject *result = 0;
+ size_t i;
+ argstuple = PyTuple_New((Py_ssize_t)nargs);
+ if (unlikely(!argstuple)) return NULL;
+ for (i = 0; i < nargs; i++) {
+ Py_INCREF(args[i]);
+ if (__Pyx_PyTuple_SET_ITEM(argstuple, (Py_ssize_t)i, args[i]) != (0)) goto bad;
+ }
+ result = __Pyx_PyObject_Call(func, argstuple, kwargs);
+ bad:
+ Py_DECREF(argstuple);
+ return result;
+}
+#endif
+#if CYTHON_VECTORCALL && !CYTHON_COMPILING_IN_LIMITED_API
+ #if PY_VERSION_HEX < 0x03090000
+ #define __Pyx_PyVectorcall_Function(callable) _PyVectorcall_Function(callable)
+ #elif CYTHON_COMPILING_IN_CPYTHON
+static CYTHON_INLINE vectorcallfunc __Pyx_PyVectorcall_Function(PyObject *callable) {
+ PyTypeObject *tp = Py_TYPE(callable);
+ #if defined(__Pyx_CyFunction_USED)
+ if (__Pyx_CyFunction_CheckExact(callable)) {
+ return __Pyx_CyFunction_func_vectorcall(callable);
+ }
+ #endif
+ if (!PyType_HasFeature(tp, Py_TPFLAGS_HAVE_VECTORCALL)) {
+ return NULL;
+ }
+ assert(PyCallable_Check(callable));
+ Py_ssize_t offset = tp->tp_vectorcall_offset;
+ assert(offset > 0);
+ vectorcallfunc ptr;
+ memcpy(&ptr, (char *) callable + offset, sizeof(ptr));
+ return ptr;
+}
+ #else
+ #define __Pyx_PyVectorcall_Function(callable) PyVectorcall_Function(callable)
+ #endif
+#endif
+static CYTHON_INLINE PyObject* __Pyx_PyObject_FastCallDict(PyObject *func, PyObject *const *args, size_t _nargs, PyObject *kwargs) {
+ Py_ssize_t nargs = __Pyx_PyVectorcall_NARGS(_nargs);
+#if CYTHON_COMPILING_IN_CPYTHON
+ if (nargs == 0 && kwargs == NULL) {
+ if (__Pyx_CyOrPyCFunction_Check(func) && likely( __Pyx_CyOrPyCFunction_GET_FLAGS(func) & METH_NOARGS))
+ return __Pyx_PyObject_CallMethO(func, NULL);
+ }
+ else if (nargs == 1 && kwargs == NULL) {
+ if (__Pyx_CyOrPyCFunction_Check(func) && likely( __Pyx_CyOrPyCFunction_GET_FLAGS(func) & METH_O))
+ return __Pyx_PyObject_CallMethO(func, args[0]);
+ }
+#endif
+ #if PY_VERSION_HEX < 0x030800B1
+ #if CYTHON_FAST_PYCCALL
+ if (PyCFunction_Check(func)) {
+ if (kwargs) {
+ return _PyCFunction_FastCallDict(func, args, nargs, kwargs);
+ } else {
+ return _PyCFunction_FastCallKeywords(func, args, nargs, NULL);
+ }
+ }
+ if (!kwargs && __Pyx_IS_TYPE(func, &PyMethodDescr_Type)) {
+ return _PyMethodDescr_FastCallKeywords(func, args, nargs, NULL);
+ }
+ #endif
+ #if CYTHON_FAST_PYCALL
+ if (PyFunction_Check(func)) {
+ return __Pyx_PyFunction_FastCallDict(func, args, nargs, kwargs);
+ }
+ #endif
+ #endif
+ if (kwargs == NULL) {
+ #if CYTHON_VECTORCALL && !CYTHON_COMPILING_IN_LIMITED_API
+ vectorcallfunc f = __Pyx_PyVectorcall_Function(func);
+ if (f) {
+ return f(func, args, _nargs, NULL);
+ }
+ #elif defined(__Pyx_CyFunction_USED) && CYTHON_BACKPORT_VECTORCALL
+ if (__Pyx_CyFunction_CheckExact(func)) {
+ __pyx_vectorcallfunc f = __Pyx_CyFunction_func_vectorcall(func);
+ if (f) return f(func, args, _nargs, NULL);
+ }
+ #elif CYTHON_COMPILING_IN_LIMITED_API && CYTHON_VECTORCALL
+ return PyObject_Vectorcall(func, args, _nargs, NULL);
+ #endif
+ }
+ if (nargs == 0) {
+ return __Pyx_PyObject_Call(func, __pyx_mstate_global->__pyx_empty_tuple, kwargs);
+ }
+ #if PY_VERSION_HEX >= 0x03090000 && !CYTHON_COMPILING_IN_LIMITED_API
+ return PyObject_VectorcallDict(func, args, (size_t)nargs, kwargs);
+ #else
+ return __Pyx_PyObject_FastCall_fallback(func, args, (size_t)nargs, kwargs);
+ #endif
+}
+
+/* PyLongCompare */
+static CYTHON_INLINE int __Pyx_PyLong_BoolEqObjC(PyObject *op1, PyObject *op2, long intval, long inplace) {
+ CYTHON_MAYBE_UNUSED_VAR(intval);
+ CYTHON_UNUSED_VAR(inplace);
+ if (op1 == op2) {
+ return 1;
+ }
+ #if CYTHON_USE_PYLONG_INTERNALS
+ if (likely(PyLong_CheckExact(op1))) {
+ int unequal;
+ unsigned long uintval;
+ Py_ssize_t size = __Pyx_PyLong_DigitCount(op1);
+ const digit* digits = __Pyx_PyLong_Digits(op1);
+ if (intval == 0) {
+ return (__Pyx_PyLong_IsZero(op1) == 1);
+ } else if (intval < 0) {
+ if (__Pyx_PyLong_IsNonNeg(op1))
+ return 0;
+ intval = -intval;
+ } else {
+ if (__Pyx_PyLong_IsNeg(op1))
+ return 0;
+ }
+ uintval = (unsigned long) intval;
+#if PyLong_SHIFT * 4 < SIZEOF_LONG*8
+ if (uintval >> (PyLong_SHIFT * 4)) {
+ unequal = (size != 5) || (digits[0] != (uintval & (unsigned long) PyLong_MASK))
+ | (digits[1] != ((uintval >> (1 * PyLong_SHIFT)) & (unsigned long) PyLong_MASK)) | (digits[2] != ((uintval >> (2 * PyLong_SHIFT)) & (unsigned long) PyLong_MASK)) | (digits[3] != ((uintval >> (3 * PyLong_SHIFT)) & (unsigned long) PyLong_MASK)) | (digits[4] != ((uintval >> (4 * PyLong_SHIFT)) & (unsigned long) PyLong_MASK));
+ } else
+#endif
+#if PyLong_SHIFT * 3 < SIZEOF_LONG*8
+ if (uintval >> (PyLong_SHIFT * 3)) {
+ unequal = (size != 4) || (digits[0] != (uintval & (unsigned long) PyLong_MASK))
+ | (digits[1] != ((uintval >> (1 * PyLong_SHIFT)) & (unsigned long) PyLong_MASK)) | (digits[2] != ((uintval >> (2 * PyLong_SHIFT)) & (unsigned long) PyLong_MASK)) | (digits[3] != ((uintval >> (3 * PyLong_SHIFT)) & (unsigned long) PyLong_MASK));
+ } else
+#endif
+#if PyLong_SHIFT * 2 < SIZEOF_LONG*8
+ if (uintval >> (PyLong_SHIFT * 2)) {
+ unequal = (size != 3) || (digits[0] != (uintval & (unsigned long) PyLong_MASK))
+ | (digits[1] != ((uintval >> (1 * PyLong_SHIFT)) & (unsigned long) PyLong_MASK)) | (digits[2] != ((uintval >> (2 * PyLong_SHIFT)) & (unsigned long) PyLong_MASK));
+ } else
+#endif
+#if PyLong_SHIFT * 1 < SIZEOF_LONG*8
+ if (uintval >> (PyLong_SHIFT * 1)) {
+ unequal = (size != 2) || (digits[0] != (uintval & (unsigned long) PyLong_MASK))
+ | (digits[1] != ((uintval >> (1 * PyLong_SHIFT)) & (unsigned long) PyLong_MASK));
+ } else
+#endif
+ unequal = (size != 1) || (((unsigned long) digits[0]) != (uintval & (unsigned long) PyLong_MASK));
+ return (unequal == 0);
+ }
+ #endif
+ if (PyFloat_CheckExact(op1)) {
+ const long b = intval;
+ double a = __Pyx_PyFloat_AS_DOUBLE(op1);
+ return ((double)a == (double)b);
+ }
+ return __Pyx_PyObject_IsTrueAndDecref(
+ PyObject_RichCompare(op1, op2, Py_EQ));
+}
+
+/* RaiseTooManyValuesToUnpack */
+static CYTHON_INLINE void __Pyx_RaiseTooManyValuesError(Py_ssize_t expected) {
+ PyErr_Format(PyExc_ValueError,
+ "too many values to unpack (expected %" CYTHON_FORMAT_SSIZE_T "d)", expected);
+}
+
+/* RaiseNeedMoreValuesToUnpack */
+static CYTHON_INLINE void __Pyx_RaiseNeedMoreValuesError(Py_ssize_t index) {
+ PyErr_Format(PyExc_ValueError,
+ "need more than %" CYTHON_FORMAT_SSIZE_T "d value%.1s to unpack",
+ index, (index == 1) ? "" : "s");
+}
+
+/* IterFinish */
+static CYTHON_INLINE int __Pyx_IterFinish(void) {
+ PyObject* exc_type;
+ __Pyx_PyThreadState_declare
+ __Pyx_PyThreadState_assign
+ exc_type = __Pyx_PyErr_CurrentExceptionType();
+ if (unlikely(exc_type)) {
+ if (unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration)))
+ return -1;
+ __Pyx_PyErr_Clear();
+ return 0;
+ }
+ return 0;
+}
+
+/* UnpackItemEndCheck */
+static int __Pyx_IternextUnpackEndCheck(PyObject *retval, Py_ssize_t expected) {
+ if (unlikely(retval)) {
+ Py_DECREF(retval);
+ __Pyx_RaiseTooManyValuesError(expected);
+ return -1;
+ }
+ return __Pyx_IterFinish();
+}
+
+/* GetItemInt */
+static PyObject *__Pyx_GetItemInt_Generic(PyObject *o, PyObject* j) {
+ PyObject *r;
+ if (unlikely(!j)) return NULL;
+ r = PyObject_GetItem(o, j);
+ Py_DECREF(j);
+ return r;
+}
+static CYTHON_INLINE PyObject *__Pyx_GetItemInt_List_Fast(PyObject *o, Py_ssize_t i,
+ CYTHON_NCP_UNUSED int wraparound,
+ CYTHON_NCP_UNUSED int boundscheck) {
+#if CYTHON_ASSUME_SAFE_MACROS && CYTHON_ASSUME_SAFE_SIZE && !CYTHON_AVOID_BORROWED_REFS && !CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS
+ Py_ssize_t wrapped_i = i;
+ if (wraparound & unlikely(i < 0)) {
+ wrapped_i += PyList_GET_SIZE(o);
+ }
+ if ((!boundscheck) || likely(__Pyx_is_valid_index(wrapped_i, PyList_GET_SIZE(o)))) {
+ PyObject *r = PyList_GET_ITEM(o, wrapped_i);
+ Py_INCREF(r);
+ return r;
+ }
+ return __Pyx_GetItemInt_Generic(o, PyLong_FromSsize_t(i));
+#else
+ return PySequence_GetItem(o, i);
+#endif
+}
+static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Tuple_Fast(PyObject *o, Py_ssize_t i,
+ CYTHON_NCP_UNUSED int wraparound,
+ CYTHON_NCP_UNUSED int boundscheck) {
+#if CYTHON_ASSUME_SAFE_MACROS && CYTHON_ASSUME_SAFE_SIZE && !CYTHON_AVOID_BORROWED_REFS
+ Py_ssize_t wrapped_i = i;
+ if (wraparound & unlikely(i < 0)) {
+ wrapped_i += PyTuple_GET_SIZE(o);
+ }
+ if ((!boundscheck) || likely(__Pyx_is_valid_index(wrapped_i, PyTuple_GET_SIZE(o)))) {
+ PyObject *r = PyTuple_GET_ITEM(o, wrapped_i);
+ Py_INCREF(r);
+ return r;
+ }
+ return __Pyx_GetItemInt_Generic(o, PyLong_FromSsize_t(i));
+#else
+ return PySequence_GetItem(o, i);
+#endif
+}
+static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Fast(PyObject *o, Py_ssize_t i, int is_list,
+ CYTHON_NCP_UNUSED int wraparound,
+ CYTHON_NCP_UNUSED int boundscheck) {
+#if CYTHON_ASSUME_SAFE_MACROS && CYTHON_ASSUME_SAFE_SIZE && !CYTHON_AVOID_BORROWED_REFS && CYTHON_USE_TYPE_SLOTS
+ if (is_list || PyList_CheckExact(o)) {
+ Py_ssize_t n = ((!wraparound) | likely(i >= 0)) ? i : i + PyList_GET_SIZE(o);
+ if ((!boundscheck) || (likely(__Pyx_is_valid_index(n, PyList_GET_SIZE(o))))) {
+ return __Pyx_PyList_GetItemRef(o, n);
+ }
+ }
+ else if (PyTuple_CheckExact(o)) {
+ Py_ssize_t n = ((!wraparound) | likely(i >= 0)) ? i : i + PyTuple_GET_SIZE(o);
+ if ((!boundscheck) || likely(__Pyx_is_valid_index(n, PyTuple_GET_SIZE(o)))) {
+ PyObject *r = PyTuple_GET_ITEM(o, n);
+ Py_INCREF(r);
+ return r;
+ }
+ } else {
+ PyMappingMethods *mm = Py_TYPE(o)->tp_as_mapping;
+ PySequenceMethods *sm = Py_TYPE(o)->tp_as_sequence;
+ if (mm && mm->mp_subscript) {
+ PyObject *r, *key = PyLong_FromSsize_t(i);
+ if (unlikely(!key)) return NULL;
+ r = mm->mp_subscript(o, key);
+ Py_DECREF(key);
+ return r;
+ }
+ if (likely(sm && sm->sq_item)) {
+ if (wraparound && unlikely(i < 0) && likely(sm->sq_length)) {
+ Py_ssize_t l = sm->sq_length(o);
+ if (likely(l >= 0)) {
+ i += l;
+ } else {
+ if (!PyErr_ExceptionMatches(PyExc_OverflowError))
+ return NULL;
+ PyErr_Clear();
+ }
+ }
+ return sm->sq_item(o, i);
+ }
+ }
+#else
+ if (is_list || !PyMapping_Check(o)) {
+ return PySequence_GetItem(o, i);
+ }
+#endif
+ return __Pyx_GetItemInt_Generic(o, PyLong_FromSsize_t(i));
+}
+
+/* PyDictVersioning */
+#if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_TYPE_SLOTS
+static CYTHON_INLINE PY_UINT64_T __Pyx_get_tp_dict_version(PyObject *obj) {
+ PyObject *dict = Py_TYPE(obj)->tp_dict;
+ return likely(dict) ? __PYX_GET_DICT_VERSION(dict) : 0;
+}
+static CYTHON_INLINE PY_UINT64_T __Pyx_get_object_dict_version(PyObject *obj) {
+ PyObject **dictptr = NULL;
+ Py_ssize_t offset = Py_TYPE(obj)->tp_dictoffset;
+ if (offset) {
+#if CYTHON_COMPILING_IN_CPYTHON
+ dictptr = (likely(offset > 0)) ? (PyObject **) ((char *)obj + offset) : _PyObject_GetDictPtr(obj);
+#else
+ dictptr = _PyObject_GetDictPtr(obj);
+#endif
+ }
+ return (dictptr && *dictptr) ? __PYX_GET_DICT_VERSION(*dictptr) : 0;
+}
+static CYTHON_INLINE int __Pyx_object_dict_version_matches(PyObject* obj, PY_UINT64_T tp_dict_version, PY_UINT64_T obj_dict_version) {
+ PyObject *dict = Py_TYPE(obj)->tp_dict;
+ if (unlikely(!dict) || unlikely(tp_dict_version != __PYX_GET_DICT_VERSION(dict)))
+ return 0;
+ return obj_dict_version == __Pyx_get_object_dict_version(obj);
+}
+#endif
+
+/* GetModuleGlobalName */
+#if CYTHON_USE_DICT_VERSIONS
+static PyObject *__Pyx__GetModuleGlobalName(PyObject *name, PY_UINT64_T *dict_version, PyObject **dict_cached_value)
+#else
+static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name)
+#endif
+{
+ PyObject *result;
+#if CYTHON_COMPILING_IN_LIMITED_API
+ if (unlikely(!__pyx_m)) {
+ if (!PyErr_Occurred())
+ PyErr_SetNone(PyExc_NameError);
+ return NULL;
+ }
+ result = PyObject_GetAttr(__pyx_m, name);
+ if (likely(result)) {
+ return result;
+ }
+ PyErr_Clear();
+#elif CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS
+ if (unlikely(__Pyx_PyDict_GetItemRef(__pyx_mstate_global->__pyx_d, name, &result) == -1)) PyErr_Clear();
+ __PYX_UPDATE_DICT_CACHE(__pyx_mstate_global->__pyx_d, result, *dict_cached_value, *dict_version)
+ if (likely(result)) {
+ return result;
+ }
+#else
+ result = _PyDict_GetItem_KnownHash(__pyx_mstate_global->__pyx_d, name, ((PyASCIIObject *) name)->hash);
+ __PYX_UPDATE_DICT_CACHE(__pyx_mstate_global->__pyx_d, result, *dict_cached_value, *dict_version)
+ if (likely(result)) {
+ return __Pyx_NewRef(result);
+ }
+ PyErr_Clear();
+#endif
+ return __Pyx_GetBuiltinName(name);
+}
+
+/* TupleAndListFromArray */
+#if !CYTHON_COMPILING_IN_CPYTHON && CYTHON_METH_FASTCALL
+static CYTHON_INLINE PyObject *
+__Pyx_PyTuple_FromArray(PyObject *const *src, Py_ssize_t n)
+{
+ PyObject *res;
+ Py_ssize_t i;
+ if (n <= 0) {
+ return __Pyx_NewRef(__pyx_mstate_global->__pyx_empty_tuple);
+ }
+ res = PyTuple_New(n);
+ if (unlikely(res == NULL)) return NULL;
+ for (i = 0; i < n; i++) {
+ if (unlikely(__Pyx_PyTuple_SET_ITEM(res, i, src[i]) < 0)) {
+ Py_DECREF(res);
+ return NULL;
+ }
+ Py_INCREF(src[i]);
+ }
+ return res;
+}
+#elif CYTHON_COMPILING_IN_CPYTHON
+static CYTHON_INLINE void __Pyx_copy_object_array(PyObject *const *CYTHON_RESTRICT src, PyObject** CYTHON_RESTRICT dest, Py_ssize_t length) {
+ PyObject *v;
+ Py_ssize_t i;
+ for (i = 0; i < length; i++) {
+ v = dest[i] = src[i];
+ Py_INCREF(v);
+ }
+}
+static CYTHON_INLINE PyObject *
+__Pyx_PyTuple_FromArray(PyObject *const *src, Py_ssize_t n)
+{
+ PyObject *res;
+ if (n <= 0) {
+ return __Pyx_NewRef(__pyx_mstate_global->__pyx_empty_tuple);
+ }
+ res = PyTuple_New(n);
+ if (unlikely(res == NULL)) return NULL;
+ __Pyx_copy_object_array(src, ((PyTupleObject*)res)->ob_item, n);
+ return res;
+}
+static CYTHON_INLINE PyObject *
+__Pyx_PyList_FromArray(PyObject *const *src, Py_ssize_t n)
+{
+ PyObject *res;
+ if (n <= 0) {
+ return PyList_New(0);
+ }
+ res = PyList_New(n);
+ if (unlikely(res == NULL)) return NULL;
+ __Pyx_copy_object_array(src, ((PyListObject*)res)->ob_item, n);
+ return res;
+}
+#endif
+
+/* BytesEquals */
+static CYTHON_INLINE int __Pyx_PyBytes_Equals(PyObject* s1, PyObject* s2, int equals) {
+#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_GRAAL ||\
+ !(CYTHON_ASSUME_SAFE_SIZE && CYTHON_ASSUME_SAFE_MACROS)
+ return PyObject_RichCompareBool(s1, s2, equals);
+#else
+ if (s1 == s2) {
+ return (equals == Py_EQ);
+ } else if (PyBytes_CheckExact(s1) & PyBytes_CheckExact(s2)) {
+ const char *ps1, *ps2;
+ Py_ssize_t length = PyBytes_GET_SIZE(s1);
+ if (length != PyBytes_GET_SIZE(s2))
+ return (equals == Py_NE);
+ ps1 = PyBytes_AS_STRING(s1);
+ ps2 = PyBytes_AS_STRING(s2);
+ if (ps1[0] != ps2[0]) {
+ return (equals == Py_NE);
+ } else if (length == 1) {
+ return (equals == Py_EQ);
+ } else {
+ int result;
+#if CYTHON_USE_UNICODE_INTERNALS && (PY_VERSION_HEX < 0x030B0000)
+ Py_hash_t hash1, hash2;
+ hash1 = ((PyBytesObject*)s1)->ob_shash;
+ hash2 = ((PyBytesObject*)s2)->ob_shash;
+ if (hash1 != hash2 && hash1 != -1 && hash2 != -1) {
+ return (equals == Py_NE);
+ }
+#endif
+ result = memcmp(ps1, ps2, (size_t)length);
+ return (equals == Py_EQ) ? (result == 0) : (result != 0);
+ }
+ } else if ((s1 == Py_None) & PyBytes_CheckExact(s2)) {
+ return (equals == Py_NE);
+ } else if ((s2 == Py_None) & PyBytes_CheckExact(s1)) {
+ return (equals == Py_NE);
+ } else {
+ int result;
+ PyObject* py_result = PyObject_RichCompare(s1, s2, equals);
+ if (!py_result)
+ return -1;
+ result = __Pyx_PyObject_IsTrue(py_result);
+ Py_DECREF(py_result);
+ return result;
+ }
+#endif
+}
+
+/* UnicodeEquals */
+static CYTHON_INLINE int __Pyx_PyUnicode_Equals(PyObject* s1, PyObject* s2, int equals) {
+#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_GRAAL
+ return PyObject_RichCompareBool(s1, s2, equals);
+#else
+ int s1_is_unicode, s2_is_unicode;
+ if (s1 == s2) {
+ goto return_eq;
+ }
+ s1_is_unicode = PyUnicode_CheckExact(s1);
+ s2_is_unicode = PyUnicode_CheckExact(s2);
+ if (s1_is_unicode & s2_is_unicode) {
+ Py_ssize_t length, length2;
+ int kind;
+ void *data1, *data2;
+ #if !CYTHON_COMPILING_IN_LIMITED_API
+ if (unlikely(__Pyx_PyUnicode_READY(s1) < 0) || unlikely(__Pyx_PyUnicode_READY(s2) < 0))
+ return -1;
+ #endif
+ length = __Pyx_PyUnicode_GET_LENGTH(s1);
+ #if !CYTHON_ASSUME_SAFE_SIZE
+ if (unlikely(length < 0)) return -1;
+ #endif
+ length2 = __Pyx_PyUnicode_GET_LENGTH(s2);
+ #if !CYTHON_ASSUME_SAFE_SIZE
+ if (unlikely(length2 < 0)) return -1;
+ #endif
+ if (length != length2) {
+ goto return_ne;
+ }
+#if CYTHON_USE_UNICODE_INTERNALS
+ {
+ Py_hash_t hash1, hash2;
+ hash1 = ((PyASCIIObject*)s1)->hash;
+ hash2 = ((PyASCIIObject*)s2)->hash;
+ if (hash1 != hash2 && hash1 != -1 && hash2 != -1) {
+ goto return_ne;
+ }
+ }
+#endif
+ kind = __Pyx_PyUnicode_KIND(s1);
+ if (kind != __Pyx_PyUnicode_KIND(s2)) {
+ goto return_ne;
+ }
+ data1 = __Pyx_PyUnicode_DATA(s1);
+ data2 = __Pyx_PyUnicode_DATA(s2);
+ if (__Pyx_PyUnicode_READ(kind, data1, 0) != __Pyx_PyUnicode_READ(kind, data2, 0)) {
+ goto return_ne;
+ } else if (length == 1) {
+ goto return_eq;
+ } else {
+ int result = memcmp(data1, data2, (size_t)(length * kind));
+ return (equals == Py_EQ) ? (result == 0) : (result != 0);
+ }
+ } else if ((s1 == Py_None) & s2_is_unicode) {
+ goto return_ne;
+ } else if ((s2 == Py_None) & s1_is_unicode) {
+ goto return_ne;
+ } else {
+ int result;
+ PyObject* py_result = PyObject_RichCompare(s1, s2, equals);
+ if (!py_result)
+ return -1;
+ result = __Pyx_PyObject_IsTrue(py_result);
+ Py_DECREF(py_result);
+ return result;
+ }
+return_eq:
+ return (equals == Py_EQ);
+return_ne:
+ return (equals == Py_NE);
+#endif
+}
+
+/* fastcall */
+#if CYTHON_METH_FASTCALL
+static CYTHON_INLINE PyObject * __Pyx_GetKwValue_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues, PyObject *s)
+{
+ Py_ssize_t i, n = __Pyx_PyTuple_GET_SIZE(kwnames);
+ #if !CYTHON_ASSUME_SAFE_SIZE
+ if (unlikely(n == -1)) return NULL;
+ #endif
+ for (i = 0; i < n; i++)
+ {
+ PyObject *namei = __Pyx_PyTuple_GET_ITEM(kwnames, i);
+ #if !CYTHON_ASSUME_SAFE_MACROS
+ if (unlikely(!namei)) return NULL;
+ #endif
+ if (s == namei) return kwvalues[i];
+ }
+ for (i = 0; i < n; i++)
+ {
+ PyObject *namei = __Pyx_PyTuple_GET_ITEM(kwnames, i);
+ #if !CYTHON_ASSUME_SAFE_MACROS
+ if (unlikely(!namei)) return NULL;
+ #endif
+ int eq = __Pyx_PyUnicode_Equals(s, namei, Py_EQ);
+ if (unlikely(eq != 0)) {
+ if (unlikely(eq < 0)) return NULL;
+ return kwvalues[i];
+ }
+ }
+ return NULL;
+}
+#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030d0000 || CYTHON_COMPILING_IN_LIMITED_API
+CYTHON_UNUSED static PyObject *__Pyx_KwargsAsDict_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues) {
+ Py_ssize_t i, nkwargs;
+ PyObject *dict;
+#if !CYTHON_ASSUME_SAFE_SIZE
+ nkwargs = PyTuple_Size(kwnames);
+ if (unlikely(nkwargs < 0)) return NULL;
+#else
+ nkwargs = PyTuple_GET_SIZE(kwnames);
+#endif
+ dict = PyDict_New();
+ if (unlikely(!dict))
+ return NULL;
+ for (i=0; itype, *target->method_name);
+ if (unlikely(!method))
+ return -1;
+ result = method;
+#if CYTHON_COMPILING_IN_CPYTHON
+ if (likely(__Pyx_TypeCheck(method, &PyMethodDescr_Type)))
+ {
+ PyMethodDescrObject *descr = (PyMethodDescrObject*) method;
+ target->func = descr->d_method->ml_meth;
+ target->flag = descr->d_method->ml_flags & ~(METH_CLASS | METH_STATIC | METH_COEXIST | METH_STACKLESS);
+ } else
+#endif
+#if CYTHON_COMPILING_IN_PYPY
+#else
+ if (PyCFunction_Check(method))
+#endif
+ {
+ PyObject *self;
+ int self_found;
+#if CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_PYPY
+ self = PyObject_GetAttrString(method, "__self__");
+ if (!self) {
+ PyErr_Clear();
+ }
+#else
+ self = PyCFunction_GET_SELF(method);
+#endif
+ self_found = (self && self != Py_None);
+#if CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_PYPY
+ Py_XDECREF(self);
+#endif
+ if (self_found) {
+ PyObject *unbound_method = PyCFunction_New(&__Pyx_UnboundCMethod_Def, method);
+ if (unlikely(!unbound_method)) return -1;
+ Py_DECREF(method);
+ result = unbound_method;
+ }
+ }
+#if !CYTHON_COMPILING_IN_CPYTHON_FREETHREADING
+ if (unlikely(target->method)) {
+ Py_DECREF(result);
+ } else
+#endif
+ target->method = result;
+ return 0;
+}
+
+/* CallUnboundCMethod2 */
+#if CYTHON_COMPILING_IN_CPYTHON
+static CYTHON_INLINE PyObject *__Pyx_CallUnboundCMethod2(__Pyx_CachedCFunction *cfunc, PyObject *self, PyObject *arg1, PyObject *arg2) {
+ int was_initialized = __Pyx_CachedCFunction_GetAndSetInitializing(cfunc);
+ if (likely(was_initialized == 2 && cfunc->func)) {
+ PyObject *args[2] = {arg1, arg2};
+ if (cfunc->flag == METH_FASTCALL) {
+ return __Pyx_CallCFunctionFast(cfunc, self, args, 2);
+ }
+ if (cfunc->flag == (METH_FASTCALL | METH_KEYWORDS))
+ return __Pyx_CallCFunctionFastWithKeywords(cfunc, self, args, 2, NULL);
+ }
+#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING
+ else if (unlikely(was_initialized == 1)) {
+ __Pyx_CachedCFunction tmp_cfunc = {
+#ifndef __cplusplus
+ 0
+#endif
+ };
+ tmp_cfunc.type = cfunc->type;
+ tmp_cfunc.method_name = cfunc->method_name;
+ return __Pyx__CallUnboundCMethod2(&tmp_cfunc, self, arg1, arg2);
+ }
+#endif
+ PyObject *result = __Pyx__CallUnboundCMethod2(cfunc, self, arg1, arg2);
+ __Pyx_CachedCFunction_SetFinishedInitializing(cfunc);
+ return result;
+}
+#endif
+static PyObject* __Pyx__CallUnboundCMethod2(__Pyx_CachedCFunction* cfunc, PyObject* self, PyObject* arg1, PyObject* arg2){
+ if (unlikely(!cfunc->func && !cfunc->method) && unlikely(__Pyx_TryUnpackUnboundCMethod(cfunc) < 0)) return NULL;
+#if CYTHON_COMPILING_IN_CPYTHON
+ if (cfunc->func && (cfunc->flag & METH_VARARGS)) {
+ PyObject *result = NULL;
+ PyObject *args = PyTuple_New(2);
+ if (unlikely(!args)) return NULL;
+ Py_INCREF(arg1);
+ PyTuple_SET_ITEM(args, 0, arg1);
+ Py_INCREF(arg2);
+ PyTuple_SET_ITEM(args, 1, arg2);
+ if (cfunc->flag & METH_KEYWORDS)
+ result = __Pyx_CallCFunctionWithKeywords(cfunc, self, args, NULL);
+ else
+ result = __Pyx_CallCFunction(cfunc, self, args);
+ Py_DECREF(args);
+ return result;
+ }
+#endif
+ {
+ PyObject *args[4] = {NULL, self, arg1, arg2};
+ return __Pyx_PyObject_FastCall(cfunc->method, args+1, 3 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET);
+ }
+}
+
+/* ParseKeywords */
+static int __Pyx_ValidateDuplicatePosArgs(
+ PyObject *kwds,
+ PyObject ** const argnames[],
+ PyObject ** const *first_kw_arg,
+ const char* function_name)
+{
+ PyObject ** const *name = argnames;
+ while (name != first_kw_arg) {
+ PyObject *key = **name;
+ int found = PyDict_Contains(kwds, key);
+ if (unlikely(found)) {
+ if (found == 1) __Pyx_RaiseDoubleKeywordsError(function_name, key);
+ goto bad;
+ }
+ name++;
+ }
+ return 0;
+bad:
+ return -1;
+}
+#if CYTHON_USE_UNICODE_INTERNALS
+static CYTHON_INLINE int __Pyx_UnicodeKeywordsEqual(PyObject *s1, PyObject *s2) {
+ int kind;
+ Py_ssize_t len = PyUnicode_GET_LENGTH(s1);
+ if (len != PyUnicode_GET_LENGTH(s2)) return 0;
+ kind = PyUnicode_KIND(s1);
+ if (kind != PyUnicode_KIND(s2)) return 0;
+ const void *data1 = PyUnicode_DATA(s1);
+ const void *data2 = PyUnicode_DATA(s2);
+ return (memcmp(data1, data2, (size_t) len * (size_t) kind) == 0);
+}
+#endif
+static int __Pyx_MatchKeywordArg_str(
+ PyObject *key,
+ PyObject ** const argnames[],
+ PyObject ** const *first_kw_arg,
+ size_t *index_found,
+ const char *function_name)
+{
+ PyObject ** const *name;
+ #if CYTHON_USE_UNICODE_INTERNALS
+ Py_hash_t key_hash = ((PyASCIIObject*)key)->hash;
+ if (unlikely(key_hash == -1)) {
+ key_hash = PyObject_Hash(key);
+ if (unlikely(key_hash == -1))
+ goto bad;
+ }
+ #endif
+ name = first_kw_arg;
+ while (*name) {
+ PyObject *name_str = **name;
+ #if CYTHON_USE_UNICODE_INTERNALS
+ if (key_hash == ((PyASCIIObject*)name_str)->hash && __Pyx_UnicodeKeywordsEqual(name_str, key)) {
+ *index_found = (size_t) (name - argnames);
+ return 1;
+ }
+ #else
+ #if CYTHON_ASSUME_SAFE_SIZE
+ if (PyUnicode_GET_LENGTH(name_str) == PyUnicode_GET_LENGTH(key))
+ #endif
+ {
+ int cmp = PyUnicode_Compare(name_str, key);
+ if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad;
+ if (cmp == 0) {
+ *index_found = (size_t) (name - argnames);
+ return 1;
+ }
+ }
+ #endif
+ name++;
+ }
+ name = argnames;
+ while (name != first_kw_arg) {
+ PyObject *name_str = **name;
+ #if CYTHON_USE_UNICODE_INTERNALS
+ if (unlikely(key_hash == ((PyASCIIObject*)name_str)->hash)) {
+ if (__Pyx_UnicodeKeywordsEqual(name_str, key))
+ goto arg_passed_twice;
+ }
+ #else
+ #if CYTHON_ASSUME_SAFE_SIZE
+ if (PyUnicode_GET_LENGTH(name_str) == PyUnicode_GET_LENGTH(key))
+ #endif
+ {
+ if (unlikely(name_str == key)) goto arg_passed_twice;
+ int cmp = PyUnicode_Compare(name_str, key);
+ if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad;
+ if (cmp == 0) goto arg_passed_twice;
+ }
+ #endif
+ name++;
+ }
+ return 0;
+arg_passed_twice:
+ __Pyx_RaiseDoubleKeywordsError(function_name, key);
+ goto bad;
+bad:
+ return -1;
+}
+static int __Pyx_MatchKeywordArg_nostr(
+ PyObject *key,
+ PyObject ** const argnames[],
+ PyObject ** const *first_kw_arg,
+ size_t *index_found,
+ const char *function_name)
+{
+ PyObject ** const *name;
+ if (unlikely(!PyUnicode_Check(key))) goto invalid_keyword_type;
+ name = first_kw_arg;
+ while (*name) {
+ int cmp = PyObject_RichCompareBool(**name, key, Py_EQ);
+ if (cmp == 1) {
+ *index_found = (size_t) (name - argnames);
+ return 1;
+ }
+ if (unlikely(cmp == -1)) goto bad;
+ name++;
+ }
+ name = argnames;
+ while (name != first_kw_arg) {
+ int cmp = PyObject_RichCompareBool(**name, key, Py_EQ);
+ if (unlikely(cmp != 0)) {
+ if (cmp == 1) goto arg_passed_twice;
+ else goto bad;
+ }
+ name++;
+ }
+ return 0;
+arg_passed_twice:
+ __Pyx_RaiseDoubleKeywordsError(function_name, key);
+ goto bad;
+invalid_keyword_type:
+ PyErr_Format(PyExc_TypeError,
+ "%.200s() keywords must be strings", function_name);
+ goto bad;
+bad:
+ return -1;
+}
+static CYTHON_INLINE int __Pyx_MatchKeywordArg(
+ PyObject *key,
+ PyObject ** const argnames[],
+ PyObject ** const *first_kw_arg,
+ size_t *index_found,
+ const char *function_name)
+{
+ return likely(PyUnicode_CheckExact(key)) ?
+ __Pyx_MatchKeywordArg_str(key, argnames, first_kw_arg, index_found, function_name) :
+ __Pyx_MatchKeywordArg_nostr(key, argnames, first_kw_arg, index_found, function_name);
+}
+static void __Pyx_RejectUnknownKeyword(
+ PyObject *kwds,
+ PyObject ** const argnames[],
+ PyObject ** const *first_kw_arg,
+ const char *function_name)
+{
+ Py_ssize_t pos = 0;
+ PyObject *key = NULL;
+ __Pyx_BEGIN_CRITICAL_SECTION(kwds);
+ while (PyDict_Next(kwds, &pos, &key, NULL)) {
+ PyObject** const *name = first_kw_arg;
+ while (*name && (**name != key)) name++;
+ if (!*name) {
+ #if CYTHON_AVOID_BORROWED_REFS
+ Py_INCREF(key);
+ #endif
+ size_t index_found = 0;
+ int cmp = __Pyx_MatchKeywordArg(key, argnames, first_kw_arg, &index_found, function_name);
+ if (cmp != 1) {
+ if (cmp == 0) {
+ PyErr_Format(PyExc_TypeError,
+ "%s() got an unexpected keyword argument '%U'",
+ function_name, key);
+ }
+ #if CYTHON_AVOID_BORROWED_REFS
+ Py_DECREF(key);
+ #endif
+ break;
+ }
+ #if CYTHON_AVOID_BORROWED_REFS
+ Py_DECREF(key);
+ #endif
+ }
+ }
+ __Pyx_END_CRITICAL_SECTION();
+ assert(PyErr_Occurred());
+}
+static int __Pyx_ParseKeywordDict(
+ PyObject *kwds,
+ PyObject ** const argnames[],
+ PyObject *values[],
+ Py_ssize_t num_pos_args,
+ Py_ssize_t num_kwargs,
+ const char* function_name,
+ int ignore_unknown_kwargs)
+{
+ PyObject** const *name;
+ PyObject** const *first_kw_arg = argnames + num_pos_args;
+ Py_ssize_t extracted = 0;
+#if !CYTHON_COMPILING_IN_PYPY || defined(PyArg_ValidateKeywordArguments)
+ if (unlikely(!PyArg_ValidateKeywordArguments(kwds))) return -1;
+#endif
+ name = first_kw_arg;
+ while (*name && num_kwargs > extracted) {
+ PyObject * key = **name;
+ PyObject *value;
+ int found = 0;
+ #if __PYX_LIMITED_VERSION_HEX >= 0x030d0000
+ found = PyDict_GetItemRef(kwds, key, &value);
+ #else
+ value = PyDict_GetItemWithError(kwds, key);
+ if (value) {
+ Py_INCREF(value);
+ found = 1;
+ } else {
+ if (unlikely(PyErr_Occurred())) goto bad;
+ }
+ #endif
+ if (found) {
+ if (unlikely(found < 0)) goto bad;
+ values[name-argnames] = value;
+ extracted++;
+ }
+ name++;
+ }
+ if (num_kwargs > extracted) {
+ if (ignore_unknown_kwargs) {
+ if (unlikely(__Pyx_ValidateDuplicatePosArgs(kwds, argnames, first_kw_arg, function_name) == -1))
+ goto bad;
+ } else {
+ __Pyx_RejectUnknownKeyword(kwds, argnames, first_kw_arg, function_name);
+ goto bad;
+ }
+ }
+ return 0;
+bad:
+ return -1;
+}
+static int __Pyx_ParseKeywordDictToDict(
+ PyObject *kwds,
+ PyObject ** const argnames[],
+ PyObject *kwds2,
+ PyObject *values[],
+ Py_ssize_t num_pos_args,
+ const char* function_name)
+{
+ PyObject** const *name;
+ PyObject** const *first_kw_arg = argnames + num_pos_args;
+ Py_ssize_t len;
+#if !CYTHON_COMPILING_IN_PYPY || defined(PyArg_ValidateKeywordArguments)
+ if (unlikely(!PyArg_ValidateKeywordArguments(kwds))) return -1;
+#endif
+ if (PyDict_Update(kwds2, kwds) < 0) goto bad;
+ name = first_kw_arg;
+ while (*name) {
+ PyObject *key = **name;
+ PyObject *value;
+#if !CYTHON_COMPILING_IN_LIMITED_API && (PY_VERSION_HEX >= 0x030d00A2 || defined(PyDict_Pop))
+ int found = PyDict_Pop(kwds2, key, &value);
+ if (found) {
+ if (unlikely(found < 0)) goto bad;
+ values[name-argnames] = value;
+ }
+#elif __PYX_LIMITED_VERSION_HEX >= 0x030d0000
+ int found = PyDict_GetItemRef(kwds2, key, &value);
+ if (found) {
+ if (unlikely(found < 0)) goto bad;
+ values[name-argnames] = value;
+ if (unlikely(PyDict_DelItem(kwds2, key) < 0)) goto bad;
+ }
+#else
+ #if CYTHON_COMPILING_IN_CPYTHON
+ value = _PyDict_Pop(kwds2, key, kwds2);
+ #else
+ value = __Pyx_CallUnboundCMethod2(&__pyx_mstate_global->__pyx_umethod_PyDict_Type_pop, kwds2, key, kwds2);
+ #endif
+ if (value == kwds2) {
+ Py_DECREF(value);
+ } else {
+ if (unlikely(!value)) goto bad;
+ values[name-argnames] = value;
+ }
+#endif
+ name++;
+ }
+ len = PyDict_Size(kwds2);
+ if (len > 0) {
+ return __Pyx_ValidateDuplicatePosArgs(kwds, argnames, first_kw_arg, function_name);
+ } else if (unlikely(len == -1)) {
+ goto bad;
+ }
+ return 0;
+bad:
+ return -1;
+}
+static int __Pyx_ParseKeywordsTuple(
+ PyObject *kwds,
+ PyObject * const *kwvalues,
+ PyObject ** const argnames[],
+ PyObject *kwds2,
+ PyObject *values[],
+ Py_ssize_t num_pos_args,
+ Py_ssize_t num_kwargs,
+ const char* function_name,
+ int ignore_unknown_kwargs)
+{
+ PyObject *key = NULL;
+ PyObject** const * name;
+ PyObject** const *first_kw_arg = argnames + num_pos_args;
+ for (Py_ssize_t pos = 0; pos < num_kwargs; pos++) {
+#if CYTHON_AVOID_BORROWED_REFS
+ key = __Pyx_PySequence_ITEM(kwds, pos);
+#else
+ key = __Pyx_PyTuple_GET_ITEM(kwds, pos);
+#endif
+#if !CYTHON_ASSUME_SAFE_MACROS
+ if (unlikely(!key)) goto bad;
+#endif
+ name = first_kw_arg;
+ while (*name && (**name != key)) name++;
+ if (*name) {
+ PyObject *value = kwvalues[pos];
+ values[name-argnames] = __Pyx_NewRef(value);
+ } else {
+ size_t index_found = 0;
+ int cmp = __Pyx_MatchKeywordArg(key, argnames, first_kw_arg, &index_found, function_name);
+ if (cmp == 1) {
+ PyObject *value = kwvalues[pos];
+ values[index_found] = __Pyx_NewRef(value);
+ } else {
+ if (unlikely(cmp == -1)) goto bad;
+ if (kwds2) {
+ PyObject *value = kwvalues[pos];
+ if (unlikely(PyDict_SetItem(kwds2, key, value))) goto bad;
+ } else if (!ignore_unknown_kwargs) {
+ goto invalid_keyword;
+ }
+ }
+ }
+ #if CYTHON_AVOID_BORROWED_REFS
+ Py_DECREF(key);
+ key = NULL;
+ #endif
+ }
+ return 0;
+invalid_keyword:
+ PyErr_Format(PyExc_TypeError,
+ "%s() got an unexpected keyword argument '%U'",
+ function_name, key);
+ goto bad;
+bad:
+ #if CYTHON_AVOID_BORROWED_REFS
+ Py_XDECREF(key);
+ #endif
+ return -1;
+}
+static int __Pyx_ParseKeywords(
+ PyObject *kwds,
+ PyObject * const *kwvalues,
+ PyObject ** const argnames[],
+ PyObject *kwds2,
+ PyObject *values[],
+ Py_ssize_t num_pos_args,
+ Py_ssize_t num_kwargs,
+ const char* function_name,
+ int ignore_unknown_kwargs)
+{
+ if (CYTHON_METH_FASTCALL && likely(PyTuple_Check(kwds)))
+ return __Pyx_ParseKeywordsTuple(kwds, kwvalues, argnames, kwds2, values, num_pos_args, num_kwargs, function_name, ignore_unknown_kwargs);
+ else if (kwds2)
+ return __Pyx_ParseKeywordDictToDict(kwds, argnames, kwds2, values, num_pos_args, function_name);
+ else
+ return __Pyx_ParseKeywordDict(kwds, argnames, values, num_pos_args, num_kwargs, function_name, ignore_unknown_kwargs);
+}
+
+/* RaiseArgTupleInvalid */
+static void __Pyx_RaiseArgtupleInvalid(
+ const char* func_name,
+ int exact,
+ Py_ssize_t num_min,
+ Py_ssize_t num_max,
+ Py_ssize_t num_found)
+{
+ Py_ssize_t num_expected;
+ const char *more_or_less;
+ if (num_found < num_min) {
+ num_expected = num_min;
+ more_or_less = "at least";
+ } else {
+ num_expected = num_max;
+ more_or_less = "at most";
+ }
+ if (exact) {
+ more_or_less = "exactly";
+ }
+ PyErr_Format(PyExc_TypeError,
+ "%.200s() takes %.8s %" CYTHON_FORMAT_SSIZE_T "d positional argument%.1s (%" CYTHON_FORMAT_SSIZE_T "d given)",
+ func_name, more_or_less, num_expected,
+ (num_expected == 1) ? "" : "s", num_found);
+}
+
+/* GetException */
+#if CYTHON_FAST_THREAD_STATE
+static int __Pyx__GetException(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb)
+#else
+static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb)
+#endif
+{
+ PyObject *local_type = NULL, *local_value, *local_tb = NULL;
+#if CYTHON_FAST_THREAD_STATE
+ PyObject *tmp_type, *tmp_value, *tmp_tb;
+ #if PY_VERSION_HEX >= 0x030C0000
+ local_value = tstate->current_exception;
+ tstate->current_exception = 0;
+ #else
+ local_type = tstate->curexc_type;
+ local_value = tstate->curexc_value;
+ local_tb = tstate->curexc_traceback;
+ tstate->curexc_type = 0;
+ tstate->curexc_value = 0;
+ tstate->curexc_traceback = 0;
+ #endif
+#elif __PYX_LIMITED_VERSION_HEX > 0x030C0000
+ local_value = PyErr_GetRaisedException();
+#else
+ PyErr_Fetch(&local_type, &local_value, &local_tb);
+#endif
+#if __PYX_LIMITED_VERSION_HEX > 0x030C0000
+ if (likely(local_value)) {
+ local_type = (PyObject*) Py_TYPE(local_value);
+ Py_INCREF(local_type);
+ local_tb = PyException_GetTraceback(local_value);
+ }
+#else
+ PyErr_NormalizeException(&local_type, &local_value, &local_tb);
+#if CYTHON_FAST_THREAD_STATE
+ if (unlikely(tstate->curexc_type))
+#else
+ if (unlikely(PyErr_Occurred()))
+#endif
+ goto bad;
+ if (local_tb) {
+ if (unlikely(PyException_SetTraceback(local_value, local_tb) < 0))
+ goto bad;
+ }
+#endif // __PYX_LIMITED_VERSION_HEX > 0x030C0000
+ Py_XINCREF(local_tb);
+ Py_XINCREF(local_type);
+ Py_XINCREF(local_value);
+ *type = local_type;
+ *value = local_value;
+ *tb = local_tb;
+#if CYTHON_FAST_THREAD_STATE
+ #if CYTHON_USE_EXC_INFO_STACK
+ {
+ _PyErr_StackItem *exc_info = tstate->exc_info;
+ #if PY_VERSION_HEX >= 0x030B00a4
+ tmp_value = exc_info->exc_value;
+ exc_info->exc_value = local_value;
+ tmp_type = NULL;
+ tmp_tb = NULL;
+ Py_XDECREF(local_type);
+ Py_XDECREF(local_tb);
+ #else
+ tmp_type = exc_info->exc_type;
+ tmp_value = exc_info->exc_value;
+ tmp_tb = exc_info->exc_traceback;
+ exc_info->exc_type = local_type;
+ exc_info->exc_value = local_value;
+ exc_info->exc_traceback = local_tb;
+ #endif
+ }
+ #else
+ tmp_type = tstate->exc_type;
+ tmp_value = tstate->exc_value;
+ tmp_tb = tstate->exc_traceback;
+ tstate->exc_type = local_type;
+ tstate->exc_value = local_value;
+ tstate->exc_traceback = local_tb;
+ #endif
+ Py_XDECREF(tmp_type);
+ Py_XDECREF(tmp_value);
+ Py_XDECREF(tmp_tb);
+#elif __PYX_LIMITED_VERSION_HEX >= 0x030b0000
+ PyErr_SetHandledException(local_value);
+ Py_XDECREF(local_value);
+ Py_XDECREF(local_type);
+ Py_XDECREF(local_tb);
+#else
+ PyErr_SetExcInfo(local_type, local_value, local_tb);
+#endif
+ return 0;
+#if __PYX_LIMITED_VERSION_HEX <= 0x030C0000
+bad:
+ *type = 0;
+ *value = 0;
+ *tb = 0;
+ Py_XDECREF(local_type);
+ Py_XDECREF(local_value);
+ Py_XDECREF(local_tb);
+ return -1;
+#endif
+}
+
+/* pep479 */
+static void __Pyx_Generator_Replace_StopIteration(int in_async_gen) {
+ PyObject *exc, *val, *tb, *cur_exc, *new_exc;
+ __Pyx_PyThreadState_declare
+ int is_async_stopiteration = 0;
+ CYTHON_MAYBE_UNUSED_VAR(in_async_gen);
+ __Pyx_PyThreadState_assign
+ cur_exc = __Pyx_PyErr_CurrentExceptionType();
+ if (likely(!__Pyx_PyErr_GivenExceptionMatches(cur_exc, PyExc_StopIteration))) {
+ if (in_async_gen && unlikely(__Pyx_PyErr_GivenExceptionMatches(cur_exc, PyExc_StopAsyncIteration))) {
+ is_async_stopiteration = 1;
+ } else {
+ return;
+ }
+ }
+ __Pyx_GetException(&exc, &val, &tb);
+ Py_XDECREF(exc);
+ Py_XDECREF(tb);
+ new_exc = PyObject_CallFunction(PyExc_RuntimeError, "s",
+ is_async_stopiteration ? "async generator raised StopAsyncIteration" :
+ in_async_gen ? "async generator raised StopIteration" :
+ "generator raised StopIteration");
+ if (!new_exc) {
+ Py_XDECREF(val);
+ return;
+ }
+ PyException_SetCause(new_exc, val); // steals ref to val
+ PyErr_SetObject(PyExc_RuntimeError, new_exc);
+}
+
+/* GetTopmostException */
+#if CYTHON_USE_EXC_INFO_STACK && CYTHON_FAST_THREAD_STATE
+static _PyErr_StackItem *
+__Pyx_PyErr_GetTopmostException(PyThreadState *tstate)
+{
+ _PyErr_StackItem *exc_info = tstate->exc_info;
+ while ((exc_info->exc_value == NULL || exc_info->exc_value == Py_None) &&
+ exc_info->previous_item != NULL)
+ {
+ exc_info = exc_info->previous_item;
+ }
+ return exc_info;
+}
+#endif
+
+/* SaveResetException */
+#if CYTHON_FAST_THREAD_STATE
+static CYTHON_INLINE void __Pyx__ExceptionSave(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) {
+ #if CYTHON_USE_EXC_INFO_STACK && PY_VERSION_HEX >= 0x030B00a4
+ _PyErr_StackItem *exc_info = __Pyx_PyErr_GetTopmostException(tstate);
+ PyObject *exc_value = exc_info->exc_value;
+ if (exc_value == NULL || exc_value == Py_None) {
+ *value = NULL;
+ *type = NULL;
+ *tb = NULL;
+ } else {
+ *value = exc_value;
+ Py_INCREF(*value);
+ *type = (PyObject*) Py_TYPE(exc_value);
+ Py_INCREF(*type);
+ *tb = PyException_GetTraceback(exc_value);
+ }
+ #elif CYTHON_USE_EXC_INFO_STACK
+ _PyErr_StackItem *exc_info = __Pyx_PyErr_GetTopmostException(tstate);
+ *type = exc_info->exc_type;
+ *value = exc_info->exc_value;
+ *tb = exc_info->exc_traceback;
+ Py_XINCREF(*type);
+ Py_XINCREF(*value);
+ Py_XINCREF(*tb);
+ #else
+ *type = tstate->exc_type;
+ *value = tstate->exc_value;
+ *tb = tstate->exc_traceback;
+ Py_XINCREF(*type);
+ Py_XINCREF(*value);
+ Py_XINCREF(*tb);
+ #endif
+}
+static CYTHON_INLINE void __Pyx__ExceptionReset(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) {
+ #if CYTHON_USE_EXC_INFO_STACK && PY_VERSION_HEX >= 0x030B00a4
+ _PyErr_StackItem *exc_info = tstate->exc_info;
+ PyObject *tmp_value = exc_info->exc_value;
+ exc_info->exc_value = value;
+ Py_XDECREF(tmp_value);
+ Py_XDECREF(type);
+ Py_XDECREF(tb);
+ #else
+ PyObject *tmp_type, *tmp_value, *tmp_tb;
+ #if CYTHON_USE_EXC_INFO_STACK
+ _PyErr_StackItem *exc_info = tstate->exc_info;
+ tmp_type = exc_info->exc_type;
+ tmp_value = exc_info->exc_value;
+ tmp_tb = exc_info->exc_traceback;
+ exc_info->exc_type = type;
+ exc_info->exc_value = value;
+ exc_info->exc_traceback = tb;
+ #else
+ tmp_type = tstate->exc_type;
+ tmp_value = tstate->exc_value;
+ tmp_tb = tstate->exc_traceback;
+ tstate->exc_type = type;
+ tstate->exc_value = value;
+ tstate->exc_traceback = tb;
+ #endif
+ Py_XDECREF(tmp_type);
+ Py_XDECREF(tmp_value);
+ Py_XDECREF(tmp_tb);
+ #endif
+}
+#endif
+
+/* IterNextPlain */
+#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030A0000
+static PyObject *__Pyx_GetBuiltinNext_LimitedAPI(void) {
+ if (unlikely(!__pyx_mstate_global->__Pyx_GetBuiltinNext_LimitedAPI_cache))
+ __pyx_mstate_global->__Pyx_GetBuiltinNext_LimitedAPI_cache = __Pyx_GetBuiltinName(__pyx_mstate_global->__pyx_n_u_next);
+ return __pyx_mstate_global->__Pyx_GetBuiltinNext_LimitedAPI_cache;
+}
+#endif
+static CYTHON_INLINE PyObject *__Pyx_PyIter_Next_Plain(PyObject *iterator) {
+#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030A0000
+ PyObject *result;
+ PyObject *next = __Pyx_GetBuiltinNext_LimitedAPI();
+ if (unlikely(!next)) return NULL;
+ result = PyObject_CallFunctionObjArgs(next, iterator, NULL);
+ return result;
+#else
+ (void)__Pyx_GetBuiltinName; // only for early limited API
+ iternextfunc iternext = __Pyx_PyObject_GetIterNextFunc(iterator);
+ assert(iternext);
+ return iternext(iterator);
+#endif
+}
+
+/* IterNext */
+#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x03080000
+static PyObject *__Pyx_PyIter_Next2(PyObject *o, PyObject *defval) {
+ PyObject *result;
+ PyObject *next = __Pyx_GetBuiltinNext_LimitedAPI();
+ if (unlikely(!next)) return NULL;
+ result = PyObject_CallFunctionObjArgs(next, o, defval, NULL);
+ return result;
+}
+#else
+static PyObject *__Pyx_PyIter_Next2Default(PyObject* defval) {
+ PyObject* exc_type;
+ __Pyx_PyThreadState_declare
+ __Pyx_PyThreadState_assign
+ exc_type = __Pyx_PyErr_CurrentExceptionType();
+ if (unlikely(exc_type)) {
+ if (!defval || unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration)))
+ return NULL;
+ __Pyx_PyErr_Clear();
+ Py_INCREF(defval);
+ return defval;
+ }
+ if (defval) {
+ Py_INCREF(defval);
+ return defval;
+ }
+ __Pyx_PyErr_SetNone(PyExc_StopIteration);
+ return NULL;
+}
+static void __Pyx_PyIter_Next_ErrorNoIterator(PyObject *iterator) {
+ __Pyx_TypeName iterator_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(iterator));
+ PyErr_Format(PyExc_TypeError,
+ __Pyx_FMT_TYPENAME " object is not an iterator", iterator_type_name);
+ __Pyx_DECREF_TypeName(iterator_type_name);
+}
+static CYTHON_INLINE PyObject *__Pyx_PyIter_Next2(PyObject* iterator, PyObject* defval) {
+ PyObject* next;
+#if !CYTHON_COMPILING_IN_LIMITED_API
+ iternextfunc iternext = __Pyx_PyObject_TryGetSlot(iterator, tp_iternext, iternextfunc);
+ if (likely(iternext)) {
+ next = iternext(iterator);
+ if (likely(next))
+ return next;
+ #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030d0000
+ if (unlikely(iternext == &_PyObject_NextNotImplemented))
+ return NULL;
+ #endif
+ } else if (CYTHON_USE_TYPE_SLOTS) {
+ __Pyx_PyIter_Next_ErrorNoIterator(iterator);
+ return NULL;
+ } else
+#endif
+ if (unlikely(!PyIter_Check(iterator))) {
+ __Pyx_PyIter_Next_ErrorNoIterator(iterator);
+ return NULL;
+ } else {
+ next = defval ? PyIter_Next(iterator) : __Pyx_PyIter_Next_Plain(iterator);
+ if (likely(next))
+ return next;
+ }
+ return __Pyx_PyIter_Next2Default(defval);
+}
+#endif
+
+/* PyLongBinop */
+#if !CYTHON_COMPILING_IN_PYPY
+static PyObject* __Pyx_Fallback___Pyx_PyLong_AddObjC(PyObject *op1, PyObject *op2, int inplace) {
+ return (inplace ? PyNumber_InPlaceAdd : PyNumber_Add)(op1, op2);
+}
+#if CYTHON_USE_PYLONG_INTERNALS
+static PyObject* __Pyx_Unpacked___Pyx_PyLong_AddObjC(PyObject *op1, PyObject *op2, long intval, int inplace, int zerodivision_check) {
+ CYTHON_MAYBE_UNUSED_VAR(inplace);
+ CYTHON_UNUSED_VAR(zerodivision_check);
+ const long b = intval;
+ long a, x;
+#ifdef HAVE_LONG_LONG
+ const PY_LONG_LONG llb = intval;
+ PY_LONG_LONG lla, llx;
+#endif
+ if (unlikely(__Pyx_PyLong_IsZero(op1))) {
+ return __Pyx_NewRef(op2);
+ }
+ if (likely(__Pyx_PyLong_IsCompact(op1))) {
+ a = __Pyx_PyLong_CompactValue(op1);
+ } else {
+ const digit* digits = __Pyx_PyLong_Digits(op1);
+ const Py_ssize_t size = __Pyx_PyLong_SignedDigitCount(op1);
+ switch (size) {
+ case -2:
+ if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) {
+ a = -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]));
+ break;
+ #ifdef HAVE_LONG_LONG
+ } else if (8 * sizeof(PY_LONG_LONG) - 1 > 2 * PyLong_SHIFT) {
+ lla = -(PY_LONG_LONG) (((((unsigned PY_LONG_LONG)digits[1]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[0]));
+ goto long_long;
+ #endif
+ }
+ CYTHON_FALLTHROUGH;
+ case 2:
+ if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) {
+ a = (long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]));
+ break;
+ #ifdef HAVE_LONG_LONG
+ } else if (8 * sizeof(PY_LONG_LONG) - 1 > 2 * PyLong_SHIFT) {
+ lla = (PY_LONG_LONG) (((((unsigned PY_LONG_LONG)digits[1]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[0]));
+ goto long_long;
+ #endif
+ }
+ CYTHON_FALLTHROUGH;
+ case -3:
+ if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) {
+ a = -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]));
+ break;
+ #ifdef HAVE_LONG_LONG
+ } else if (8 * sizeof(PY_LONG_LONG) - 1 > 3 * PyLong_SHIFT) {
+ lla = -(PY_LONG_LONG) (((((((unsigned PY_LONG_LONG)digits[2]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[1]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[0]));
+ goto long_long;
+ #endif
+ }
+ CYTHON_FALLTHROUGH;
+ case 3:
+ if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) {
+ a = (long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]));
+ break;
+ #ifdef HAVE_LONG_LONG
+ } else if (8 * sizeof(PY_LONG_LONG) - 1 > 3 * PyLong_SHIFT) {
+ lla = (PY_LONG_LONG) (((((((unsigned PY_LONG_LONG)digits[2]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[1]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[0]));
+ goto long_long;
+ #endif
+ }
+ CYTHON_FALLTHROUGH;
+ case -4:
+ if (8 * sizeof(long) - 1 > 4 * PyLong_SHIFT) {
+ a = -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]));
+ break;
+ #ifdef HAVE_LONG_LONG
+ } else if (8 * sizeof(PY_LONG_LONG) - 1 > 4 * PyLong_SHIFT) {
+ lla = -(PY_LONG_LONG) (((((((((unsigned PY_LONG_LONG)digits[3]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[2]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[1]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[0]));
+ goto long_long;
+ #endif
+ }
+ CYTHON_FALLTHROUGH;
+ case 4:
+ if (8 * sizeof(long) - 1 > 4 * PyLong_SHIFT) {
+ a = (long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]));
+ break;
+ #ifdef HAVE_LONG_LONG
+ } else if (8 * sizeof(PY_LONG_LONG) - 1 > 4 * PyLong_SHIFT) {
+ lla = (PY_LONG_LONG) (((((((((unsigned PY_LONG_LONG)digits[3]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[2]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[1]) << PyLong_SHIFT) | (unsigned PY_LONG_LONG)digits[0]));
+ goto long_long;
+ #endif
+ }
+ CYTHON_FALLTHROUGH;
+ default: return PyLong_Type.tp_as_number->nb_add(op1, op2);
+ }
+ }
+ x = a + b;
+ return PyLong_FromLong(x);
+#ifdef HAVE_LONG_LONG
+ long_long:
+ llx = lla + llb;
+ return PyLong_FromLongLong(llx);
+#endif
+ return __Pyx_Fallback___Pyx_PyLong_AddObjC(op1, op2, inplace);
+
+
+}
+#endif
+static PyObject* __Pyx_Float___Pyx_PyLong_AddObjC(PyObject *float_val, long intval, int zerodivision_check) {
+ CYTHON_UNUSED_VAR(zerodivision_check);
+ const long b = intval;
+ double a = __Pyx_PyFloat_AS_DOUBLE(float_val);
+ double result;
+
+ result = ((double)a) + (double)b;
+ return PyFloat_FromDouble(result);
+}
+static CYTHON_INLINE PyObject* __Pyx_PyLong_AddObjC(PyObject *op1, PyObject *op2, long intval, int inplace, int zerodivision_check) {
+ CYTHON_MAYBE_UNUSED_VAR(intval);
+ CYTHON_UNUSED_VAR(zerodivision_check);
+ #if CYTHON_USE_PYLONG_INTERNALS
+ if (likely(PyLong_CheckExact(op1))) {
+ return __Pyx_Unpacked___Pyx_PyLong_AddObjC(op1, op2, intval, inplace, zerodivision_check);
+ }
+ #endif
+ if (PyFloat_CheckExact(op1)) {
+ return __Pyx_Float___Pyx_PyLong_AddObjC(op1, intval, zerodivision_check);
+ }
+ return __Pyx_Fallback___Pyx_PyLong_AddObjC(op1, op2, inplace);
+}
+#endif
+
+/* RaiseException */
+static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause) {
+ PyObject* owned_instance = NULL;
+ if (tb == Py_None) {
+ tb = 0;
+ } else if (tb && !PyTraceBack_Check(tb)) {
+ PyErr_SetString(PyExc_TypeError,
+ "raise: arg 3 must be a traceback or None");
+ goto bad;
+ }
+ if (value == Py_None)
+ value = 0;
+ if (PyExceptionInstance_Check(type)) {
+ if (value) {
+ PyErr_SetString(PyExc_TypeError,
+ "instance exception may not have a separate value");
+ goto bad;
+ }
+ value = type;
+ type = (PyObject*) Py_TYPE(value);
+ } else if (PyExceptionClass_Check(type)) {
+ PyObject *instance_class = NULL;
+ if (value && PyExceptionInstance_Check(value)) {
+ instance_class = (PyObject*) Py_TYPE(value);
+ if (instance_class != type) {
+ int is_subclass = PyObject_IsSubclass(instance_class, type);
+ if (!is_subclass) {
+ instance_class = NULL;
+ } else if (unlikely(is_subclass == -1)) {
+ goto bad;
+ } else {
+ type = instance_class;
+ }
+ }
+ }
+ if (!instance_class) {
+ PyObject *args;
+ if (!value)
+ args = PyTuple_New(0);
+ else if (PyTuple_Check(value)) {
+ Py_INCREF(value);
+ args = value;
+ } else
+ args = PyTuple_Pack(1, value);
+ if (!args)
+ goto bad;
+ owned_instance = PyObject_Call(type, args, NULL);
+ Py_DECREF(args);
+ if (!owned_instance)
+ goto bad;
+ value = owned_instance;
+ if (!PyExceptionInstance_Check(value)) {
+ PyErr_Format(PyExc_TypeError,
+ "calling %R should have returned an instance of "
+ "BaseException, not %R",
+ type, Py_TYPE(value));
+ goto bad;
+ }
+ }
+ } else {
+ PyErr_SetString(PyExc_TypeError,
+ "raise: exception class must be a subclass of BaseException");
+ goto bad;
+ }
+ if (cause) {
+ PyObject *fixed_cause;
+ if (cause == Py_None) {
+ fixed_cause = NULL;
+ } else if (PyExceptionClass_Check(cause)) {
+ fixed_cause = PyObject_CallObject(cause, NULL);
+ if (fixed_cause == NULL)
+ goto bad;
+ } else if (PyExceptionInstance_Check(cause)) {
+ fixed_cause = cause;
+ Py_INCREF(fixed_cause);
+ } else {
+ PyErr_SetString(PyExc_TypeError,
+ "exception causes must derive from "
+ "BaseException");
+ goto bad;
+ }
+ PyException_SetCause(value, fixed_cause);
+ }
+ PyErr_SetObject(type, value);
+ if (tb) {
+#if PY_VERSION_HEX >= 0x030C00A6
+ PyException_SetTraceback(value, tb);
+#elif CYTHON_FAST_THREAD_STATE
+ PyThreadState *tstate = __Pyx_PyThreadState_Current;
+ PyObject* tmp_tb = tstate->curexc_traceback;
+ if (tb != tmp_tb) {
+ Py_INCREF(tb);
+ tstate->curexc_traceback = tb;
+ Py_XDECREF(tmp_tb);
+ }
+#else
+ PyObject *tmp_type, *tmp_value, *tmp_tb;
+ PyErr_Fetch(&tmp_type, &tmp_value, &tmp_tb);
+ Py_INCREF(tb);
+ PyErr_Restore(tmp_type, tmp_value, tb);
+ Py_XDECREF(tmp_tb);
+#endif
+ }
+bad:
+ Py_XDECREF(owned_instance);
+ return;
+}
+
+/* SetItemInt */
+static int __Pyx_SetItemInt_Generic(PyObject *o, PyObject *j, PyObject *v) {
+ int r;
+ if (unlikely(!j)) return -1;
+ r = PyObject_SetItem(o, j, v);
+ Py_DECREF(j);
+ return r;
+}
+static CYTHON_INLINE int __Pyx_SetItemInt_Fast(PyObject *o, Py_ssize_t i, PyObject *v, int is_list,
+ CYTHON_NCP_UNUSED int wraparound, CYTHON_NCP_UNUSED int boundscheck) {
+#if CYTHON_ASSUME_SAFE_MACROS && CYTHON_ASSUME_SAFE_SIZE && !CYTHON_AVOID_BORROWED_REFS && CYTHON_USE_TYPE_SLOTS
+ if (is_list || PyList_CheckExact(o)) {
+ Py_ssize_t n = (!wraparound) ? i : ((likely(i >= 0)) ? i : i + PyList_GET_SIZE(o));
+ if ((!boundscheck) || likely(__Pyx_is_valid_index(n, PyList_GET_SIZE(o)))) {
+ Py_INCREF(v);
+#if CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS
+ PyList_SetItem(o, n, v);
+#else
+ PyObject* old = PyList_GET_ITEM(o, n);
+ PyList_SET_ITEM(o, n, v);
+ Py_DECREF(old);
+#endif
+ return 1;
+ }
+ } else {
+ PyMappingMethods *mm = Py_TYPE(o)->tp_as_mapping;
+ PySequenceMethods *sm = Py_TYPE(o)->tp_as_sequence;
+ if (mm && mm->mp_ass_subscript) {
+ int r;
+ PyObject *key = PyLong_FromSsize_t(i);
+ if (unlikely(!key)) return -1;
+ r = mm->mp_ass_subscript(o, key, v);
+ Py_DECREF(key);
+ return r;
+ }
+ if (likely(sm && sm->sq_ass_item)) {
+ if (wraparound && unlikely(i < 0) && likely(sm->sq_length)) {
+ Py_ssize_t l = sm->sq_length(o);
+ if (likely(l >= 0)) {
+ i += l;
+ } else {
+ if (!PyErr_ExceptionMatches(PyExc_OverflowError))
+ return -1;
+ PyErr_Clear();
+ }
+ }
+ return sm->sq_ass_item(o, i, v);
+ }
+ }
+#else
+ if (is_list || !PyMapping_Check(o))
+ {
+ return PySequence_SetItem(o, i, v);
+ }
+#endif
+ return __Pyx_SetItemInt_Generic(o, PyLong_FromSsize_t(i), v);
+}
+
+/* ModInt[long] */
+static CYTHON_INLINE long __Pyx_mod_long(long a, long b, int b_is_constant) {
+ long r = a % b;
+ long adapt_python = (b_is_constant ?
+ ((r != 0) & ((r < 0) ^ (b < 0))) :
+ ((r != 0) & ((r ^ b) < 0))
+ );
+ return r + adapt_python * b;
+}
+
+/* LimitedApiGetTypeDict */
+#if CYTHON_COMPILING_IN_LIMITED_API
+static Py_ssize_t __Pyx_GetTypeDictOffset(void) {
+ PyObject *tp_dictoffset_o;
+ Py_ssize_t tp_dictoffset;
+ tp_dictoffset_o = PyObject_GetAttrString((PyObject*)(&PyType_Type), "__dictoffset__");
+ if (unlikely(!tp_dictoffset_o)) return -1;
+ tp_dictoffset = PyLong_AsSsize_t(tp_dictoffset_o);
+ Py_DECREF(tp_dictoffset_o);
+ if (unlikely(tp_dictoffset == 0)) {
+ PyErr_SetString(
+ PyExc_TypeError,
+ "'type' doesn't have a dictoffset");
+ return -1;
+ } else if (unlikely(tp_dictoffset < 0)) {
+ PyErr_SetString(
+ PyExc_TypeError,
+ "'type' has an unexpected negative dictoffset. "
+ "Please report this as Cython bug");
+ return -1;
+ }
+ return tp_dictoffset;
+}
+static PyObject *__Pyx_GetTypeDict(PyTypeObject *tp) {
+ static Py_ssize_t tp_dictoffset = 0;
+ if (unlikely(tp_dictoffset == 0)) {
+ tp_dictoffset = __Pyx_GetTypeDictOffset();
+ if (unlikely(tp_dictoffset == -1 && PyErr_Occurred())) {
+ tp_dictoffset = 0; // try again next time?
+ return NULL;
+ }
+ }
+ return *(PyObject**)((char*)tp + tp_dictoffset);
+}
+#endif
+
+/* SetItemOnTypeDict */
+static int __Pyx__SetItemOnTypeDict(PyTypeObject *tp, PyObject *k, PyObject *v) {
+ int result;
+ PyObject *tp_dict;
+#if CYTHON_COMPILING_IN_LIMITED_API
+ tp_dict = __Pyx_GetTypeDict(tp);
+ if (unlikely(!tp_dict)) return -1;
+#else
+ tp_dict = tp->tp_dict;
+#endif
+ result = PyDict_SetItem(tp_dict, k, v);
+ if (likely(!result)) {
+ PyType_Modified(tp);
+ if (unlikely(PyObject_HasAttr(v, __pyx_mstate_global->__pyx_n_u_set_name))) {
+ PyObject *setNameResult = PyObject_CallMethodObjArgs(v, __pyx_mstate_global->__pyx_n_u_set_name, (PyObject *) tp, k, NULL);
+ if (!setNameResult) return -1;
+ Py_DECREF(setNameResult);
+ }
+ }
+ return result;
+}
+
+/* FixUpExtensionType */
+static int __Pyx_fix_up_extension_type_from_spec(PyType_Spec *spec, PyTypeObject *type) {
+#if __PYX_LIMITED_VERSION_HEX > 0x030900B1
+ CYTHON_UNUSED_VAR(spec);
+ CYTHON_UNUSED_VAR(type);
+ CYTHON_UNUSED_VAR(__Pyx__SetItemOnTypeDict);
+#else
+ const PyType_Slot *slot = spec->slots;
+ int changed = 0;
+#if !CYTHON_COMPILING_IN_LIMITED_API
+ while (slot && slot->slot && slot->slot != Py_tp_members)
+ slot++;
+ if (slot && slot->slot == Py_tp_members) {
+#if !CYTHON_COMPILING_IN_CPYTHON
+ const
+#endif // !CYTHON_COMPILING_IN_CPYTHON)
+ PyMemberDef *memb = (PyMemberDef*) slot->pfunc;
+ while (memb && memb->name) {
+ if (memb->name[0] == '_' && memb->name[1] == '_') {
+ if (strcmp(memb->name, "__weaklistoffset__") == 0) {
+ assert(memb->type == T_PYSSIZET);
+ assert(memb->flags == READONLY);
+ type->tp_weaklistoffset = memb->offset;
+ changed = 1;
+ }
+ else if (strcmp(memb->name, "__dictoffset__") == 0) {
+ assert(memb->type == T_PYSSIZET);
+ assert(memb->flags == READONLY);
+ type->tp_dictoffset = memb->offset;
+ changed = 1;
+ }
+#if CYTHON_METH_FASTCALL
+ else if (strcmp(memb->name, "__vectorcalloffset__") == 0) {
+ assert(memb->type == T_PYSSIZET);
+ assert(memb->flags == READONLY);
+#if PY_VERSION_HEX >= 0x030800b4
+ type->tp_vectorcall_offset = memb->offset;
+#else
+ type->tp_print = (printfunc) memb->offset;
+#endif
+ changed = 1;
+ }
+#endif // CYTHON_METH_FASTCALL
+#if !CYTHON_COMPILING_IN_PYPY
+ else if (strcmp(memb->name, "__module__") == 0) {
+ PyObject *descr;
+ assert(memb->type == T_OBJECT);
+ assert(memb->flags == 0 || memb->flags == READONLY);
+ descr = PyDescr_NewMember(type, memb);
+ if (unlikely(!descr))
+ return -1;
+ int set_item_result = PyDict_SetItem(type->tp_dict, PyDescr_NAME(descr), descr);
+ Py_DECREF(descr);
+ if (unlikely(set_item_result < 0)) {
+ return -1;
+ }
+ changed = 1;
+ }
+#endif // !CYTHON_COMPILING_IN_PYPY
+ }
+ memb++;
+ }
+ }
+#endif // !CYTHON_COMPILING_IN_LIMITED_API
+#if !CYTHON_COMPILING_IN_PYPY
+ slot = spec->slots;
+ while (slot && slot->slot && slot->slot != Py_tp_getset)
+ slot++;
+ if (slot && slot->slot == Py_tp_getset) {
+ PyGetSetDef *getset = (PyGetSetDef*) slot->pfunc;
+ while (getset && getset->name) {
+ if (getset->name[0] == '_' && getset->name[1] == '_' && strcmp(getset->name, "__module__") == 0) {
+ PyObject *descr = PyDescr_NewGetSet(type, getset);
+ if (unlikely(!descr))
+ return -1;
+ #if CYTHON_COMPILING_IN_LIMITED_API
+ PyObject *pyname = PyUnicode_FromString(getset->name);
+ if (unlikely(!pyname)) {
+ Py_DECREF(descr);
+ return -1;
+ }
+ int set_item_result = __Pyx_SetItemOnTypeDict(type, pyname, descr);
+ Py_DECREF(pyname);
+ #else
+ CYTHON_UNUSED_VAR(__Pyx__SetItemOnTypeDict);
+ int set_item_result = PyDict_SetItem(type->tp_dict, PyDescr_NAME(descr), descr);
+ #endif
+ Py_DECREF(descr);
+ if (unlikely(set_item_result < 0)) {
+ return -1;
+ }
+ changed = 1;
+ }
+ ++getset;
+ }
+ }
+#endif // !CYTHON_COMPILING_IN_PYPY
+ if (changed)
+ PyType_Modified(type);
+#endif // PY_VERSION_HEX > 0x030900B1
+ return 0;
+}
+
+/* PyObjectCallNoArg */
+static CYTHON_INLINE PyObject* __Pyx_PyObject_CallNoArg(PyObject *func) {
+ PyObject *arg[2] = {NULL, NULL};
+ return __Pyx_PyObject_FastCall(func, arg + 1, 0 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET);
+}
+
+/* PyObjectCallOneArg */
+static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) {
+ PyObject *args[2] = {NULL, arg};
+ return __Pyx_PyObject_FastCall(func, args+1, 1 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET);
+}
+
+/* PyObjectGetMethod */
+static int __Pyx_PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method) {
+ PyObject *attr;
+#if CYTHON_UNPACK_METHODS && CYTHON_COMPILING_IN_CPYTHON && CYTHON_USE_PYTYPE_LOOKUP
+ __Pyx_TypeName type_name;
+ PyTypeObject *tp = Py_TYPE(obj);
+ PyObject *descr;
+ descrgetfunc f = NULL;
+ PyObject **dictptr, *dict;
+ int meth_found = 0;
+ assert (*method == NULL);
+ if (unlikely(tp->tp_getattro != PyObject_GenericGetAttr)) {
+ attr = __Pyx_PyObject_GetAttrStr(obj, name);
+ goto try_unpack;
+ }
+ if (unlikely(tp->tp_dict == NULL) && unlikely(PyType_Ready(tp) < 0)) {
+ return 0;
+ }
+ descr = _PyType_Lookup(tp, name);
+ if (likely(descr != NULL)) {
+ Py_INCREF(descr);
+#if defined(Py_TPFLAGS_METHOD_DESCRIPTOR) && Py_TPFLAGS_METHOD_DESCRIPTOR
+ if (__Pyx_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR))
+#else
+ #ifdef __Pyx_CyFunction_USED
+ if (likely(PyFunction_Check(descr) || __Pyx_IS_TYPE(descr, &PyMethodDescr_Type) || __Pyx_CyFunction_Check(descr)))
+ #else
+ if (likely(PyFunction_Check(descr) || __Pyx_IS_TYPE(descr, &PyMethodDescr_Type)))
+ #endif
+#endif
+ {
+ meth_found = 1;
+ } else {
+ f = Py_TYPE(descr)->tp_descr_get;
+ if (f != NULL && PyDescr_IsData(descr)) {
+ attr = f(descr, obj, (PyObject *)Py_TYPE(obj));
+ Py_DECREF(descr);
+ goto try_unpack;
+ }
+ }
+ }
+ dictptr = _PyObject_GetDictPtr(obj);
+ if (dictptr != NULL && (dict = *dictptr) != NULL) {
+ Py_INCREF(dict);
+ attr = __Pyx_PyDict_GetItemStr(dict, name);
+ if (attr != NULL) {
+ Py_INCREF(attr);
+ Py_DECREF(dict);
+ Py_XDECREF(descr);
+ goto try_unpack;
+ }
+ Py_DECREF(dict);
+ }
+ if (meth_found) {
+ *method = descr;
+ return 1;
+ }
+ if (f != NULL) {
+ attr = f(descr, obj, (PyObject *)Py_TYPE(obj));
+ Py_DECREF(descr);
+ goto try_unpack;
+ }
+ if (likely(descr != NULL)) {
+ *method = descr;
+ return 0;
+ }
+ type_name = __Pyx_PyType_GetFullyQualifiedName(tp);
+ PyErr_Format(PyExc_AttributeError,
+ "'" __Pyx_FMT_TYPENAME "' object has no attribute '%U'",
+ type_name, name);
+ __Pyx_DECREF_TypeName(type_name);
+ return 0;
+#else
+ attr = __Pyx_PyObject_GetAttrStr(obj, name);
+ goto try_unpack;
+#endif
+try_unpack:
+#if CYTHON_UNPACK_METHODS
+ if (likely(attr) && PyMethod_Check(attr) && likely(PyMethod_GET_SELF(attr) == obj)) {
+ PyObject *function = PyMethod_GET_FUNCTION(attr);
+ Py_INCREF(function);
+ Py_DECREF(attr);
+ *method = function;
+ return 1;
+ }
+#endif
+ *method = attr;
+ return 0;
+}
+
+/* PyObjectCallMethod0 */
+static PyObject* __Pyx_PyObject_CallMethod0(PyObject* obj, PyObject* method_name) {
+#if CYTHON_VECTORCALL && (__PYX_LIMITED_VERSION_HEX >= 0x030C0000 || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x03090000))
+ PyObject *args[1] = {obj};
+ (void) __Pyx_PyObject_GetMethod;
+ (void) __Pyx_PyObject_CallOneArg;
+ (void) __Pyx_PyObject_CallNoArg;
+ return PyObject_VectorcallMethod(method_name, args, 1 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL);
+#else
+ PyObject *method = NULL, *result = NULL;
+ int is_method = __Pyx_PyObject_GetMethod(obj, method_name, &method);
+ if (likely(is_method)) {
+ result = __Pyx_PyObject_CallOneArg(method, obj);
+ Py_DECREF(method);
+ return result;
+ }
+ if (unlikely(!method)) goto bad;
+ result = __Pyx_PyObject_CallNoArg(method);
+ Py_DECREF(method);
+bad:
+ return result;
+#endif
+}
+
+/* ValidateBasesTuple */
+#if CYTHON_COMPILING_IN_CPYTHON || CYTHON_COMPILING_IN_LIMITED_API || CYTHON_USE_TYPE_SPECS
+static int __Pyx_validate_bases_tuple(const char *type_name, Py_ssize_t dictoffset, PyObject *bases) {
+ Py_ssize_t i, n;
+#if CYTHON_ASSUME_SAFE_SIZE
+ n = PyTuple_GET_SIZE(bases);
+#else
+ n = PyTuple_Size(bases);
+ if (unlikely(n < 0)) return -1;
+#endif
+ for (i = 1; i < n; i++)
+ {
+ PyTypeObject *b;
+#if CYTHON_AVOID_BORROWED_REFS
+ PyObject *b0 = PySequence_GetItem(bases, i);
+ if (!b0) return -1;
+#elif CYTHON_ASSUME_SAFE_MACROS
+ PyObject *b0 = PyTuple_GET_ITEM(bases, i);
+#else
+ PyObject *b0 = PyTuple_GetItem(bases, i);
+ if (!b0) return -1;
+#endif
+ b = (PyTypeObject*) b0;
+ if (!__Pyx_PyType_HasFeature(b, Py_TPFLAGS_HEAPTYPE))
+ {
+ __Pyx_TypeName b_name = __Pyx_PyType_GetFullyQualifiedName(b);
+ PyErr_Format(PyExc_TypeError,
+ "base class '" __Pyx_FMT_TYPENAME "' is not a heap type", b_name);
+ __Pyx_DECREF_TypeName(b_name);
+#if CYTHON_AVOID_BORROWED_REFS
+ Py_DECREF(b0);
+#endif
+ return -1;
+ }
+ if (dictoffset == 0)
+ {
+ Py_ssize_t b_dictoffset = 0;
+#if CYTHON_USE_TYPE_SLOTS
+ b_dictoffset = b->tp_dictoffset;
+#else
+ PyObject *py_b_dictoffset = PyObject_GetAttrString((PyObject*)b, "__dictoffset__");
+ if (!py_b_dictoffset) goto dictoffset_return;
+ b_dictoffset = PyLong_AsSsize_t(py_b_dictoffset);
+ Py_DECREF(py_b_dictoffset);
+ if (b_dictoffset == -1 && PyErr_Occurred()) goto dictoffset_return;
+#endif
+ if (b_dictoffset) {
+ {
+ __Pyx_TypeName b_name = __Pyx_PyType_GetFullyQualifiedName(b);
+ PyErr_Format(PyExc_TypeError,
+ "extension type '%.200s' has no __dict__ slot, "
+ "but base type '" __Pyx_FMT_TYPENAME "' has: "
+ "either add 'cdef dict __dict__' to the extension type "
+ "or add '__slots__ = [...]' to the base type",
+ type_name, b_name);
+ __Pyx_DECREF_TypeName(b_name);
+ }
+#if !CYTHON_USE_TYPE_SLOTS
+ dictoffset_return:
+#endif
+#if CYTHON_AVOID_BORROWED_REFS
+ Py_DECREF(b0);
+#endif
+ return -1;
+ }
+ }
+#if CYTHON_AVOID_BORROWED_REFS
+ Py_DECREF(b0);
+#endif
+ }
+ return 0;
+}
+#endif
+
+/* PyType_Ready */
+CYTHON_UNUSED static int __Pyx_PyType_HasMultipleInheritance(PyTypeObject *t) {
+ while (t) {
+ PyObject *bases = __Pyx_PyType_GetSlot(t, tp_bases, PyObject*);
+ if (bases) {
+ return 1;
+ }
+ t = __Pyx_PyType_GetSlot(t, tp_base, PyTypeObject*);
+ }
+ return 0;
+}
+static int __Pyx_PyType_Ready(PyTypeObject *t) {
+#if CYTHON_USE_TYPE_SPECS || !CYTHON_COMPILING_IN_CPYTHON || defined(PYSTON_MAJOR_VERSION)
+ (void)__Pyx_PyObject_CallMethod0;
+#if CYTHON_USE_TYPE_SPECS
+ (void)__Pyx_validate_bases_tuple;
+#endif
+ return PyType_Ready(t);
+#else
+ int r;
+ if (!__Pyx_PyType_HasMultipleInheritance(t)) {
+ return PyType_Ready(t);
+ }
+ PyObject *bases = __Pyx_PyType_GetSlot(t, tp_bases, PyObject*);
+ if (bases && unlikely(__Pyx_validate_bases_tuple(t->tp_name, t->tp_dictoffset, bases) == -1))
+ return -1;
+#if !defined(PYSTON_MAJOR_VERSION)
+ {
+ int gc_was_enabled;
+ #if PY_VERSION_HEX >= 0x030A00b1
+ gc_was_enabled = PyGC_Disable();
+ (void)__Pyx_PyObject_CallMethod0;
+ #else
+ PyObject *ret, *py_status;
+ PyObject *gc = NULL;
+ #if (!CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM+0 >= 0x07030400) &&\
+ !CYTHON_COMPILING_IN_GRAAL
+ gc = PyImport_GetModule(__pyx_mstate_global->__pyx_kp_u_gc);
+ #endif
+ if (unlikely(!gc)) gc = PyImport_Import(__pyx_mstate_global->__pyx_kp_u_gc);
+ if (unlikely(!gc)) return -1;
+ py_status = __Pyx_PyObject_CallMethod0(gc, __pyx_mstate_global->__pyx_kp_u_isenabled);
+ if (unlikely(!py_status)) {
+ Py_DECREF(gc);
+ return -1;
+ }
+ gc_was_enabled = __Pyx_PyObject_IsTrue(py_status);
+ Py_DECREF(py_status);
+ if (gc_was_enabled > 0) {
+ ret = __Pyx_PyObject_CallMethod0(gc, __pyx_mstate_global->__pyx_kp_u_disable);
+ if (unlikely(!ret)) {
+ Py_DECREF(gc);
+ return -1;
+ }
+ Py_DECREF(ret);
+ } else if (unlikely(gc_was_enabled == -1)) {
+ Py_DECREF(gc);
+ return -1;
+ }
+ #endif
+ t->tp_flags |= Py_TPFLAGS_HEAPTYPE;
+#if PY_VERSION_HEX >= 0x030A0000
+ t->tp_flags |= Py_TPFLAGS_IMMUTABLETYPE;
+#endif
+#else
+ (void)__Pyx_PyObject_CallMethod0;
+#endif
+ r = PyType_Ready(t);
+#if !defined(PYSTON_MAJOR_VERSION)
+ t->tp_flags &= ~Py_TPFLAGS_HEAPTYPE;
+ #if PY_VERSION_HEX >= 0x030A00b1
+ if (gc_was_enabled)
+ PyGC_Enable();
+ #else
+ if (gc_was_enabled) {
+ PyObject *tp, *v, *tb;
+ PyErr_Fetch(&tp, &v, &tb);
+ ret = __Pyx_PyObject_CallMethod0(gc, __pyx_mstate_global->__pyx_kp_u_enable);
+ if (likely(ret || r == -1)) {
+ Py_XDECREF(ret);
+ PyErr_Restore(tp, v, tb);
+ } else {
+ Py_XDECREF(tp);
+ Py_XDECREF(v);
+ Py_XDECREF(tb);
+ r = -1;
+ }
+ }
+ Py_DECREF(gc);
+ #endif
+ }
+#endif
+ return r;
+#endif
+}
+
+/* Import */
+static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level) {
+ PyObject *module = 0;
+ PyObject *empty_dict = 0;
+ PyObject *empty_list = 0;
+ empty_dict = PyDict_New();
+ if (unlikely(!empty_dict))
+ goto bad;
+ if (level == -1) {
+ const char* package_sep = strchr(__Pyx_MODULE_NAME, '.');
+ if (package_sep != (0)) {
+ module = PyImport_ImportModuleLevelObject(
+ name, __pyx_mstate_global->__pyx_d, empty_dict, from_list, 1);
+ if (unlikely(!module)) {
+ if (unlikely(!PyErr_ExceptionMatches(PyExc_ImportError)))
+ goto bad;
+ PyErr_Clear();
+ }
+ }
+ level = 0;
+ }
+ if (!module) {
+ module = PyImport_ImportModuleLevelObject(
+ name, __pyx_mstate_global->__pyx_d, empty_dict, from_list, level);
+ }
+bad:
+ Py_XDECREF(empty_dict);
+ Py_XDECREF(empty_list);
+ return module;
+}
+
+/* ImportDottedModule */
+static PyObject *__Pyx__ImportDottedModule_Error(PyObject *name, PyObject *parts_tuple, Py_ssize_t count) {
+ PyObject *partial_name = NULL, *slice = NULL, *sep = NULL;
+ Py_ssize_t size;
+ if (unlikely(PyErr_Occurred())) {
+ PyErr_Clear();
+ }
+#if CYTHON_ASSUME_SAFE_SIZE
+ size = PyTuple_GET_SIZE(parts_tuple);
+#else
+ size = PyTuple_Size(parts_tuple);
+ if (size < 0) goto bad;
+#endif
+ if (likely(size == count)) {
+ partial_name = name;
+ } else {
+ slice = PySequence_GetSlice(parts_tuple, 0, count);
+ if (unlikely(!slice))
+ goto bad;
+ sep = PyUnicode_FromStringAndSize(".", 1);
+ if (unlikely(!sep))
+ goto bad;
+ partial_name = PyUnicode_Join(sep, slice);
+ }
+ PyErr_Format(
+ PyExc_ModuleNotFoundError,
+ "No module named '%U'", partial_name);
+bad:
+ Py_XDECREF(sep);
+ Py_XDECREF(slice);
+ Py_XDECREF(partial_name);
+ return NULL;
+}
+static PyObject *__Pyx__ImportDottedModule_Lookup(PyObject *name) {
+ PyObject *imported_module;
+#if (CYTHON_COMPILING_IN_PYPY && PYPY_VERSION_NUM < 0x07030400) ||\
+ CYTHON_COMPILING_IN_GRAAL
+ PyObject *modules = PyImport_GetModuleDict();
+ if (unlikely(!modules))
+ return NULL;
+ imported_module = __Pyx_PyDict_GetItemStr(modules, name);
+ Py_XINCREF(imported_module);
+#else
+ imported_module = PyImport_GetModule(name);
+#endif
+ return imported_module;
+}
+static PyObject *__Pyx_ImportDottedModule_WalkParts(PyObject *module, PyObject *name, PyObject *parts_tuple) {
+ Py_ssize_t i, nparts;
+#if CYTHON_ASSUME_SAFE_SIZE
+ nparts = PyTuple_GET_SIZE(parts_tuple);
+#else
+ nparts = PyTuple_Size(parts_tuple);
+ if (nparts < 0) return NULL;
+#endif
+ for (i=1; i < nparts && module; i++) {
+ PyObject *part, *submodule;
+#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS
+ part = PyTuple_GET_ITEM(parts_tuple, i);
+#else
+ part = __Pyx_PySequence_ITEM(parts_tuple, i);
+ if (!part) return NULL;
+#endif
+ submodule = __Pyx_PyObject_GetAttrStrNoError(module, part);
+#if !(CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS)
+ Py_DECREF(part);
+#endif
+ Py_DECREF(module);
+ module = submodule;
+ }
+ if (unlikely(!module)) {
+ return __Pyx__ImportDottedModule_Error(name, parts_tuple, i);
+ }
+ return module;
+}
+static PyObject *__Pyx__ImportDottedModule(PyObject *name, PyObject *parts_tuple) {
+ PyObject *imported_module;
+ PyObject *module = __Pyx_Import(name, NULL, 0);
+ if (!parts_tuple || unlikely(!module))
+ return module;
+ imported_module = __Pyx__ImportDottedModule_Lookup(name);
+ if (likely(imported_module)) {
+ Py_DECREF(module);
+ return imported_module;
+ }
+ PyErr_Clear();
+ return __Pyx_ImportDottedModule_WalkParts(module, name, parts_tuple);
+}
+static PyObject *__Pyx_ImportDottedModule(PyObject *name, PyObject *parts_tuple) {
+#if CYTHON_COMPILING_IN_CPYTHON
+ PyObject *module = __Pyx__ImportDottedModule_Lookup(name);
+ if (likely(module)) {
+ PyObject *spec = __Pyx_PyObject_GetAttrStrNoError(module, __pyx_mstate_global->__pyx_n_u_spec);
+ if (likely(spec)) {
+ PyObject *unsafe = __Pyx_PyObject_GetAttrStrNoError(spec, __pyx_mstate_global->__pyx_n_u_initializing);
+ if (likely(!unsafe || !__Pyx_PyObject_IsTrue(unsafe))) {
+ Py_DECREF(spec);
+ spec = NULL;
+ }
+ Py_XDECREF(unsafe);
+ }
+ if (likely(!spec)) {
+ PyErr_Clear();
+ return module;
+ }
+ Py_DECREF(spec);
+ Py_DECREF(module);
+ } else if (PyErr_Occurred()) {
+ PyErr_Clear();
+ }
+#endif
+ return __Pyx__ImportDottedModule(name, parts_tuple);
+}
+
+/* ListPack */
+static PyObject *__Pyx_PyList_Pack(Py_ssize_t n, ...) {
+ va_list va;
+ PyObject *l = PyList_New(n);
+ va_start(va, n);
+ if (unlikely(!l)) goto end;
+ for (Py_ssize_t i=0; i__pyx_kp_u_);
+ if (unlikely(!module_dot)) { goto modbad; }
+ full_name = PyUnicode_Concat(module_dot, name);
+ if (unlikely(!full_name)) { goto modbad; }
+ #if (CYTHON_COMPILING_IN_PYPY && PYPY_VERSION_NUM < 0x07030400) ||\
+ CYTHON_COMPILING_IN_GRAAL
+ {
+ PyObject *modules = PyImport_GetModuleDict();
+ if (unlikely(!modules))
+ goto modbad;
+ value = PyObject_GetItem(modules, full_name);
+ }
+ #else
+ value = PyImport_GetModule(full_name);
+ #endif
+ modbad:
+ Py_XDECREF(full_name);
+ Py_XDECREF(module_dot);
+ Py_XDECREF(module_name);
+ }
+ if (unlikely(!value)) {
+ PyErr_Format(PyExc_ImportError, "cannot import name %S", name);
+ }
+ return value;
+}
+
+/* pybytes_as_double */
+static double __Pyx_SlowPyString_AsDouble(PyObject *obj) {
+ PyObject *float_value = PyFloat_FromString(obj);
+ if (likely(float_value)) {
+ double value = __Pyx_PyFloat_AS_DOUBLE(float_value);
+ Py_DECREF(float_value);
+ return value;
+ }
+ return (double)-1;
+}
+static const char* __Pyx__PyBytes_AsDouble_Copy(const char* start, char* buffer, Py_ssize_t length) {
+ int last_was_punctuation = 1;
+ int parse_error_found = 0;
+ Py_ssize_t i;
+ for (i=0; i < length; i++) {
+ char chr = start[i];
+ int is_punctuation = (chr == '_') | (chr == '.') | (chr == 'e') | (chr == 'E');
+ *buffer = chr;
+ buffer += (chr != '_');
+ parse_error_found |= last_was_punctuation & is_punctuation;
+ last_was_punctuation = is_punctuation;
+ }
+ parse_error_found |= last_was_punctuation;
+ *buffer = '\0';
+ return unlikely(parse_error_found) ? NULL : buffer;
+}
+static double __Pyx__PyBytes_AsDouble_inf_nan(const char* start, Py_ssize_t length) {
+ int matches = 1;
+ char sign = start[0];
+ int is_signed = (sign == '+') | (sign == '-');
+ start += is_signed;
+ length -= is_signed;
+ switch (start[0]) {
+ #ifdef Py_NAN
+ case 'n':
+ case 'N':
+ if (unlikely(length != 3)) goto parse_failure;
+ matches &= (start[1] == 'a' || start[1] == 'A');
+ matches &= (start[2] == 'n' || start[2] == 'N');
+ if (unlikely(!matches)) goto parse_failure;
+ return (sign == '-') ? -Py_NAN : Py_NAN;
+ #endif
+ case 'i':
+ case 'I':
+ if (unlikely(length < 3)) goto parse_failure;
+ matches &= (start[1] == 'n' || start[1] == 'N');
+ matches &= (start[2] == 'f' || start[2] == 'F');
+ if (likely(length == 3 && matches))
+ return (sign == '-') ? -Py_HUGE_VAL : Py_HUGE_VAL;
+ if (unlikely(length != 8)) goto parse_failure;
+ matches &= (start[3] == 'i' || start[3] == 'I');
+ matches &= (start[4] == 'n' || start[4] == 'N');
+ matches &= (start[5] == 'i' || start[5] == 'I');
+ matches &= (start[6] == 't' || start[6] == 'T');
+ matches &= (start[7] == 'y' || start[7] == 'Y');
+ if (unlikely(!matches)) goto parse_failure;
+ return (sign == '-') ? -Py_HUGE_VAL : Py_HUGE_VAL;
+ case '.': case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9':
+ break;
+ default:
+ goto parse_failure;
+ }
+ return 0.0;
+parse_failure:
+ return -1.0;
+}
+static CYTHON_INLINE int __Pyx__PyBytes_AsDouble_IsSpace(char ch) {
+ return (ch == 0x20) | !((ch < 0x9) | (ch > 0xd));
+}
+CYTHON_UNUSED static double __Pyx__PyBytes_AsDouble(PyObject *obj, const char* start, Py_ssize_t length) {
+ double value;
+ Py_ssize_t i, digits;
+ const char *last = start + length;
+ char *end;
+ while (__Pyx__PyBytes_AsDouble_IsSpace(*start))
+ start++;
+ while (start < last - 1 && __Pyx__PyBytes_AsDouble_IsSpace(last[-1]))
+ last--;
+ length = last - start;
+ if (unlikely(length <= 0)) goto fallback;
+ value = __Pyx__PyBytes_AsDouble_inf_nan(start, length);
+ if (unlikely(value == -1.0)) goto fallback;
+ if (value != 0.0) return value;
+ digits = 0;
+ for (i=0; i < length; digits += start[i++] != '_');
+ if (likely(digits == length)) {
+ value = PyOS_string_to_double(start, &end, NULL);
+ } else if (digits < 40) {
+ char number[40];
+ last = __Pyx__PyBytes_AsDouble_Copy(start, number, length);
+ if (unlikely(!last)) goto fallback;
+ value = PyOS_string_to_double(number, &end, NULL);
+ } else {
+ char *number = (char*) PyMem_Malloc((digits + 1) * sizeof(char));
+ if (unlikely(!number)) goto fallback;
+ last = __Pyx__PyBytes_AsDouble_Copy(start, number, length);
+ if (unlikely(!last)) {
+ PyMem_Free(number);
+ goto fallback;
+ }
+ value = PyOS_string_to_double(number, &end, NULL);
+ PyMem_Free(number);
+ }
+ if (likely(end == last) || (value == (double)-1 && PyErr_Occurred())) {
+ return value;
+ }
+fallback:
+ return __Pyx_SlowPyString_AsDouble(obj);
+}
+
+/* FetchSharedCythonModule */
+static PyObject *__Pyx_FetchSharedCythonABIModule(void) {
+ return __Pyx_PyImport_AddModuleRef(__PYX_ABI_MODULE_NAME);
+}
+
+/* dict_setdefault */
+static CYTHON_INLINE PyObject *__Pyx_PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *default_value,
+ int is_safe_type) {
+ PyObject* value;
+ CYTHON_MAYBE_UNUSED_VAR(is_safe_type);
+#if CYTHON_COMPILING_IN_LIMITED_API
+ value = PyObject_CallMethod(d, "setdefault", "OO", key, default_value);
+#elif PY_VERSION_HEX >= 0x030d0000
+ PyDict_SetDefaultRef(d, key, default_value, &value);
+#else
+ value = PyDict_SetDefault(d, key, default_value);
+ if (unlikely(!value)) return NULL;
+ Py_INCREF(value);
+#endif
+ return value;
+}
+
+/* FetchCommonType */
+#if __PYX_LIMITED_VERSION_HEX < 0x030C0000
+static PyObject* __Pyx_PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, PyType_Spec *spec, PyObject *bases) {
+ PyObject *result = __Pyx_PyType_FromModuleAndSpec(module, spec, bases);
+ if (result && metaclass) {
+ PyObject *old_tp = (PyObject*)Py_TYPE(result);
+ Py_INCREF((PyObject*)metaclass);
+#if __PYX_LIMITED_VERSION_HEX >= 0x03090000
+ Py_SET_TYPE(result, metaclass);
+#else
+ result->ob_type = metaclass;
+#endif
+ Py_DECREF(old_tp);
+ }
+ return result;
+}
+#else
+#define __Pyx_PyType_FromMetaclass(me, mo, s, b) PyType_FromMetaclass(me, mo, s, b)
+#endif
+static int __Pyx_VerifyCachedType(PyObject *cached_type,
+ const char *name,
+ Py_ssize_t expected_basicsize) {
+ Py_ssize_t basicsize;
+ if (!PyType_Check(cached_type)) {
+ PyErr_Format(PyExc_TypeError,
+ "Shared Cython type %.200s is not a type object", name);
+ return -1;
+ }
+ if (expected_basicsize == 0) {
+ return 0; // size is inherited, nothing useful to check
+ }
+#if CYTHON_COMPILING_IN_LIMITED_API
+ PyObject *py_basicsize;
+ py_basicsize = PyObject_GetAttrString(cached_type, "__basicsize__");
+ if (unlikely(!py_basicsize)) return -1;
+ basicsize = PyLong_AsSsize_t(py_basicsize);
+ Py_DECREF(py_basicsize);
+ py_basicsize = NULL;
+ if (unlikely(basicsize == (Py_ssize_t)-1) && PyErr_Occurred()) return -1;
+#else
+ basicsize = ((PyTypeObject*) cached_type)->tp_basicsize;
+#endif
+ if (basicsize != expected_basicsize) {
+ PyErr_Format(PyExc_TypeError,
+ "Shared Cython type %.200s has the wrong size, try recompiling",
+ name);
+ return -1;
+ }
+ return 0;
+}
+static PyTypeObject *__Pyx_FetchCommonTypeFromSpec(PyTypeObject *metaclass, PyObject *module, PyType_Spec *spec, PyObject *bases) {
+ PyObject *abi_module = NULL, *cached_type = NULL, *abi_module_dict, *new_cached_type, *py_object_name;
+ int get_item_ref_result;
+ const char* object_name = strrchr(spec->name, '.');
+ object_name = object_name ? object_name+1 : spec->name;
+ py_object_name = PyUnicode_FromString(object_name);
+ if (!py_object_name) return NULL;
+ abi_module = __Pyx_FetchSharedCythonABIModule();
+ if (!abi_module) goto done;
+ abi_module_dict = PyModule_GetDict(abi_module);
+ if (!abi_module_dict) goto done;
+ get_item_ref_result = __Pyx_PyDict_GetItemRef(abi_module_dict, py_object_name, &cached_type);
+ if (get_item_ref_result == 1) {
+ if (__Pyx_VerifyCachedType(
+ cached_type,
+ object_name,
+ spec->basicsize) < 0) {
+ goto bad;
+ }
+ goto done;
+ } else if (unlikely(get_item_ref_result == -1)) {
+ goto bad;
+ }
+ CYTHON_UNUSED_VAR(module);
+ cached_type = __Pyx_PyType_FromMetaclass(metaclass, abi_module, spec, bases);
+ if (unlikely(!cached_type)) goto bad;
+ if (unlikely(__Pyx_fix_up_extension_type_from_spec(spec, (PyTypeObject *) cached_type) < 0)) goto bad;
+ new_cached_type = __Pyx_PyDict_SetDefault(abi_module_dict, py_object_name, cached_type, 1);
+ if (unlikely(new_cached_type != cached_type)) {
+ if (unlikely(!new_cached_type)) goto bad;
+ Py_DECREF(cached_type);
+ cached_type = new_cached_type;
+ if (__Pyx_VerifyCachedType(
+ cached_type,
+ object_name,
+ spec->basicsize) < 0) {
+ goto bad;
+ }
+ goto done;
+ } else {
+ Py_DECREF(new_cached_type);
+ }
+done:
+ Py_XDECREF(abi_module);
+ Py_DECREF(py_object_name);
+ assert(cached_type == NULL || PyType_Check(cached_type));
+ return (PyTypeObject *) cached_type;
+bad:
+ Py_XDECREF(cached_type);
+ cached_type = NULL;
+ goto done;
+}
+
+/* CommonTypesMetaclass */
+static PyObject* __pyx_CommonTypesMetaclass_get_module(CYTHON_UNUSED PyObject *self, CYTHON_UNUSED void* context) {
+ return PyUnicode_FromString(__PYX_ABI_MODULE_NAME);
+}
+static PyGetSetDef __pyx_CommonTypesMetaclass_getset[] = {
+ {"__module__", __pyx_CommonTypesMetaclass_get_module, NULL, NULL, NULL},
+ {0, 0, 0, 0, 0}
+};
+static PyType_Slot __pyx_CommonTypesMetaclass_slots[] = {
+ {Py_tp_getset, (void *)__pyx_CommonTypesMetaclass_getset},
+ {0, 0}
+};
+static PyType_Spec __pyx_CommonTypesMetaclass_spec = {
+ __PYX_TYPE_MODULE_PREFIX "_common_types_metatype",
+ 0,
+ 0,
+#if PY_VERSION_HEX >= 0x030A0000
+ Py_TPFLAGS_IMMUTABLETYPE |
+ Py_TPFLAGS_DISALLOW_INSTANTIATION |
+#endif
+ Py_TPFLAGS_DEFAULT,
+ __pyx_CommonTypesMetaclass_slots
+};
+static int __pyx_CommonTypesMetaclass_init(PyObject *module) {
+ __pyx_mstatetype *mstate = __Pyx_PyModule_GetState(module);
+ PyObject *bases = PyTuple_Pack(1, &PyType_Type);
+ if (unlikely(!bases)) {
+ return -1;
+ }
+ mstate->__pyx_CommonTypesMetaclassType = __Pyx_FetchCommonTypeFromSpec(NULL, module, &__pyx_CommonTypesMetaclass_spec, bases);
+ Py_DECREF(bases);
+ if (unlikely(mstate->__pyx_CommonTypesMetaclassType == NULL)) {
+ return -1;
+ }
+ return 0;
+}
+
+/* CallTypeTraverse */
+#if !CYTHON_USE_TYPE_SPECS || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x03090000)
+#else
+static int __Pyx_call_type_traverse(PyObject *o, int always_call, visitproc visit, void *arg) {
+ #if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x03090000
+ if (__Pyx_get_runtime_version() < 0x03090000) return 0;
+ #endif
+ if (!always_call) {
+ PyTypeObject *base = __Pyx_PyObject_GetSlot(o, tp_base, PyTypeObject*);
+ unsigned long flags = PyType_GetFlags(base);
+ if (flags & Py_TPFLAGS_HEAPTYPE) {
+ return 0;
+ }
+ }
+ Py_VISIT((PyObject*)Py_TYPE(o));
+ return 0;
+}
+#endif
+
+/* PyMethodNew */
+#if CYTHON_COMPILING_IN_LIMITED_API
+static PyObject *__Pyx_PyMethod_New(PyObject *func, PyObject *self, PyObject *typ) {
+ PyObject *result;
+ CYTHON_UNUSED_VAR(typ);
+ if (!self)
+ return __Pyx_NewRef(func);
+ #if __PYX_LIMITED_VERSION_HEX >= 0x030C0000
+ {
+ PyObject *args[] = {func, self};
+ result = PyObject_Vectorcall(__pyx_mstate_global->__Pyx_CachedMethodType, args, 2, NULL);
+ }
+ #else
+ result = PyObject_CallFunctionObjArgs(__pyx_mstate_global->__Pyx_CachedMethodType, func, self, NULL);
+ #endif
+ return result;
+}
+#else
+static PyObject *__Pyx_PyMethod_New(PyObject *func, PyObject *self, PyObject *typ) {
+ CYTHON_UNUSED_VAR(typ);
+ if (!self)
+ return __Pyx_NewRef(func);
+ return PyMethod_New(func, self);
+}
+#endif
+
+/* PyVectorcallFastCallDict */
+#if CYTHON_METH_FASTCALL && (CYTHON_VECTORCALL || CYTHON_BACKPORT_VECTORCALL)
+static PyObject *__Pyx_PyVectorcall_FastCallDict_kw(PyObject *func, __pyx_vectorcallfunc vc, PyObject *const *args, size_t nargs, PyObject *kw)
+{
+ PyObject *res = NULL;
+ PyObject *kwnames;
+ PyObject **newargs;
+ PyObject **kwvalues;
+ Py_ssize_t i, pos;
+ size_t j;
+ PyObject *key, *value;
+ unsigned long keys_are_strings;
+ #if !CYTHON_ASSUME_SAFE_SIZE
+ Py_ssize_t nkw = PyDict_Size(kw);
+ if (unlikely(nkw == -1)) return NULL;
+ #else
+ Py_ssize_t nkw = PyDict_GET_SIZE(kw);
+ #endif
+ newargs = (PyObject **)PyMem_Malloc((nargs + (size_t)nkw) * sizeof(args[0]));
+ if (unlikely(newargs == NULL)) {
+ PyErr_NoMemory();
+ return NULL;
+ }
+ for (j = 0; j < nargs; j++) newargs[j] = args[j];
+ kwnames = PyTuple_New(nkw);
+ if (unlikely(kwnames == NULL)) {
+ PyMem_Free(newargs);
+ return NULL;
+ }
+ kwvalues = newargs + nargs;
+ pos = i = 0;
+ keys_are_strings = Py_TPFLAGS_UNICODE_SUBCLASS;
+ while (PyDict_Next(kw, &pos, &key, &value)) {
+ keys_are_strings &=
+ #if CYTHON_COMPILING_IN_LIMITED_API
+ PyType_GetFlags(Py_TYPE(key));
+ #else
+ Py_TYPE(key)->tp_flags;
+ #endif
+ Py_INCREF(key);
+ Py_INCREF(value);
+ #if !CYTHON_ASSUME_SAFE_MACROS
+ if (unlikely(PyTuple_SetItem(kwnames, i, key) < 0)) goto cleanup;
+ #else
+ PyTuple_SET_ITEM(kwnames, i, key);
+ #endif
+ kwvalues[i] = value;
+ i++;
+ }
+ if (unlikely(!keys_are_strings)) {
+ PyErr_SetString(PyExc_TypeError, "keywords must be strings");
+ goto cleanup;
+ }
+ res = vc(func, newargs, nargs, kwnames);
+cleanup:
+ Py_DECREF(kwnames);
+ for (i = 0; i < nkw; i++)
+ Py_DECREF(kwvalues[i]);
+ PyMem_Free(newargs);
+ return res;
+}
+static CYTHON_INLINE PyObject *__Pyx_PyVectorcall_FastCallDict(PyObject *func, __pyx_vectorcallfunc vc, PyObject *const *args, size_t nargs, PyObject *kw)
+{
+ Py_ssize_t kw_size =
+ likely(kw == NULL) ?
+ 0 :
+#if !CYTHON_ASSUME_SAFE_SIZE
+ PyDict_Size(kw);
+#else
+ PyDict_GET_SIZE(kw);
+#endif
+ if (kw_size == 0) {
+ return vc(func, args, nargs, NULL);
+ }
+#if !CYTHON_ASSUME_SAFE_SIZE
+ else if (unlikely(kw_size == -1)) {
+ return NULL;
+ }
+#endif
+ return __Pyx_PyVectorcall_FastCallDict_kw(func, vc, args, nargs, kw);
+}
+#endif
+
+/* CythonFunctionShared */
+#if CYTHON_COMPILING_IN_LIMITED_API
+static CYTHON_INLINE int __Pyx__IsSameCyOrCFunctionNoMethod(PyObject *func, void (*cfunc)(void)) {
+ if (__Pyx_CyFunction_Check(func)) {
+ return PyCFunction_GetFunction(((__pyx_CyFunctionObject*)func)->func) == (PyCFunction) cfunc;
+ } else if (PyCFunction_Check(func)) {
+ return PyCFunction_GetFunction(func) == (PyCFunction) cfunc;
+ }
+ return 0;
+}
+static CYTHON_INLINE int __Pyx__IsSameCyOrCFunction(PyObject *func, void (*cfunc)(void)) {
+ if ((PyObject*)Py_TYPE(func) == __pyx_mstate_global->__Pyx_CachedMethodType) {
+ int result;
+ PyObject *newFunc = PyObject_GetAttr(func, __pyx_mstate_global->__pyx_n_u_func);
+ if (unlikely(!newFunc)) {
+ PyErr_Clear(); // It's only an optimization, so don't throw an error
+ return 0;
+ }
+ result = __Pyx__IsSameCyOrCFunctionNoMethod(newFunc, cfunc);
+ Py_DECREF(newFunc);
+ return result;
+ }
+ return __Pyx__IsSameCyOrCFunctionNoMethod(func, cfunc);
+}
+#else
+static CYTHON_INLINE int __Pyx__IsSameCyOrCFunction(PyObject *func, void (*cfunc)(void)) {
+ if (PyMethod_Check(func)) {
+ func = PyMethod_GET_FUNCTION(func);
+ }
+ return __Pyx_CyOrPyCFunction_Check(func) && __Pyx_CyOrPyCFunction_GET_FUNCTION(func) == (PyCFunction) cfunc;
+}
+#endif
+static CYTHON_INLINE void __Pyx__CyFunction_SetClassObj(__pyx_CyFunctionObject* f, PyObject* classobj) {
+#if PY_VERSION_HEX < 0x030900B1 || CYTHON_COMPILING_IN_LIMITED_API
+ __Pyx_Py_XDECREF_SET(
+ __Pyx_CyFunction_GetClassObj(f),
+ ((classobj) ? __Pyx_NewRef(classobj) : NULL));
+#else
+ __Pyx_Py_XDECREF_SET(
+ ((PyCMethodObject *) (f))->mm_class,
+ (PyTypeObject*)((classobj) ? __Pyx_NewRef(classobj) : NULL));
+#endif
+}
+static PyObject *
+__Pyx_CyFunction_get_doc_locked(__pyx_CyFunctionObject *op)
+{
+ if (unlikely(op->func_doc == NULL)) {
+#if CYTHON_COMPILING_IN_LIMITED_API
+ op->func_doc = PyObject_GetAttrString(op->func, "__doc__");
+ if (unlikely(!op->func_doc)) return NULL;
+#else
+ if (((PyCFunctionObject*)op)->m_ml->ml_doc) {
+ op->func_doc = PyUnicode_FromString(((PyCFunctionObject*)op)->m_ml->ml_doc);
+ if (unlikely(op->func_doc == NULL))
+ return NULL;
+ } else {
+ Py_INCREF(Py_None);
+ return Py_None;
+ }
+#endif
+ }
+ Py_INCREF(op->func_doc);
+ return op->func_doc;
+}
+static PyObject *
+__Pyx_CyFunction_get_doc(__pyx_CyFunctionObject *op, void *closure) {
+ PyObject *result;
+ CYTHON_UNUSED_VAR(closure);
+ __Pyx_BEGIN_CRITICAL_SECTION(op);
+ result = __Pyx_CyFunction_get_doc_locked(op);
+ __Pyx_END_CRITICAL_SECTION();
+ return result;
+}
+static int
+__Pyx_CyFunction_set_doc(__pyx_CyFunctionObject *op, PyObject *value, void *context)
+{
+ CYTHON_UNUSED_VAR(context);
+ if (value == NULL) {
+ value = Py_None;
+ }
+ Py_INCREF(value);
+ __Pyx_BEGIN_CRITICAL_SECTION(op);
+ __Pyx_Py_XDECREF_SET(op->func_doc, value);
+ __Pyx_END_CRITICAL_SECTION();
+ return 0;
+}
+static PyObject *
+__Pyx_CyFunction_get_name_locked(__pyx_CyFunctionObject *op)
+{
+ if (unlikely(op->func_name == NULL)) {
+#if CYTHON_COMPILING_IN_LIMITED_API
+ op->func_name = PyObject_GetAttrString(op->func, "__name__");
+#else
+ op->func_name = PyUnicode_InternFromString(((PyCFunctionObject*)op)->m_ml->ml_name);
+#endif
+ if (unlikely(op->func_name == NULL))
+ return NULL;
+ }
+ Py_INCREF(op->func_name);
+ return op->func_name;
+}
+static PyObject *
+__Pyx_CyFunction_get_name(__pyx_CyFunctionObject *op, void *context)
+{
+ PyObject *result = NULL;
+ CYTHON_UNUSED_VAR(context);
+ __Pyx_BEGIN_CRITICAL_SECTION(op);
+ result = __Pyx_CyFunction_get_name_locked(op);
+ __Pyx_END_CRITICAL_SECTION();
+ return result;
+}
+static int
+__Pyx_CyFunction_set_name(__pyx_CyFunctionObject *op, PyObject *value, void *context)
+{
+ CYTHON_UNUSED_VAR(context);
+ if (unlikely(value == NULL || !PyUnicode_Check(value))) {
+ PyErr_SetString(PyExc_TypeError,
+ "__name__ must be set to a string object");
+ return -1;
+ }
+ Py_INCREF(value);
+ __Pyx_BEGIN_CRITICAL_SECTION(op);
+ __Pyx_Py_XDECREF_SET(op->func_name, value);
+ __Pyx_END_CRITICAL_SECTION();
+ return 0;
+}
+static PyObject *
+__Pyx_CyFunction_get_qualname(__pyx_CyFunctionObject *op, void *context)
+{
+ CYTHON_UNUSED_VAR(context);
+ PyObject *result;
+ __Pyx_BEGIN_CRITICAL_SECTION(op);
+ Py_INCREF(op->func_qualname);
+ result = op->func_qualname;
+ __Pyx_END_CRITICAL_SECTION();
+ return result;
+}
+static int
+__Pyx_CyFunction_set_qualname(__pyx_CyFunctionObject *op, PyObject *value, void *context)
+{
+ CYTHON_UNUSED_VAR(context);
+ if (unlikely(value == NULL || !PyUnicode_Check(value))) {
+ PyErr_SetString(PyExc_TypeError,
+ "__qualname__ must be set to a string object");
+ return -1;
+ }
+ Py_INCREF(value);
+ __Pyx_BEGIN_CRITICAL_SECTION(op);
+ __Pyx_Py_XDECREF_SET(op->func_qualname, value);
+ __Pyx_END_CRITICAL_SECTION();
+ return 0;
+}
+static PyObject *
+__Pyx_CyFunction_get_dict_locked(__pyx_CyFunctionObject *op)
+{
+ if (unlikely(op->func_dict == NULL)) {
+ op->func_dict = PyDict_New();
+ if (unlikely(op->func_dict == NULL))
+ return NULL;
+ }
+ Py_INCREF(op->func_dict);
+ return op->func_dict;
+}
+static PyObject *
+__Pyx_CyFunction_get_dict(__pyx_CyFunctionObject *op, void *context)
+{
+ CYTHON_UNUSED_VAR(context);
+ PyObject *result;
+ __Pyx_BEGIN_CRITICAL_SECTION(op);
+ result = __Pyx_CyFunction_get_dict_locked(op);
+ __Pyx_END_CRITICAL_SECTION();
+ return result;
+}
+static int
+__Pyx_CyFunction_set_dict(__pyx_CyFunctionObject *op, PyObject *value, void *context)
+{
+ CYTHON_UNUSED_VAR(context);
+ if (unlikely(value == NULL)) {
+ PyErr_SetString(PyExc_TypeError,
+ "function's dictionary may not be deleted");
+ return -1;
+ }
+ if (unlikely(!PyDict_Check(value))) {
+ PyErr_SetString(PyExc_TypeError,
+ "setting function's dictionary to a non-dict");
+ return -1;
+ }
+ Py_INCREF(value);
+ __Pyx_BEGIN_CRITICAL_SECTION(op);
+ __Pyx_Py_XDECREF_SET(op->func_dict, value);
+ __Pyx_END_CRITICAL_SECTION();
+ return 0;
+}
+static PyObject *
+__Pyx_CyFunction_get_globals(__pyx_CyFunctionObject *op, void *context)
+{
+ CYTHON_UNUSED_VAR(context);
+ Py_INCREF(op->func_globals);
+ return op->func_globals;
+}
+static PyObject *
+__Pyx_CyFunction_get_closure(__pyx_CyFunctionObject *op, void *context)
+{
+ CYTHON_UNUSED_VAR(op);
+ CYTHON_UNUSED_VAR(context);
+ Py_INCREF(Py_None);
+ return Py_None;
+}
+static PyObject *
+__Pyx_CyFunction_get_code(__pyx_CyFunctionObject *op, void *context)
+{
+ PyObject* result = (op->func_code) ? op->func_code : Py_None;
+ CYTHON_UNUSED_VAR(context);
+ Py_INCREF(result);
+ return result;
+}
+static int
+__Pyx_CyFunction_init_defaults(__pyx_CyFunctionObject *op) {
+ int result = 0;
+ PyObject *res = op->defaults_getter((PyObject *) op);
+ if (unlikely(!res))
+ return -1;
+ #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS
+ op->defaults_tuple = PyTuple_GET_ITEM(res, 0);
+ Py_INCREF(op->defaults_tuple);
+ op->defaults_kwdict = PyTuple_GET_ITEM(res, 1);
+ Py_INCREF(op->defaults_kwdict);
+ #else
+ op->defaults_tuple = __Pyx_PySequence_ITEM(res, 0);
+ if (unlikely(!op->defaults_tuple)) result = -1;
+ else {
+ op->defaults_kwdict = __Pyx_PySequence_ITEM(res, 1);
+ if (unlikely(!op->defaults_kwdict)) result = -1;
+ }
+ #endif
+ Py_DECREF(res);
+ return result;
+}
+static int
+__Pyx_CyFunction_set_defaults(__pyx_CyFunctionObject *op, PyObject* value, void *context) {
+ CYTHON_UNUSED_VAR(context);
+ if (!value) {
+ value = Py_None;
+ } else if (unlikely(value != Py_None && !PyTuple_Check(value))) {
+ PyErr_SetString(PyExc_TypeError,
+ "__defaults__ must be set to a tuple object");
+ return -1;
+ }
+ PyErr_WarnEx(PyExc_RuntimeWarning, "changes to cyfunction.__defaults__ will not "
+ "currently affect the values used in function calls", 1);
+ Py_INCREF(value);
+ __Pyx_BEGIN_CRITICAL_SECTION(op);
+ __Pyx_Py_XDECREF_SET(op->defaults_tuple, value);
+ __Pyx_END_CRITICAL_SECTION();
+ return 0;
+}
+static PyObject *
+__Pyx_CyFunction_get_defaults_locked(__pyx_CyFunctionObject *op) {
+ PyObject* result = op->defaults_tuple;
+ if (unlikely(!result)) {
+ if (op->defaults_getter) {
+ if (unlikely(__Pyx_CyFunction_init_defaults(op) < 0)) return NULL;
+ result = op->defaults_tuple;
+ } else {
+ result = Py_None;
+ }
+ }
+ Py_INCREF(result);
+ return result;
+}
+static PyObject *
+__Pyx_CyFunction_get_defaults(__pyx_CyFunctionObject *op, void *context) {
+ PyObject* result = NULL;
+ CYTHON_UNUSED_VAR(context);
+ __Pyx_BEGIN_CRITICAL_SECTION(op);
+ result = __Pyx_CyFunction_get_defaults_locked(op);
+ __Pyx_END_CRITICAL_SECTION();
+ return result;
+}
+static int
+__Pyx_CyFunction_set_kwdefaults(__pyx_CyFunctionObject *op, PyObject* value, void *context) {
+ CYTHON_UNUSED_VAR(context);
+ if (!value) {
+ value = Py_None;
+ } else if (unlikely(value != Py_None && !PyDict_Check(value))) {
+ PyErr_SetString(PyExc_TypeError,
+ "__kwdefaults__ must be set to a dict object");
+ return -1;
+ }
+ PyErr_WarnEx(PyExc_RuntimeWarning, "changes to cyfunction.__kwdefaults__ will not "
+ "currently affect the values used in function calls", 1);
+ Py_INCREF(value);
+ __Pyx_BEGIN_CRITICAL_SECTION(op);
+ __Pyx_Py_XDECREF_SET(op->defaults_kwdict, value);
+ __Pyx_END_CRITICAL_SECTION();
+ return 0;
+}
+static PyObject *
+__Pyx_CyFunction_get_kwdefaults_locked(__pyx_CyFunctionObject *op) {
+ PyObject* result = op->defaults_kwdict;
+ if (unlikely(!result)) {
+ if (op->defaults_getter) {
+ if (unlikely(__Pyx_CyFunction_init_defaults(op) < 0)) return NULL;
+ result = op->defaults_kwdict;
+ } else {
+ result = Py_None;
+ }
+ }
+ Py_INCREF(result);
+ return result;
+}
+static PyObject *
+__Pyx_CyFunction_get_kwdefaults(__pyx_CyFunctionObject *op, void *context) {
+ PyObject* result;
+ CYTHON_UNUSED_VAR(context);
+ __Pyx_BEGIN_CRITICAL_SECTION(op);
+ result = __Pyx_CyFunction_get_kwdefaults_locked(op);
+ __Pyx_END_CRITICAL_SECTION();
+ return result;
+}
+static int
+__Pyx_CyFunction_set_annotations(__pyx_CyFunctionObject *op, PyObject* value, void *context) {
+ CYTHON_UNUSED_VAR(context);
+ if (!value || value == Py_None) {
+ value = NULL;
+ } else if (unlikely(!PyDict_Check(value))) {
+ PyErr_SetString(PyExc_TypeError,
+ "__annotations__ must be set to a dict object");
+ return -1;
+ }
+ Py_XINCREF(value);
+ __Pyx_BEGIN_CRITICAL_SECTION(op);
+ __Pyx_Py_XDECREF_SET(op->func_annotations, value);
+ __Pyx_END_CRITICAL_SECTION();
+ return 0;
+}
+static PyObject *
+__Pyx_CyFunction_get_annotations_locked(__pyx_CyFunctionObject *op) {
+ PyObject* result = op->func_annotations;
+ if (unlikely(!result)) {
+ result = PyDict_New();
+ if (unlikely(!result)) return NULL;
+ op->func_annotations = result;
+ }
+ Py_INCREF(result);
+ return result;
+}
+static PyObject *
+__Pyx_CyFunction_get_annotations(__pyx_CyFunctionObject *op, void *context) {
+ PyObject *result;
+ CYTHON_UNUSED_VAR(context);
+ __Pyx_BEGIN_CRITICAL_SECTION(op);
+ result = __Pyx_CyFunction_get_annotations_locked(op);
+ __Pyx_END_CRITICAL_SECTION();
+ return result;
+}
+static PyObject *
+__Pyx_CyFunction_get_is_coroutine_value(__pyx_CyFunctionObject *op) {
+ int is_coroutine = op->flags & __Pyx_CYFUNCTION_COROUTINE;
+ if (is_coroutine) {
+ PyObject *is_coroutine_value, *module, *fromlist, *marker = __pyx_mstate_global->__pyx_n_u_is_coroutine;
+ fromlist = PyList_New(1);
+ if (unlikely(!fromlist)) return NULL;
+ Py_INCREF(marker);
+#if CYTHON_ASSUME_SAFE_MACROS
+ PyList_SET_ITEM(fromlist, 0, marker);
+#else
+ if (unlikely(PyList_SetItem(fromlist, 0, marker) < 0)) {
+ Py_DECREF(marker);
+ Py_DECREF(fromlist);
+ return NULL;
+ }
+#endif
+ module = PyImport_ImportModuleLevelObject(__pyx_mstate_global->__pyx_n_u_asyncio_coroutines, NULL, NULL, fromlist, 0);
+ Py_DECREF(fromlist);
+ if (unlikely(!module)) goto ignore;
+ is_coroutine_value = __Pyx_PyObject_GetAttrStr(module, marker);
+ Py_DECREF(module);
+ if (likely(is_coroutine_value)) {
+ return is_coroutine_value;
+ }
+ignore:
+ PyErr_Clear();
+ }
+ return __Pyx_PyBool_FromLong(is_coroutine);
+}
+static PyObject *
+__Pyx_CyFunction_get_is_coroutine(__pyx_CyFunctionObject *op, void *context) {
+ PyObject *result;
+ CYTHON_UNUSED_VAR(context);
+ if (op->func_is_coroutine) {
+ return __Pyx_NewRef(op->func_is_coroutine);
+ }
+ result = __Pyx_CyFunction_get_is_coroutine_value(op);
+ if (unlikely(!result))
+ return NULL;
+ __Pyx_BEGIN_CRITICAL_SECTION(op);
+ if (op->func_is_coroutine) {
+ Py_DECREF(result);
+ result = __Pyx_NewRef(op->func_is_coroutine);
+ } else {
+ op->func_is_coroutine = __Pyx_NewRef(result);
+ }
+ __Pyx_END_CRITICAL_SECTION();
+ return result;
+}
+static void __Pyx_CyFunction_raise_argument_count_error(__pyx_CyFunctionObject *func, const char* message, Py_ssize_t size) {
+#if CYTHON_COMPILING_IN_LIMITED_API
+ PyObject *py_name = __Pyx_CyFunction_get_name(func, NULL);
+ if (!py_name) return;
+ PyErr_Format(PyExc_TypeError,
+ "%.200S() %s (%" CYTHON_FORMAT_SSIZE_T "d given)",
+ py_name, message, size);
+ Py_DECREF(py_name);
+#else
+ const char* name = ((PyCFunctionObject*)func)->m_ml->ml_name;
+ PyErr_Format(PyExc_TypeError,
+ "%.200s() %s (%" CYTHON_FORMAT_SSIZE_T "d given)",
+ name, message, size);
+#endif
+}
+static void __Pyx_CyFunction_raise_type_error(__pyx_CyFunctionObject *func, const char* message) {
+#if CYTHON_COMPILING_IN_LIMITED_API
+ PyObject *py_name = __Pyx_CyFunction_get_name(func, NULL);
+ if (!py_name) return;
+ PyErr_Format(PyExc_TypeError,
+ "%.200S() %s",
+ py_name, message);
+ Py_DECREF(py_name);
+#else
+ const char* name = ((PyCFunctionObject*)func)->m_ml->ml_name;
+ PyErr_Format(PyExc_TypeError,
+ "%.200s() %s",
+ name, message);
+#endif
+}
+#if CYTHON_COMPILING_IN_LIMITED_API
+static PyObject *
+__Pyx_CyFunction_get_module(__pyx_CyFunctionObject *op, void *context) {
+ CYTHON_UNUSED_VAR(context);
+ return PyObject_GetAttrString(op->func, "__module__");
+}
+static int
+__Pyx_CyFunction_set_module(__pyx_CyFunctionObject *op, PyObject* value, void *context) {
+ CYTHON_UNUSED_VAR(context);
+ return PyObject_SetAttrString(op->func, "__module__", value);
+}
+#endif
+static PyGetSetDef __pyx_CyFunction_getsets[] = {
+ {"func_doc", (getter)__Pyx_CyFunction_get_doc, (setter)__Pyx_CyFunction_set_doc, 0, 0},
+ {"__doc__", (getter)__Pyx_CyFunction_get_doc, (setter)__Pyx_CyFunction_set_doc, 0, 0},
+ {"func_name", (getter)__Pyx_CyFunction_get_name, (setter)__Pyx_CyFunction_set_name, 0, 0},
+ {"__name__", (getter)__Pyx_CyFunction_get_name, (setter)__Pyx_CyFunction_set_name, 0, 0},
+ {"__qualname__", (getter)__Pyx_CyFunction_get_qualname, (setter)__Pyx_CyFunction_set_qualname, 0, 0},
+ {"func_dict", (getter)__Pyx_CyFunction_get_dict, (setter)__Pyx_CyFunction_set_dict, 0, 0},
+ {"__dict__", (getter)__Pyx_CyFunction_get_dict, (setter)__Pyx_CyFunction_set_dict, 0, 0},
+ {"func_globals", (getter)__Pyx_CyFunction_get_globals, 0, 0, 0},
+ {"__globals__", (getter)__Pyx_CyFunction_get_globals, 0, 0, 0},
+ {"func_closure", (getter)__Pyx_CyFunction_get_closure, 0, 0, 0},
+ {"__closure__", (getter)__Pyx_CyFunction_get_closure, 0, 0, 0},
+ {"func_code", (getter)__Pyx_CyFunction_get_code, 0, 0, 0},
+ {"__code__", (getter)__Pyx_CyFunction_get_code, 0, 0, 0},
+ {"func_defaults", (getter)__Pyx_CyFunction_get_defaults, (setter)__Pyx_CyFunction_set_defaults, 0, 0},
+ {"__defaults__", (getter)__Pyx_CyFunction_get_defaults, (setter)__Pyx_CyFunction_set_defaults, 0, 0},
+ {"__kwdefaults__", (getter)__Pyx_CyFunction_get_kwdefaults, (setter)__Pyx_CyFunction_set_kwdefaults, 0, 0},
+ {"__annotations__", (getter)__Pyx_CyFunction_get_annotations, (setter)__Pyx_CyFunction_set_annotations, 0, 0},
+ {"_is_coroutine", (getter)__Pyx_CyFunction_get_is_coroutine, 0, 0, 0},
+#if CYTHON_COMPILING_IN_LIMITED_API
+ {"__module__", (getter)__Pyx_CyFunction_get_module, (setter)__Pyx_CyFunction_set_module, 0, 0},
+#endif
+ {0, 0, 0, 0, 0}
+};
+static PyMemberDef __pyx_CyFunction_members[] = {
+#if !CYTHON_COMPILING_IN_LIMITED_API
+ {"__module__", T_OBJECT, offsetof(PyCFunctionObject, m_module), 0, 0},
+#endif
+ {"__dictoffset__", T_PYSSIZET, offsetof(__pyx_CyFunctionObject, func_dict), READONLY, 0},
+#if CYTHON_METH_FASTCALL
+#if CYTHON_BACKPORT_VECTORCALL || CYTHON_COMPILING_IN_LIMITED_API
+ {"__vectorcalloffset__", T_PYSSIZET, offsetof(__pyx_CyFunctionObject, func_vectorcall), READONLY, 0},
+#else
+ {"__vectorcalloffset__", T_PYSSIZET, offsetof(PyCFunctionObject, vectorcall), READONLY, 0},
+#endif
+#if CYTHON_COMPILING_IN_LIMITED_API
+ {"__weaklistoffset__", T_PYSSIZET, offsetof(__pyx_CyFunctionObject, func_weakreflist), READONLY, 0},
+#else
+ {"__weaklistoffset__", T_PYSSIZET, offsetof(PyCFunctionObject, m_weakreflist), READONLY, 0},
+#endif
+#endif
+ {0, 0, 0, 0, 0}
+};
+static PyObject *
+__Pyx_CyFunction_reduce(__pyx_CyFunctionObject *m, PyObject *args)
+{
+ PyObject *result = NULL;
+ CYTHON_UNUSED_VAR(args);
+ __Pyx_BEGIN_CRITICAL_SECTION(m);
+ Py_INCREF(m->func_qualname);
+ result = m->func_qualname;
+ __Pyx_END_CRITICAL_SECTION();
+ return result;
+}
+static PyMethodDef __pyx_CyFunction_methods[] = {
+ {"__reduce__", (PyCFunction)__Pyx_CyFunction_reduce, METH_VARARGS, 0},
+ {0, 0, 0, 0}
+};
+#if CYTHON_COMPILING_IN_LIMITED_API
+#define __Pyx_CyFunction_weakreflist(cyfunc) ((cyfunc)->func_weakreflist)
+#else
+#define __Pyx_CyFunction_weakreflist(cyfunc) (((PyCFunctionObject*)cyfunc)->m_weakreflist)
+#endif
+static PyObject *__Pyx_CyFunction_Init(__pyx_CyFunctionObject *op, PyMethodDef *ml, int flags, PyObject* qualname,
+ PyObject *closure, PyObject *module, PyObject* globals, PyObject* code) {
+#if !CYTHON_COMPILING_IN_LIMITED_API
+ PyCFunctionObject *cf = (PyCFunctionObject*) op;
+#endif
+ if (unlikely(op == NULL))
+ return NULL;
+#if CYTHON_COMPILING_IN_LIMITED_API
+ op->func = PyCFunction_NewEx(ml, (PyObject*)op, module);
+ if (unlikely(!op->func)) return NULL;
+#endif
+ op->flags = flags;
+ __Pyx_CyFunction_weakreflist(op) = NULL;
+#if !CYTHON_COMPILING_IN_LIMITED_API
+ cf->m_ml = ml;
+ cf->m_self = (PyObject *) op;
+#endif
+ Py_XINCREF(closure);
+ op->func_closure = closure;
+#if !CYTHON_COMPILING_IN_LIMITED_API
+ Py_XINCREF(module);
+ cf->m_module = module;
+#endif
+ op->func_dict = NULL;
+ op->func_name = NULL;
+ Py_INCREF(qualname);
+ op->func_qualname = qualname;
+ op->func_doc = NULL;
+#if PY_VERSION_HEX < 0x030900B1 || CYTHON_COMPILING_IN_LIMITED_API
+ op->func_classobj = NULL;
+#else
+ ((PyCMethodObject*)op)->mm_class = NULL;
+#endif
+ op->func_globals = globals;
+ Py_INCREF(op->func_globals);
+ Py_XINCREF(code);
+ op->func_code = code;
+ op->defaults = NULL;
+ op->defaults_tuple = NULL;
+ op->defaults_kwdict = NULL;
+ op->defaults_getter = NULL;
+ op->func_annotations = NULL;
+ op->func_is_coroutine = NULL;
+#if CYTHON_METH_FASTCALL
+ switch (ml->ml_flags & (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O | METH_KEYWORDS | METH_METHOD)) {
+ case METH_NOARGS:
+ __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_NOARGS;
+ break;
+ case METH_O:
+ __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_O;
+ break;
+ case METH_METHOD | METH_FASTCALL | METH_KEYWORDS:
+ __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS_METHOD;
+ break;
+ case METH_FASTCALL | METH_KEYWORDS:
+ __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS;
+ break;
+ case METH_VARARGS | METH_KEYWORDS:
+ __Pyx_CyFunction_func_vectorcall(op) = NULL;
+ break;
+ default:
+ PyErr_SetString(PyExc_SystemError, "Bad call flags for CyFunction");
+ Py_DECREF(op);
+ return NULL;
+ }
+#endif
+ return (PyObject *) op;
+}
+static int
+__Pyx_CyFunction_clear(__pyx_CyFunctionObject *m)
+{
+ Py_CLEAR(m->func_closure);
+#if CYTHON_COMPILING_IN_LIMITED_API
+ Py_CLEAR(m->func);
+#else
+ Py_CLEAR(((PyCFunctionObject*)m)->m_module);
+#endif
+ Py_CLEAR(m->func_dict);
+ Py_CLEAR(m->func_name);
+ Py_CLEAR(m->func_qualname);
+ Py_CLEAR(m->func_doc);
+ Py_CLEAR(m->func_globals);
+ Py_CLEAR(m->func_code);
+#if !CYTHON_COMPILING_IN_LIMITED_API
+#if PY_VERSION_HEX < 0x030900B1
+ Py_CLEAR(__Pyx_CyFunction_GetClassObj(m));
+#else
+ {
+ PyObject *cls = (PyObject*) ((PyCMethodObject *) (m))->mm_class;
+ ((PyCMethodObject *) (m))->mm_class = NULL;
+ Py_XDECREF(cls);
+ }
+#endif
+#endif
+ Py_CLEAR(m->defaults_tuple);
+ Py_CLEAR(m->defaults_kwdict);
+ Py_CLEAR(m->func_annotations);
+ Py_CLEAR(m->func_is_coroutine);
+ Py_CLEAR(m->defaults);
+ return 0;
+}
+static void __Pyx__CyFunction_dealloc(__pyx_CyFunctionObject *m)
+{
+ if (__Pyx_CyFunction_weakreflist(m) != NULL)
+ PyObject_ClearWeakRefs((PyObject *) m);
+ __Pyx_CyFunction_clear(m);
+ __Pyx_PyHeapTypeObject_GC_Del(m);
+}
+static void __Pyx_CyFunction_dealloc(__pyx_CyFunctionObject *m)
+{
+ PyObject_GC_UnTrack(m);
+ __Pyx__CyFunction_dealloc(m);
+}
+static int __Pyx_CyFunction_traverse(__pyx_CyFunctionObject *m, visitproc visit, void *arg)
+{
+ {
+ int e = __Pyx_call_type_traverse((PyObject*)m, 1, visit, arg);
+ if (e) return e;
+ }
+ Py_VISIT(m->func_closure);
+#if CYTHON_COMPILING_IN_LIMITED_API
+ Py_VISIT(m->func);
+#else
+ Py_VISIT(((PyCFunctionObject*)m)->m_module);
+#endif
+ Py_VISIT(m->func_dict);
+ __Pyx_VISIT_CONST(m->func_name);
+ __Pyx_VISIT_CONST(m->func_qualname);
+ Py_VISIT(m->func_doc);
+ Py_VISIT(m->func_globals);
+ __Pyx_VISIT_CONST(m->func_code);
+#if !CYTHON_COMPILING_IN_LIMITED_API
+ Py_VISIT(__Pyx_CyFunction_GetClassObj(m));
+#endif
+ Py_VISIT(m->defaults_tuple);
+ Py_VISIT(m->defaults_kwdict);
+ Py_VISIT(m->func_is_coroutine);
+ Py_VISIT(m->defaults);
+ return 0;
+}
+static PyObject*
+__Pyx_CyFunction_repr(__pyx_CyFunctionObject *op)
+{
+ PyObject *repr;
+ __Pyx_BEGIN_CRITICAL_SECTION(op);
+ repr = PyUnicode_FromFormat("",
+ op->func_qualname, (void *)op);
+ __Pyx_END_CRITICAL_SECTION();
+ return repr;
+}
+static PyObject * __Pyx_CyFunction_CallMethod(PyObject *func, PyObject *self, PyObject *arg, PyObject *kw) {
+#if CYTHON_COMPILING_IN_LIMITED_API
+ PyObject *f = ((__pyx_CyFunctionObject*)func)->func;
+ PyCFunction meth;
+ int flags;
+ meth = PyCFunction_GetFunction(f);
+ if (unlikely(!meth)) return NULL;
+ flags = PyCFunction_GetFlags(f);
+ if (unlikely(flags < 0)) return NULL;
+#else
+ PyCFunctionObject* f = (PyCFunctionObject*)func;
+ PyCFunction meth = f->m_ml->ml_meth;
+ int flags = f->m_ml->ml_flags;
+#endif
+ Py_ssize_t size;
+ switch (flags & (METH_VARARGS | METH_KEYWORDS | METH_NOARGS | METH_O)) {
+ case METH_VARARGS:
+ if (likely(kw == NULL || PyDict_Size(kw) == 0))
+ return (*meth)(self, arg);
+ break;
+ case METH_VARARGS | METH_KEYWORDS:
+ return (*(PyCFunctionWithKeywords)(void(*)(void))meth)(self, arg, kw);
+ case METH_NOARGS:
+ if (likely(kw == NULL || PyDict_Size(kw) == 0)) {
+#if CYTHON_ASSUME_SAFE_SIZE
+ size = PyTuple_GET_SIZE(arg);
+#else
+ size = PyTuple_Size(arg);
+ if (unlikely(size < 0)) return NULL;
+#endif
+ if (likely(size == 0))
+ return (*meth)(self, NULL);
+ __Pyx_CyFunction_raise_argument_count_error(
+ (__pyx_CyFunctionObject*)func,
+ "takes no arguments", size);
+ return NULL;
+ }
+ break;
+ case METH_O:
+ if (likely(kw == NULL || PyDict_Size(kw) == 0)) {
+#if CYTHON_ASSUME_SAFE_SIZE
+ size = PyTuple_GET_SIZE(arg);
+#else
+ size = PyTuple_Size(arg);
+ if (unlikely(size < 0)) return NULL;
+#endif
+ if (likely(size == 1)) {
+ PyObject *result, *arg0;
+ #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS
+ arg0 = PyTuple_GET_ITEM(arg, 0);
+ #else
+ arg0 = __Pyx_PySequence_ITEM(arg, 0); if (unlikely(!arg0)) return NULL;
+ #endif
+ result = (*meth)(self, arg0);
+ #if !(CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS)
+ Py_DECREF(arg0);
+ #endif
+ return result;
+ }
+ __Pyx_CyFunction_raise_argument_count_error(
+ (__pyx_CyFunctionObject*)func,
+ "takes exactly one argument", size);
+ return NULL;
+ }
+ break;
+ default:
+ PyErr_SetString(PyExc_SystemError, "Bad call flags for CyFunction");
+ return NULL;
+ }
+ __Pyx_CyFunction_raise_type_error(
+ (__pyx_CyFunctionObject*)func, "takes no keyword arguments");
+ return NULL;
+}
+static CYTHON_INLINE PyObject *__Pyx_CyFunction_Call(PyObject *func, PyObject *arg, PyObject *kw) {
+ PyObject *self, *result;
+#if CYTHON_COMPILING_IN_LIMITED_API
+ self = PyCFunction_GetSelf(((__pyx_CyFunctionObject*)func)->func);
+ if (unlikely(!self) && PyErr_Occurred()) return NULL;
+#else
+ self = ((PyCFunctionObject*)func)->m_self;
+#endif
+ result = __Pyx_CyFunction_CallMethod(func, self, arg, kw);
+ return result;
+}
+static PyObject *__Pyx_CyFunction_CallAsMethod(PyObject *func, PyObject *args, PyObject *kw) {
+ PyObject *result;
+ __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *) func;
+#if CYTHON_METH_FASTCALL && (CYTHON_VECTORCALL || CYTHON_BACKPORT_VECTORCALL)
+ __pyx_vectorcallfunc vc = __Pyx_CyFunction_func_vectorcall(cyfunc);
+ if (vc) {
+#if CYTHON_ASSUME_SAFE_MACROS && CYTHON_ASSUME_SAFE_SIZE
+ return __Pyx_PyVectorcall_FastCallDict(func, vc, &PyTuple_GET_ITEM(args, 0), (size_t)PyTuple_GET_SIZE(args), kw);
+#else
+ (void) &__Pyx_PyVectorcall_FastCallDict;
+ return PyVectorcall_Call(func, args, kw);
+#endif
+ }
+#endif
+ if ((cyfunc->flags & __Pyx_CYFUNCTION_CCLASS) && !(cyfunc->flags & __Pyx_CYFUNCTION_STATICMETHOD)) {
+ Py_ssize_t argc;
+ PyObject *new_args;
+ PyObject *self;
+#if CYTHON_ASSUME_SAFE_SIZE
+ argc = PyTuple_GET_SIZE(args);
+#else
+ argc = PyTuple_Size(args);
+ if (unlikely(argc < 0)) return NULL;
+#endif
+ new_args = PyTuple_GetSlice(args, 1, argc);
+ if (unlikely(!new_args))
+ return NULL;
+ self = PyTuple_GetItem(args, 0);
+ if (unlikely(!self)) {
+ Py_DECREF(new_args);
+ PyErr_Format(PyExc_TypeError,
+ "unbound method %.200S() needs an argument",
+ cyfunc->func_qualname);
+ return NULL;
+ }
+ result = __Pyx_CyFunction_CallMethod(func, self, new_args, kw);
+ Py_DECREF(new_args);
+ } else {
+ result = __Pyx_CyFunction_Call(func, args, kw);
+ }
+ return result;
+}
+#if CYTHON_METH_FASTCALL && (CYTHON_VECTORCALL || CYTHON_BACKPORT_VECTORCALL)
+static CYTHON_INLINE int __Pyx_CyFunction_Vectorcall_CheckArgs(__pyx_CyFunctionObject *cyfunc, Py_ssize_t nargs, PyObject *kwnames)
+{
+ int ret = 0;
+ if ((cyfunc->flags & __Pyx_CYFUNCTION_CCLASS) && !(cyfunc->flags & __Pyx_CYFUNCTION_STATICMETHOD)) {
+ if (unlikely(nargs < 1)) {
+ __Pyx_CyFunction_raise_type_error(
+ cyfunc, "needs an argument");
+ return -1;
+ }
+ ret = 1;
+ }
+ if (unlikely(kwnames) && unlikely(__Pyx_PyTuple_GET_SIZE(kwnames))) {
+ __Pyx_CyFunction_raise_type_error(
+ cyfunc, "takes no keyword arguments");
+ return -1;
+ }
+ return ret;
+}
+static PyObject * __Pyx_CyFunction_Vectorcall_NOARGS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames)
+{
+ __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func;
+#if CYTHON_BACKPORT_VECTORCALL
+ Py_ssize_t nargs = (Py_ssize_t)nargsf;
+#else
+ Py_ssize_t nargs = PyVectorcall_NARGS(nargsf);
+#endif
+ PyObject *self;
+#if CYTHON_COMPILING_IN_LIMITED_API
+ PyCFunction meth = PyCFunction_GetFunction(cyfunc->func);
+ if (unlikely(!meth)) return NULL;
+#else
+ PyCFunction meth = ((PyCFunctionObject*)cyfunc)->m_ml->ml_meth;
+#endif
+ switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, kwnames)) {
+ case 1:
+ self = args[0];
+ args += 1;
+ nargs -= 1;
+ break;
+ case 0:
+#if CYTHON_COMPILING_IN_LIMITED_API
+ self = PyCFunction_GetSelf(((__pyx_CyFunctionObject*)cyfunc)->func);
+ if (unlikely(!self) && PyErr_Occurred()) return NULL;
+#else
+ self = ((PyCFunctionObject*)cyfunc)->m_self;
+#endif
+ break;
+ default:
+ return NULL;
+ }
+ if (unlikely(nargs != 0)) {
+ __Pyx_CyFunction_raise_argument_count_error(
+ cyfunc, "takes no arguments", nargs);
+ return NULL;
+ }
+ return meth(self, NULL);
+}
+static PyObject * __Pyx_CyFunction_Vectorcall_O(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames)
+{
+ __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func;
+#if CYTHON_BACKPORT_VECTORCALL
+ Py_ssize_t nargs = (Py_ssize_t)nargsf;
+#else
+ Py_ssize_t nargs = PyVectorcall_NARGS(nargsf);
+#endif
+ PyObject *self;
+#if CYTHON_COMPILING_IN_LIMITED_API
+ PyCFunction meth = PyCFunction_GetFunction(cyfunc->func);
+ if (unlikely(!meth)) return NULL;
+#else
+ PyCFunction meth = ((PyCFunctionObject*)cyfunc)->m_ml->ml_meth;
+#endif
+ switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, kwnames)) {
+ case 1:
+ self = args[0];
+ args += 1;
+ nargs -= 1;
+ break;
+ case 0:
+#if CYTHON_COMPILING_IN_LIMITED_API
+ self = PyCFunction_GetSelf(((__pyx_CyFunctionObject*)cyfunc)->func);
+ if (unlikely(!self) && PyErr_Occurred()) return NULL;
+#else
+ self = ((PyCFunctionObject*)cyfunc)->m_self;
+#endif
+ break;
+ default:
+ return NULL;
+ }
+ if (unlikely(nargs != 1)) {
+ __Pyx_CyFunction_raise_argument_count_error(
+ cyfunc, "takes exactly one argument", nargs);
+ return NULL;
+ }
+ return meth(self, args[0]);
+}
+static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames)
+{
+ __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func;
+#if CYTHON_BACKPORT_VECTORCALL
+ Py_ssize_t nargs = (Py_ssize_t)nargsf;
+#else
+ Py_ssize_t nargs = PyVectorcall_NARGS(nargsf);
+#endif
+ PyObject *self;
+#if CYTHON_COMPILING_IN_LIMITED_API
+ PyCFunction meth = PyCFunction_GetFunction(cyfunc->func);
+ if (unlikely(!meth)) return NULL;
+#else
+ PyCFunction meth = ((PyCFunctionObject*)cyfunc)->m_ml->ml_meth;
+#endif
+ switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, NULL)) {
+ case 1:
+ self = args[0];
+ args += 1;
+ nargs -= 1;
+ break;
+ case 0:
+#if CYTHON_COMPILING_IN_LIMITED_API
+ self = PyCFunction_GetSelf(((__pyx_CyFunctionObject*)cyfunc)->func);
+ if (unlikely(!self) && PyErr_Occurred()) return NULL;
+#else
+ self = ((PyCFunctionObject*)cyfunc)->m_self;
+#endif
+ break;
+ default:
+ return NULL;
+ }
+ return ((__Pyx_PyCFunctionFastWithKeywords)(void(*)(void))meth)(self, args, nargs, kwnames);
+}
+static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS_METHOD(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames)
+{
+ __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func;
+ PyTypeObject *cls = (PyTypeObject *) __Pyx_CyFunction_GetClassObj(cyfunc);
+#if CYTHON_BACKPORT_VECTORCALL
+ Py_ssize_t nargs = (Py_ssize_t)nargsf;
+#else
+ Py_ssize_t nargs = PyVectorcall_NARGS(nargsf);
+#endif
+ PyObject *self;
+#if CYTHON_COMPILING_IN_LIMITED_API
+ PyCFunction meth = PyCFunction_GetFunction(cyfunc->func);
+ if (unlikely(!meth)) return NULL;
+#else
+ PyCFunction meth = ((PyCFunctionObject*)cyfunc)->m_ml->ml_meth;
+#endif
+ switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, NULL)) {
+ case 1:
+ self = args[0];
+ args += 1;
+ nargs -= 1;
+ break;
+ case 0:
+#if CYTHON_COMPILING_IN_LIMITED_API
+ self = PyCFunction_GetSelf(((__pyx_CyFunctionObject*)cyfunc)->func);
+ if (unlikely(!self) && PyErr_Occurred()) return NULL;
+#else
+ self = ((PyCFunctionObject*)cyfunc)->m_self;
+#endif
+ break;
+ default:
+ return NULL;
+ }
+ return ((__Pyx_PyCMethod)(void(*)(void))meth)(self, cls, args, (size_t)nargs, kwnames);
+}
+#endif
+static PyType_Slot __pyx_CyFunctionType_slots[] = {
+ {Py_tp_dealloc, (void *)__Pyx_CyFunction_dealloc},
+ {Py_tp_repr, (void *)__Pyx_CyFunction_repr},
+ {Py_tp_call, (void *)__Pyx_CyFunction_CallAsMethod},
+ {Py_tp_traverse, (void *)__Pyx_CyFunction_traverse},
+ {Py_tp_clear, (void *)__Pyx_CyFunction_clear},
+ {Py_tp_methods, (void *)__pyx_CyFunction_methods},
+ {Py_tp_members, (void *)__pyx_CyFunction_members},
+ {Py_tp_getset, (void *)__pyx_CyFunction_getsets},
+ {Py_tp_descr_get, (void *)__Pyx_PyMethod_New},
+ {0, 0},
+};
+static PyType_Spec __pyx_CyFunctionType_spec = {
+ __PYX_TYPE_MODULE_PREFIX "cython_function_or_method",
+ sizeof(__pyx_CyFunctionObject),
+ 0,
+#ifdef Py_TPFLAGS_METHOD_DESCRIPTOR
+ Py_TPFLAGS_METHOD_DESCRIPTOR |
+#endif
+#if CYTHON_METH_FASTCALL
+#if defined(Py_TPFLAGS_HAVE_VECTORCALL)
+ Py_TPFLAGS_HAVE_VECTORCALL |
+#elif defined(_Py_TPFLAGS_HAVE_VECTORCALL)
+ _Py_TPFLAGS_HAVE_VECTORCALL |
+#endif
+#endif // CYTHON_METH_FASTCALL
+#if PY_VERSION_HEX >= 0x030A0000
+ Py_TPFLAGS_IMMUTABLETYPE |
+#endif
+ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE,
+ __pyx_CyFunctionType_slots
+};
+static int __pyx_CyFunction_init(PyObject *module) {
+ __pyx_mstatetype *mstate = __Pyx_PyModule_GetState(module);
+ mstate->__pyx_CyFunctionType = __Pyx_FetchCommonTypeFromSpec(
+ mstate->__pyx_CommonTypesMetaclassType, module, &__pyx_CyFunctionType_spec, NULL);
+ if (unlikely(mstate->__pyx_CyFunctionType == NULL)) {
+ return -1;
+ }
+ return 0;
+}
+static CYTHON_INLINE PyObject *__Pyx_CyFunction_InitDefaults(PyObject *func, PyTypeObject *defaults_type) {
+ __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func;
+ m->defaults = PyObject_CallObject((PyObject*)defaults_type, NULL); // _PyObject_New(defaults_type);
+ if (unlikely(!m->defaults))
+ return NULL;
+ return m->defaults;
+}
+static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsTuple(PyObject *func, PyObject *tuple) {
+ __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func;
+ m->defaults_tuple = tuple;
+ Py_INCREF(tuple);
+}
+static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsKwDict(PyObject *func, PyObject *dict) {
+ __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func;
+ m->defaults_kwdict = dict;
+ Py_INCREF(dict);
+}
+static CYTHON_INLINE void __Pyx_CyFunction_SetAnnotationsDict(PyObject *func, PyObject *dict) {
+ __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func;
+ m->func_annotations = dict;
+ Py_INCREF(dict);
+}
+
+/* CythonFunction */
+static PyObject *__Pyx_CyFunction_New(PyMethodDef *ml, int flags, PyObject* qualname,
+ PyObject *closure, PyObject *module, PyObject* globals, PyObject* code) {
+ PyObject *op = __Pyx_CyFunction_Init(
+ PyObject_GC_New(__pyx_CyFunctionObject, __pyx_mstate_global->__pyx_CyFunctionType),
+ ml, flags, qualname, closure, module, globals, code
+ );
+ if (likely(op)) {
+ PyObject_GC_Track(op);
+ }
+ return op;
+}
+
+/* CLineInTraceback */
+#if CYTHON_CLINE_IN_TRACEBACK && CYTHON_CLINE_IN_TRACEBACK_RUNTIME
+static int __Pyx_CLineForTraceback(PyThreadState *tstate, int c_line) {
+ PyObject *use_cline;
+ PyObject *ptype, *pvalue, *ptraceback;
+#if CYTHON_COMPILING_IN_CPYTHON
+ PyObject **cython_runtime_dict;
+#endif
+ CYTHON_MAYBE_UNUSED_VAR(tstate);
+ if (unlikely(!__pyx_mstate_global->__pyx_cython_runtime)) {
+ return c_line;
+ }
+ __Pyx_ErrFetchInState(tstate, &ptype, &pvalue, &ptraceback);
+#if CYTHON_COMPILING_IN_CPYTHON
+ cython_runtime_dict = _PyObject_GetDictPtr(__pyx_mstate_global->__pyx_cython_runtime);
+ if (likely(cython_runtime_dict)) {
+ __Pyx_BEGIN_CRITICAL_SECTION(*cython_runtime_dict);
+ __PYX_PY_DICT_LOOKUP_IF_MODIFIED(
+ use_cline, *cython_runtime_dict,
+ __Pyx_PyDict_GetItemStr(*cython_runtime_dict, __pyx_mstate_global->__pyx_n_u_cline_in_traceback))
+ Py_XINCREF(use_cline);
+ __Pyx_END_CRITICAL_SECTION();
+ } else
+#endif
+ {
+ PyObject *use_cline_obj = __Pyx_PyObject_GetAttrStrNoError(__pyx_mstate_global->__pyx_cython_runtime, __pyx_mstate_global->__pyx_n_u_cline_in_traceback);
+ if (use_cline_obj) {
+ use_cline = PyObject_Not(use_cline_obj) ? Py_False : Py_True;
+ Py_INCREF(use_cline);
+ Py_DECREF(use_cline_obj);
+ } else {
+ PyErr_Clear();
+ use_cline = NULL;
+ }
+ }
+ if (!use_cline) {
+ c_line = 0;
+ (void) PyObject_SetAttr(__pyx_mstate_global->__pyx_cython_runtime, __pyx_mstate_global->__pyx_n_u_cline_in_traceback, Py_False);
+ }
+ else if (use_cline == Py_False || (use_cline != Py_True && PyObject_Not(use_cline) != 0)) {
+ c_line = 0;
+ }
+ Py_XDECREF(use_cline);
+ __Pyx_ErrRestoreInState(tstate, ptype, pvalue, ptraceback);
+ return c_line;
+}
+#endif
+
+/* CodeObjectCache */
+static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line) {
+ int start = 0, mid = 0, end = count - 1;
+ if (end >= 0 && code_line > entries[end].code_line) {
+ return count;
+ }
+ while (start < end) {
+ mid = start + (end - start) / 2;
+ if (code_line < entries[mid].code_line) {
+ end = mid;
+ } else if (code_line > entries[mid].code_line) {
+ start = mid + 1;
+ } else {
+ return mid;
+ }
+ }
+ if (code_line <= entries[mid].code_line) {
+ return mid;
+ } else {
+ return mid + 1;
+ }
+}
+static __Pyx_CachedCodeObjectType *__pyx__find_code_object(struct __Pyx_CodeObjectCache *code_cache, int code_line) {
+ __Pyx_CachedCodeObjectType* code_object;
+ int pos;
+ if (unlikely(!code_line) || unlikely(!code_cache->entries)) {
+ return NULL;
+ }
+ pos = __pyx_bisect_code_objects(code_cache->entries, code_cache->count, code_line);
+ if (unlikely(pos >= code_cache->count) || unlikely(code_cache->entries[pos].code_line != code_line)) {
+ return NULL;
+ }
+ code_object = code_cache->entries[pos].code_object;
+ Py_INCREF(code_object);
+ return code_object;
+}
+static __Pyx_CachedCodeObjectType *__pyx_find_code_object(int code_line) {
+#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING && !CYTHON_ATOMICS
+ (void)__pyx__find_code_object;
+ return NULL; // Most implementation should have atomics. But otherwise, don't make it thread-safe, just miss.
+#else
+ struct __Pyx_CodeObjectCache *code_cache = &__pyx_mstate_global->__pyx_code_cache;
+#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING
+ __pyx_nonatomic_int_type old_count = __pyx_atomic_incr_acq_rel(&code_cache->accessor_count);
+ if (old_count < 0) {
+ __pyx_atomic_decr_acq_rel(&code_cache->accessor_count);
+ return NULL;
+ }
+#endif
+ __Pyx_CachedCodeObjectType *result = __pyx__find_code_object(code_cache, code_line);
+#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING
+ __pyx_atomic_decr_acq_rel(&code_cache->accessor_count);
+#endif
+ return result;
+#endif
+}
+static void __pyx__insert_code_object(struct __Pyx_CodeObjectCache *code_cache, int code_line, __Pyx_CachedCodeObjectType* code_object)
+{
+ int pos, i;
+ __Pyx_CodeObjectCacheEntry* entries = code_cache->entries;
+ if (unlikely(!code_line)) {
+ return;
+ }
+ if (unlikely(!entries)) {
+ entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Malloc(64*sizeof(__Pyx_CodeObjectCacheEntry));
+ if (likely(entries)) {
+ code_cache->entries = entries;
+ code_cache->max_count = 64;
+ code_cache->count = 1;
+ entries[0].code_line = code_line;
+ entries[0].code_object = code_object;
+ Py_INCREF(code_object);
+ }
+ return;
+ }
+ pos = __pyx_bisect_code_objects(code_cache->entries, code_cache->count, code_line);
+ if ((pos < code_cache->count) && unlikely(code_cache->entries[pos].code_line == code_line)) {
+ __Pyx_CachedCodeObjectType* tmp = entries[pos].code_object;
+ entries[pos].code_object = code_object;
+ Py_INCREF(code_object);
+ Py_DECREF(tmp);
+ return;
+ }
+ if (code_cache->count == code_cache->max_count) {
+ int new_max = code_cache->max_count + 64;
+ entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Realloc(
+ code_cache->entries, ((size_t)new_max) * sizeof(__Pyx_CodeObjectCacheEntry));
+ if (unlikely(!entries)) {
+ return;
+ }
+ code_cache->entries = entries;
+ code_cache->max_count = new_max;
+ }
+ for (i=code_cache->count; i>pos; i--) {
+ entries[i] = entries[i-1];
+ }
+ entries[pos].code_line = code_line;
+ entries[pos].code_object = code_object;
+ code_cache->count++;
+ Py_INCREF(code_object);
+}
+static void __pyx_insert_code_object(int code_line, __Pyx_CachedCodeObjectType* code_object) {
+#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING && !CYTHON_ATOMICS
+ (void)__pyx__insert_code_object;
+ return; // Most implementation should have atomics. But otherwise, don't make it thread-safe, just fail.
+#else
+ struct __Pyx_CodeObjectCache *code_cache = &__pyx_mstate_global->__pyx_code_cache;
+#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING
+ __pyx_nonatomic_int_type expected = 0;
+ if (!__pyx_atomic_int_cmp_exchange(&code_cache->accessor_count, &expected, INT_MIN)) {
+ return;
+ }
+#endif
+ __pyx__insert_code_object(code_cache, code_line, code_object);
+#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING
+ __pyx_atomic_sub(&code_cache->accessor_count, INT_MIN);
+#endif
+#endif
+}
+
+/* AddTraceback */
+#include "compile.h"
+#include "frameobject.h"
+#include "traceback.h"
+#if PY_VERSION_HEX >= 0x030b00a6 && !CYTHON_COMPILING_IN_LIMITED_API && !defined(PYPY_VERSION)
+ #ifndef Py_BUILD_CORE
+ #define Py_BUILD_CORE 1
+ #endif
+ #include "internal/pycore_frame.h"
+#endif
+#if CYTHON_COMPILING_IN_LIMITED_API
+static PyObject *__Pyx_PyCode_Replace_For_AddTraceback(PyObject *code, PyObject *scratch_dict,
+ PyObject *firstlineno, PyObject *name) {
+ PyObject *replace = NULL;
+ if (unlikely(PyDict_SetItemString(scratch_dict, "co_firstlineno", firstlineno))) return NULL;
+ if (unlikely(PyDict_SetItemString(scratch_dict, "co_name", name))) return NULL;
+ replace = PyObject_GetAttrString(code, "replace");
+ if (likely(replace)) {
+ PyObject *result = PyObject_Call(replace, __pyx_mstate_global->__pyx_empty_tuple, scratch_dict);
+ Py_DECREF(replace);
+ return result;
+ }
+ PyErr_Clear();
+ return NULL;
+}
+static void __Pyx_AddTraceback(const char *funcname, int c_line,
+ int py_line, const char *filename) {
+ PyObject *code_object = NULL, *py_py_line = NULL, *py_funcname = NULL, *dict = NULL;
+ PyObject *replace = NULL, *getframe = NULL, *frame = NULL;
+ PyObject *exc_type, *exc_value, *exc_traceback;
+ int success = 0;
+ if (c_line) {
+ (void) __pyx_cfilenm;
+ (void) __Pyx_CLineForTraceback(__Pyx_PyThreadState_Current, c_line);
+ }
+ PyErr_Fetch(&exc_type, &exc_value, &exc_traceback);
+ code_object = __pyx_find_code_object(c_line ? -c_line : py_line);
+ if (!code_object) {
+ code_object = Py_CompileString("_getframe()", filename, Py_eval_input);
+ if (unlikely(!code_object)) goto bad;
+ py_py_line = PyLong_FromLong(py_line);
+ if (unlikely(!py_py_line)) goto bad;
+ py_funcname = PyUnicode_FromString(funcname);
+ if (unlikely(!py_funcname)) goto bad;
+ dict = PyDict_New();
+ if (unlikely(!dict)) goto bad;
+ {
+ PyObject *old_code_object = code_object;
+ code_object = __Pyx_PyCode_Replace_For_AddTraceback(code_object, dict, py_py_line, py_funcname);
+ Py_DECREF(old_code_object);
+ }
+ if (unlikely(!code_object)) goto bad;
+ __pyx_insert_code_object(c_line ? -c_line : py_line, code_object);
+ } else {
+ dict = PyDict_New();
+ }
+ getframe = PySys_GetObject("_getframe");
+ if (unlikely(!getframe)) goto bad;
+ if (unlikely(PyDict_SetItemString(dict, "_getframe", getframe))) goto bad;
+ frame = PyEval_EvalCode(code_object, dict, dict);
+ if (unlikely(!frame) || frame == Py_None) goto bad;
+ success = 1;
+ bad:
+ PyErr_Restore(exc_type, exc_value, exc_traceback);
+ Py_XDECREF(code_object);
+ Py_XDECREF(py_py_line);
+ Py_XDECREF(py_funcname);
+ Py_XDECREF(dict);
+ Py_XDECREF(replace);
+ if (success) {
+ PyTraceBack_Here(
+ (struct _frame*)frame);
+ }
+ Py_XDECREF(frame);
+}
+#else
+static PyCodeObject* __Pyx_CreateCodeObjectForTraceback(
+ const char *funcname, int c_line,
+ int py_line, const char *filename) {
+ PyCodeObject *py_code = NULL;
+ PyObject *py_funcname = NULL;
+ if (c_line) {
+ py_funcname = PyUnicode_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line);
+ if (!py_funcname) goto bad;
+ funcname = PyUnicode_AsUTF8(py_funcname);
+ if (!funcname) goto bad;
+ }
+ py_code = PyCode_NewEmpty(filename, funcname, py_line);
+ Py_XDECREF(py_funcname);
+ return py_code;
+bad:
+ Py_XDECREF(py_funcname);
+ return NULL;
+}
+static void __Pyx_AddTraceback(const char *funcname, int c_line,
+ int py_line, const char *filename) {
+ PyCodeObject *py_code = 0;
+ PyFrameObject *py_frame = 0;
+ PyThreadState *tstate = __Pyx_PyThreadState_Current;
+ PyObject *ptype, *pvalue, *ptraceback;
+ if (c_line) {
+ c_line = __Pyx_CLineForTraceback(tstate, c_line);
+ }
+ py_code = __pyx_find_code_object(c_line ? -c_line : py_line);
+ if (!py_code) {
+ __Pyx_ErrFetchInState(tstate, &ptype, &pvalue, &ptraceback);
+ py_code = __Pyx_CreateCodeObjectForTraceback(
+ funcname, c_line, py_line, filename);
+ if (!py_code) {
+ /* If the code object creation fails, then we should clear the
+ fetched exception references and propagate the new exception */
+ Py_XDECREF(ptype);
+ Py_XDECREF(pvalue);
+ Py_XDECREF(ptraceback);
+ goto bad;
+ }
+ __Pyx_ErrRestoreInState(tstate, ptype, pvalue, ptraceback);
+ __pyx_insert_code_object(c_line ? -c_line : py_line, py_code);
+ }
+ py_frame = PyFrame_New(
+ tstate, /*PyThreadState *tstate,*/
+ py_code, /*PyCodeObject *code,*/
+ __pyx_mstate_global->__pyx_d, /*PyObject *globals,*/
+ 0 /*PyObject *locals*/
+ );
+ if (!py_frame) goto bad;
+ __Pyx_PyFrame_SetLineNumber(py_frame, py_line);
+ PyTraceBack_Here(py_frame);
+bad:
+ Py_XDECREF(py_code);
+ Py_XDECREF(py_frame);
+}
+#endif
+
+/* Declarations */
+#if CYTHON_CCOMPLEX && (1) && (!0 || __cplusplus)
+ #ifdef __cplusplus
+ static CYTHON_INLINE __pyx_t_double_complex __pyx_t_double_complex_from_parts(double x, double y) {
+ return ::std::complex< double >(x, y);
+ }
+ #else
+ static CYTHON_INLINE __pyx_t_double_complex __pyx_t_double_complex_from_parts(double x, double y) {
+ return x + y*(__pyx_t_double_complex)_Complex_I;
+ }
+ #endif
+#else
+ static CYTHON_INLINE __pyx_t_double_complex __pyx_t_double_complex_from_parts(double x, double y) {
+ __pyx_t_double_complex z;
+ z.real = x;
+ z.imag = y;
+ return z;
+ }
+#endif
+
+/* Arithmetic */
+#if CYTHON_CCOMPLEX && (1) && (!0 || __cplusplus)
+#else
+ static CYTHON_INLINE int __Pyx_c_eq_double(__pyx_t_double_complex a, __pyx_t_double_complex b) {
+ return (a.real == b.real) && (a.imag == b.imag);
+ }
+ static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_sum_double(__pyx_t_double_complex a, __pyx_t_double_complex b) {
+ __pyx_t_double_complex z;
+ z.real = a.real + b.real;
+ z.imag = a.imag + b.imag;
+ return z;
+ }
+ static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_diff_double(__pyx_t_double_complex a, __pyx_t_double_complex b) {
+ __pyx_t_double_complex z;
+ z.real = a.real - b.real;
+ z.imag = a.imag - b.imag;
+ return z;
+ }
+ static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_prod_double(__pyx_t_double_complex a, __pyx_t_double_complex b) {
+ __pyx_t_double_complex z;
+ z.real = a.real * b.real - a.imag * b.imag;
+ z.imag = a.real * b.imag + a.imag * b.real;
+ return z;
+ }
+ #if 1
+ static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_quot_double(__pyx_t_double_complex a, __pyx_t_double_complex b) {
+ if (b.imag == 0) {
+ return __pyx_t_double_complex_from_parts(a.real / b.real, a.imag / b.real);
+ } else if (fabs(b.real) >= fabs(b.imag)) {
+ if (b.real == 0 && b.imag == 0) {
+ return __pyx_t_double_complex_from_parts(a.real / b.real, a.imag / b.imag);
+ } else {
+ double r = b.imag / b.real;
+ double s = (double)(1.0) / (b.real + b.imag * r);
+ return __pyx_t_double_complex_from_parts(
+ (a.real + a.imag * r) * s, (a.imag - a.real * r) * s);
+ }
+ } else {
+ double r = b.real / b.imag;
+ double s = (double)(1.0) / (b.imag + b.real * r);
+ return __pyx_t_double_complex_from_parts(
+ (a.real * r + a.imag) * s, (a.imag * r - a.real) * s);
+ }
+ }
+ #else
+ static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_quot_double(__pyx_t_double_complex a, __pyx_t_double_complex b) {
+ if (b.imag == 0) {
+ return __pyx_t_double_complex_from_parts(a.real / b.real, a.imag / b.real);
+ } else {
+ double denom = b.real * b.real + b.imag * b.imag;
+ return __pyx_t_double_complex_from_parts(
+ (a.real * b.real + a.imag * b.imag) / denom,
+ (a.imag * b.real - a.real * b.imag) / denom);
+ }
+ }
+ #endif
+ static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_neg_double(__pyx_t_double_complex a) {
+ __pyx_t_double_complex z;
+ z.real = -a.real;
+ z.imag = -a.imag;
+ return z;
+ }
+ static CYTHON_INLINE int __Pyx_c_is_zero_double(__pyx_t_double_complex a) {
+ return (a.real == 0) && (a.imag == 0);
+ }
+ static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_conj_double(__pyx_t_double_complex a) {
+ __pyx_t_double_complex z;
+ z.real = a.real;
+ z.imag = -a.imag;
+ return z;
+ }
+ #if 1
+ static CYTHON_INLINE double __Pyx_c_abs_double(__pyx_t_double_complex z) {
+ #if !defined(HAVE_HYPOT) || defined(_MSC_VER)
+ return sqrt(z.real*z.real + z.imag*z.imag);
+ #else
+ return hypot(z.real, z.imag);
+ #endif
+ }
+ static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_pow_double(__pyx_t_double_complex a, __pyx_t_double_complex b) {
+ __pyx_t_double_complex z;
+ double r, lnr, theta, z_r, z_theta;
+ if (b.imag == 0 && b.real == (int)b.real) {
+ if (b.real < 0) {
+ double denom = a.real * a.real + a.imag * a.imag;
+ a.real = a.real / denom;
+ a.imag = -a.imag / denom;
+ b.real = -b.real;
+ }
+ switch ((int)b.real) {
+ case 0:
+ z.real = 1;
+ z.imag = 0;
+ return z;
+ case 1:
+ return a;
+ case 2:
+ return __Pyx_c_prod_double(a, a);
+ case 3:
+ z = __Pyx_c_prod_double(a, a);
+ return __Pyx_c_prod_double(z, a);
+ case 4:
+ z = __Pyx_c_prod_double(a, a);
+ return __Pyx_c_prod_double(z, z);
+ }
+ }
+ if (a.imag == 0) {
+ if (a.real == 0) {
+ return a;
+ } else if ((b.imag == 0) && (a.real >= 0)) {
+ z.real = pow(a.real, b.real);
+ z.imag = 0;
+ return z;
+ } else if (a.real > 0) {
+ r = a.real;
+ theta = 0;
+ } else {
+ r = -a.real;
+ theta = atan2(0.0, -1.0);
+ }
+ } else {
+ r = __Pyx_c_abs_double(a);
+ theta = atan2(a.imag, a.real);
+ }
+ lnr = log(r);
+ z_r = exp(lnr * b.real - theta * b.imag);
+ z_theta = theta * b.real + lnr * b.imag;
+ z.real = z_r * cos(z_theta);
+ z.imag = z_r * sin(z_theta);
+ return z;
+ }
+ #endif
+#endif
+
+/* FromPy */
+static __pyx_t_double_complex __Pyx_PyComplex_As___pyx_t_double_complex(PyObject* o) {
+#if CYTHON_COMPILING_IN_LIMITED_API
+ double real=-1.0, imag=-1.0;
+ real = PyComplex_RealAsDouble(o);
+ if (unlikely(real == -1.0 && PyErr_Occurred())) goto end;
+ imag = PyComplex_ImagAsDouble(o);
+ end:
+ return __pyx_t_double_complex_from_parts(
+ (double)real, (double)imag
+ );
+#else
+ Py_complex cval;
+#if !CYTHON_COMPILING_IN_PYPY && !CYTHON_COMPILING_IN_GRAAL
+ if (PyComplex_CheckExact(o))
+ cval = ((PyComplexObject *)o)->cval;
+ else
+#endif
+ cval = PyComplex_AsCComplex(o);
+ return __pyx_t_double_complex_from_parts(
+ (double)cval.real,
+ (double)cval.imag);
+#endif
+}
+
+/* CIntFromPyVerify */
+#define __PYX_VERIFY_RETURN_INT(target_type, func_type, func_value)\
+ __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 0)
+#define __PYX_VERIFY_RETURN_INT_EXC(target_type, func_type, func_value)\
+ __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 1)
+#define __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, exc)\
+ {\
+ func_type value = func_value;\
+ if (sizeof(target_type) < sizeof(func_type)) {\
+ if (unlikely(value != (func_type) (target_type) value)) {\
+ func_type zero = 0;\
+ if (exc && unlikely(value == (func_type)-1 && PyErr_Occurred()))\
+ return (target_type) -1;\
+ if (is_unsigned && unlikely(value < zero))\
+ goto raise_neg_overflow;\
+ else\
+ goto raise_overflow;\
+ }\
+ }\
+ return (target_type) value;\
+ }
+
+/* CIntFromPy */
+static CYTHON_INLINE int __Pyx_PyLong_As_int(PyObject *x) {
+#ifdef __Pyx_HAS_GCC_DIAGNOSTIC
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wconversion"
+#endif
+ const int neg_one = (int) -1, const_zero = (int) 0;
+#ifdef __Pyx_HAS_GCC_DIAGNOSTIC
+#pragma GCC diagnostic pop
+#endif
+ const int is_unsigned = neg_one > const_zero;
+ if (unlikely(!PyLong_Check(x))) {
+ int val;
+ PyObject *tmp = __Pyx_PyNumber_Long(x);
+ if (!tmp) return (int) -1;
+ val = __Pyx_PyLong_As_int(tmp);
+ Py_DECREF(tmp);
+ return val;
+ }
+ if (is_unsigned) {
+#if CYTHON_USE_PYLONG_INTERNALS
+ if (unlikely(__Pyx_PyLong_IsNeg(x))) {
+ goto raise_neg_overflow;
+ } else if (__Pyx_PyLong_IsCompact(x)) {
+ __PYX_VERIFY_RETURN_INT(int, __Pyx_compact_upylong, __Pyx_PyLong_CompactValueUnsigned(x))
+ } else {
+ const digit* digits = __Pyx_PyLong_Digits(x);
+ assert(__Pyx_PyLong_DigitCount(x) > 1);
+ switch (__Pyx_PyLong_DigitCount(x)) {
+ case 2:
+ if ((8 * sizeof(int) > 1 * PyLong_SHIFT)) {
+ if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) {
+ __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])))
+ } else if ((8 * sizeof(int) >= 2 * PyLong_SHIFT)) {
+ return (int) (((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]));
+ }
+ }
+ break;
+ case 3:
+ if ((8 * sizeof(int) > 2 * PyLong_SHIFT)) {
+ if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) {
+ __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])))
+ } else if ((8 * sizeof(int) >= 3 * PyLong_SHIFT)) {
+ return (int) (((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]));
+ }
+ }
+ break;
+ case 4:
+ if ((8 * sizeof(int) > 3 * PyLong_SHIFT)) {
+ if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) {
+ __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])))
+ } else if ((8 * sizeof(int) >= 4 * PyLong_SHIFT)) {
+ return (int) (((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]));
+ }
+ }
+ break;
+ }
+ }
+#endif
+#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A7
+ if (unlikely(Py_SIZE(x) < 0)) {
+ goto raise_neg_overflow;
+ }
+#else
+ {
+ int result = PyObject_RichCompareBool(x, Py_False, Py_LT);
+ if (unlikely(result < 0))
+ return (int) -1;
+ if (unlikely(result == 1))
+ goto raise_neg_overflow;
+ }
+#endif
+ if ((sizeof(int) <= sizeof(unsigned long))) {
+ __PYX_VERIFY_RETURN_INT_EXC(int, unsigned long, PyLong_AsUnsignedLong(x))
+#ifdef HAVE_LONG_LONG
+ } else if ((sizeof(int) <= sizeof(unsigned PY_LONG_LONG))) {
+ __PYX_VERIFY_RETURN_INT_EXC(int, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x))
+#endif
+ }
+ } else {
+#if CYTHON_USE_PYLONG_INTERNALS
+ if (__Pyx_PyLong_IsCompact(x)) {
+ __PYX_VERIFY_RETURN_INT(int, __Pyx_compact_pylong, __Pyx_PyLong_CompactValue(x))
+ } else {
+ const digit* digits = __Pyx_PyLong_Digits(x);
+ assert(__Pyx_PyLong_DigitCount(x) > 1);
+ switch (__Pyx_PyLong_SignedDigitCount(x)) {
+ case -2:
+ if ((8 * sizeof(int) - 1 > 1 * PyLong_SHIFT)) {
+ if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) {
+ __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])))
+ } else if ((8 * sizeof(int) - 1 > 2 * PyLong_SHIFT)) {
+ return (int) (((int)-1)*(((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0])));
+ }
+ }
+ break;
+ case 2:
+ if ((8 * sizeof(int) > 1 * PyLong_SHIFT)) {
+ if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) {
+ __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])))
+ } else if ((8 * sizeof(int) - 1 > 2 * PyLong_SHIFT)) {
+ return (int) ((((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0])));
+ }
+ }
+ break;
+ case -3:
+ if ((8 * sizeof(int) - 1 > 2 * PyLong_SHIFT)) {
+ if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) {
+ __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])))
+ } else if ((8 * sizeof(int) - 1 > 3 * PyLong_SHIFT)) {
+ return (int) (((int)-1)*(((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])));
+ }
+ }
+ break;
+ case 3:
+ if ((8 * sizeof(int) > 2 * PyLong_SHIFT)) {
+ if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) {
+ __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])))
+ } else if ((8 * sizeof(int) - 1 > 3 * PyLong_SHIFT)) {
+ return (int) ((((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])));
+ }
+ }
+ break;
+ case -4:
+ if ((8 * sizeof(int) - 1 > 3 * PyLong_SHIFT)) {
+ if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) {
+ __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])))
+ } else if ((8 * sizeof(int) - 1 > 4 * PyLong_SHIFT)) {
+ return (int) (((int)-1)*(((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])));
+ }
+ }
+ break;
+ case 4:
+ if ((8 * sizeof(int) > 3 * PyLong_SHIFT)) {
+ if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) {
+ __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])))
+ } else if ((8 * sizeof(int) - 1 > 4 * PyLong_SHIFT)) {
+ return (int) ((((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])));
+ }
+ }
+ break;
+ }
+ }
+#endif
+ if ((sizeof(int) <= sizeof(long))) {
+ __PYX_VERIFY_RETURN_INT_EXC(int, long, PyLong_AsLong(x))
+#ifdef HAVE_LONG_LONG
+ } else if ((sizeof(int) <= sizeof(PY_LONG_LONG))) {
+ __PYX_VERIFY_RETURN_INT_EXC(int, PY_LONG_LONG, PyLong_AsLongLong(x))
+#endif
+ }
+ }
+ {
+ int val;
+ int ret = -1;
+#if PY_VERSION_HEX >= 0x030d00A6 && !CYTHON_COMPILING_IN_LIMITED_API
+ Py_ssize_t bytes_copied = PyLong_AsNativeBytes(
+ x, &val, sizeof(val), Py_ASNATIVEBYTES_NATIVE_ENDIAN | (is_unsigned ? Py_ASNATIVEBYTES_UNSIGNED_BUFFER | Py_ASNATIVEBYTES_REJECT_NEGATIVE : 0));
+ if (unlikely(bytes_copied == -1)) {
+ } else if (unlikely(bytes_copied > (Py_ssize_t) sizeof(val))) {
+ goto raise_overflow;
+ } else {
+ ret = 0;
+ }
+#elif PY_VERSION_HEX < 0x030d0000 && !(CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) || defined(_PyLong_AsByteArray)
+ int one = 1; int is_little = (int)*(unsigned char *)&one;
+ unsigned char *bytes = (unsigned char *)&val;
+ ret = _PyLong_AsByteArray((PyLongObject *)x,
+ bytes, sizeof(val),
+ is_little, !is_unsigned);
+#else
+ PyObject *v;
+ PyObject *stepval = NULL, *mask = NULL, *shift = NULL;
+ int bits, remaining_bits, is_negative = 0;
+ int chunk_size = (sizeof(long) < 8) ? 30 : 62;
+ if (likely(PyLong_CheckExact(x))) {
+ v = __Pyx_NewRef(x);
+ } else {
+ v = PyNumber_Long(x);
+ if (unlikely(!v)) return (int) -1;
+ assert(PyLong_CheckExact(v));
+ }
+ {
+ int result = PyObject_RichCompareBool(v, Py_False, Py_LT);
+ if (unlikely(result < 0)) {
+ Py_DECREF(v);
+ return (int) -1;
+ }
+ is_negative = result == 1;
+ }
+ if (is_unsigned && unlikely(is_negative)) {
+ Py_DECREF(v);
+ goto raise_neg_overflow;
+ } else if (is_negative) {
+ stepval = PyNumber_Invert(v);
+ Py_DECREF(v);
+ if (unlikely(!stepval))
+ return (int) -1;
+ } else {
+ stepval = v;
+ }
+ v = NULL;
+ val = (int) 0;
+ mask = PyLong_FromLong((1L << chunk_size) - 1); if (unlikely(!mask)) goto done;
+ shift = PyLong_FromLong(chunk_size); if (unlikely(!shift)) goto done;
+ for (bits = 0; bits < (int) sizeof(int) * 8 - chunk_size; bits += chunk_size) {
+ PyObject *tmp, *digit;
+ long idigit;
+ digit = PyNumber_And(stepval, mask);
+ if (unlikely(!digit)) goto done;
+ idigit = PyLong_AsLong(digit);
+ Py_DECREF(digit);
+ if (unlikely(idigit < 0)) goto done;
+ val |= ((int) idigit) << bits;
+ tmp = PyNumber_Rshift(stepval, shift);
+ if (unlikely(!tmp)) goto done;
+ Py_DECREF(stepval); stepval = tmp;
+ }
+ Py_DECREF(shift); shift = NULL;
+ Py_DECREF(mask); mask = NULL;
+ {
+ long idigit = PyLong_AsLong(stepval);
+ if (unlikely(idigit < 0)) goto done;
+ remaining_bits = ((int) sizeof(int) * 8) - bits - (is_unsigned ? 0 : 1);
+ if (unlikely(idigit >= (1L << remaining_bits)))
+ goto raise_overflow;
+ val |= ((int) idigit) << bits;
+ }
+ if (!is_unsigned) {
+ if (unlikely(val & (((int) 1) << (sizeof(int) * 8 - 1))))
+ goto raise_overflow;
+ if (is_negative)
+ val = ~val;
+ }
+ ret = 0;
+ done:
+ Py_XDECREF(shift);
+ Py_XDECREF(mask);
+ Py_XDECREF(stepval);
+#endif
+ if (unlikely(ret))
+ return (int) -1;
+ return val;
+ }
+raise_overflow:
+ PyErr_SetString(PyExc_OverflowError,
+ "value too large to convert to int");
+ return (int) -1;
+raise_neg_overflow:
+ PyErr_SetString(PyExc_OverflowError,
+ "can't convert negative value to int");
+ return (int) -1;
+}
+
+/* PyObjectVectorCallKwBuilder */
+#if CYTHON_VECTORCALL
+static int __Pyx_VectorcallBuilder_AddArg(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n) {
+ (void)__Pyx_PyObject_FastCallDict;
+ if (__Pyx_PyTuple_SET_ITEM(builder, n, key) != (0)) return -1;
+ Py_INCREF(key);
+ args[n] = value;
+ return 0;
+}
+CYTHON_UNUSED static int __Pyx_VectorcallBuilder_AddArg_Check(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n) {
+ (void)__Pyx_VectorcallBuilder_AddArgStr;
+ if (unlikely(!PyUnicode_Check(key))) {
+ PyErr_SetString(PyExc_TypeError, "keywords must be strings");
+ return -1;
+ }
+ return __Pyx_VectorcallBuilder_AddArg(key, value, builder, args, n);
+}
+static int __Pyx_VectorcallBuilder_AddArgStr(const char *key, PyObject *value, PyObject *builder, PyObject **args, int n) {
+ PyObject *pyKey = PyUnicode_FromString(key);
+ if (!pyKey) return -1;
+ return __Pyx_VectorcallBuilder_AddArg(pyKey, value, builder, args, n);
+}
+#else // CYTHON_VECTORCALL
+CYTHON_UNUSED static int __Pyx_VectorcallBuilder_AddArg_Check(PyObject *key, PyObject *value, PyObject *builder, CYTHON_UNUSED PyObject **args, CYTHON_UNUSED int n) {
+ if (unlikely(!PyUnicode_Check(key))) {
+ PyErr_SetString(PyExc_TypeError, "keywords must be strings");
+ return -1;
+ }
+ return PyDict_SetItem(builder, key, value);
+}
+#endif
+
+/* CIntToPy */
+static CYTHON_INLINE PyObject* __Pyx_PyLong_From_long(long value) {
+#ifdef __Pyx_HAS_GCC_DIAGNOSTIC
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wconversion"
+#endif
+ const long neg_one = (long) -1, const_zero = (long) 0;
+#ifdef __Pyx_HAS_GCC_DIAGNOSTIC
+#pragma GCC diagnostic pop
+#endif
+ const int is_unsigned = neg_one > const_zero;
+ if (is_unsigned) {
+ if (sizeof(long) < sizeof(long)) {
+ return PyLong_FromLong((long) value);
+ } else if (sizeof(long) <= sizeof(unsigned long)) {
+ return PyLong_FromUnsignedLong((unsigned long) value);
+#if defined(HAVE_LONG_LONG) && !CYTHON_COMPILING_IN_PYPY
+ } else if (sizeof(long) <= sizeof(unsigned PY_LONG_LONG)) {
+ return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value);
+#endif
+ }
+ } else {
+ if (sizeof(long) <= sizeof(long)) {
+ return PyLong_FromLong((long) value);
+#ifdef HAVE_LONG_LONG
+ } else if (sizeof(long) <= sizeof(PY_LONG_LONG)) {
+ return PyLong_FromLongLong((PY_LONG_LONG) value);
+#endif
+ }
+ }
+ {
+ unsigned char *bytes = (unsigned char *)&value;
+#if !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d00A4
+ if (is_unsigned) {
+ return PyLong_FromUnsignedNativeBytes(bytes, sizeof(value), -1);
+ } else {
+ return PyLong_FromNativeBytes(bytes, sizeof(value), -1);
+ }
+#elif !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x030d0000
+ int one = 1; int little = (int)*(unsigned char *)&one;
+ return _PyLong_FromByteArray(bytes, sizeof(long),
+ little, !is_unsigned);
+#else
+ int one = 1; int little = (int)*(unsigned char *)&one;
+ PyObject *from_bytes, *result = NULL, *kwds = NULL;
+ PyObject *py_bytes = NULL, *order_str = NULL;
+ from_bytes = PyObject_GetAttrString((PyObject*)&PyLong_Type, "from_bytes");
+ if (!from_bytes) return NULL;
+ py_bytes = PyBytes_FromStringAndSize((char*)bytes, sizeof(long));
+ if (!py_bytes) goto limited_bad;
+ order_str = PyUnicode_FromString(little ? "little" : "big");
+ if (!order_str) goto limited_bad;
+ {
+ PyObject *args[3+(CYTHON_VECTORCALL ? 1 : 0)] = { NULL, py_bytes, order_str };
+ if (!is_unsigned) {
+ kwds = __Pyx_MakeVectorcallBuilderKwds(1);
+ if (!kwds) goto limited_bad;
+ if (__Pyx_VectorcallBuilder_AddArgStr("signed", __Pyx_NewRef(Py_True), kwds, args+3, 0) < 0) goto limited_bad;
+ }
+ result = __Pyx_Object_Vectorcall_CallFromBuilder(from_bytes, args+1, 2 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET, kwds);
+ }
+ limited_bad:
+ Py_XDECREF(kwds);
+ Py_XDECREF(order_str);
+ Py_XDECREF(py_bytes);
+ Py_XDECREF(from_bytes);
+ return result;
+#endif
+ }
+}
+
+/* CIntToPy */
+static CYTHON_INLINE PyObject* __Pyx_PyLong_From_int(int value) {
+#ifdef __Pyx_HAS_GCC_DIAGNOSTIC
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wconversion"
+#endif
+ const int neg_one = (int) -1, const_zero = (int) 0;
+#ifdef __Pyx_HAS_GCC_DIAGNOSTIC
+#pragma GCC diagnostic pop
+#endif
+ const int is_unsigned = neg_one > const_zero;
+ if (is_unsigned) {
+ if (sizeof(int) < sizeof(long)) {
+ return PyLong_FromLong((long) value);
+ } else if (sizeof(int) <= sizeof(unsigned long)) {
+ return PyLong_FromUnsignedLong((unsigned long) value);
+#if defined(HAVE_LONG_LONG) && !CYTHON_COMPILING_IN_PYPY
+ } else if (sizeof(int) <= sizeof(unsigned PY_LONG_LONG)) {
+ return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value);
+#endif
+ }
+ } else {
+ if (sizeof(int) <= sizeof(long)) {
+ return PyLong_FromLong((long) value);
+#ifdef HAVE_LONG_LONG
+ } else if (sizeof(int) <= sizeof(PY_LONG_LONG)) {
+ return PyLong_FromLongLong((PY_LONG_LONG) value);
+#endif
+ }
+ }
+ {
+ unsigned char *bytes = (unsigned char *)&value;
+#if !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d00A4
+ if (is_unsigned) {
+ return PyLong_FromUnsignedNativeBytes(bytes, sizeof(value), -1);
+ } else {
+ return PyLong_FromNativeBytes(bytes, sizeof(value), -1);
+ }
+#elif !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x030d0000
+ int one = 1; int little = (int)*(unsigned char *)&one;
+ return _PyLong_FromByteArray(bytes, sizeof(int),
+ little, !is_unsigned);
+#else
+ int one = 1; int little = (int)*(unsigned char *)&one;
+ PyObject *from_bytes, *result = NULL, *kwds = NULL;
+ PyObject *py_bytes = NULL, *order_str = NULL;
+ from_bytes = PyObject_GetAttrString((PyObject*)&PyLong_Type, "from_bytes");
+ if (!from_bytes) return NULL;
+ py_bytes = PyBytes_FromStringAndSize((char*)bytes, sizeof(int));
+ if (!py_bytes) goto limited_bad;
+ order_str = PyUnicode_FromString(little ? "little" : "big");
+ if (!order_str) goto limited_bad;
+ {
+ PyObject *args[3+(CYTHON_VECTORCALL ? 1 : 0)] = { NULL, py_bytes, order_str };
+ if (!is_unsigned) {
+ kwds = __Pyx_MakeVectorcallBuilderKwds(1);
+ if (!kwds) goto limited_bad;
+ if (__Pyx_VectorcallBuilder_AddArgStr("signed", __Pyx_NewRef(Py_True), kwds, args+3, 0) < 0) goto limited_bad;
+ }
+ result = __Pyx_Object_Vectorcall_CallFromBuilder(from_bytes, args+1, 2 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET, kwds);
+ }
+ limited_bad:
+ Py_XDECREF(kwds);
+ Py_XDECREF(order_str);
+ Py_XDECREF(py_bytes);
+ Py_XDECREF(from_bytes);
+ return result;
+#endif
+ }
+}
+
+/* FormatTypeName */
+#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030d0000
+static __Pyx_TypeName
+__Pyx_PyType_GetFullyQualifiedName(PyTypeObject* tp)
+{
+ PyObject *module = NULL, *name = NULL, *result = NULL;
+ #if __PYX_LIMITED_VERSION_HEX < 0x030b0000
+ name = __Pyx_PyObject_GetAttrStr((PyObject *)tp,
+ __pyx_mstate_global->__pyx_n_u_qualname);
+ #else
+ name = PyType_GetQualName(tp);
+ #endif
+ if (unlikely(name == NULL) || unlikely(!PyUnicode_Check(name))) goto bad;
+ module = __Pyx_PyObject_GetAttrStr((PyObject *)tp,
+ __pyx_mstate_global->__pyx_n_u_module);
+ if (unlikely(module == NULL) || unlikely(!PyUnicode_Check(module))) goto bad;
+ if (PyUnicode_CompareWithASCIIString(module, "builtins") == 0) {
+ result = name;
+ name = NULL;
+ goto done;
+ }
+ result = PyUnicode_FromFormat("%U.%U", module, name);
+ if (unlikely(result == NULL)) goto bad;
+ done:
+ Py_XDECREF(name);
+ Py_XDECREF(module);
+ return result;
+ bad:
+ PyErr_Clear();
+ if (name) {
+ result = name;
+ name = NULL;
+ } else {
+ result = __Pyx_NewRef(__pyx_mstate_global->__pyx_kp_u__2);
+ }
+ goto done;
+}
+#endif
+
+/* CIntFromPy */
+static CYTHON_INLINE long __Pyx_PyLong_As_long(PyObject *x) {
+#ifdef __Pyx_HAS_GCC_DIAGNOSTIC
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wconversion"
+#endif
+ const long neg_one = (long) -1, const_zero = (long) 0;
+#ifdef __Pyx_HAS_GCC_DIAGNOSTIC
+#pragma GCC diagnostic pop
+#endif
+ const int is_unsigned = neg_one > const_zero;
+ if (unlikely(!PyLong_Check(x))) {
+ long val;
+ PyObject *tmp = __Pyx_PyNumber_Long(x);
+ if (!tmp) return (long) -1;
+ val = __Pyx_PyLong_As_long(tmp);
+ Py_DECREF(tmp);
+ return val;
+ }
+ if (is_unsigned) {
+#if CYTHON_USE_PYLONG_INTERNALS
+ if (unlikely(__Pyx_PyLong_IsNeg(x))) {
+ goto raise_neg_overflow;
+ } else if (__Pyx_PyLong_IsCompact(x)) {
+ __PYX_VERIFY_RETURN_INT(long, __Pyx_compact_upylong, __Pyx_PyLong_CompactValueUnsigned(x))
+ } else {
+ const digit* digits = __Pyx_PyLong_Digits(x);
+ assert(__Pyx_PyLong_DigitCount(x) > 1);
+ switch (__Pyx_PyLong_DigitCount(x)) {
+ case 2:
+ if ((8 * sizeof(long) > 1 * PyLong_SHIFT)) {
+ if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) {
+ __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])))
+ } else if ((8 * sizeof(long) >= 2 * PyLong_SHIFT)) {
+ return (long) (((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]));
+ }
+ }
+ break;
+ case 3:
+ if ((8 * sizeof(long) > 2 * PyLong_SHIFT)) {
+ if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) {
+ __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])))
+ } else if ((8 * sizeof(long) >= 3 * PyLong_SHIFT)) {
+ return (long) (((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]));
+ }
+ }
+ break;
+ case 4:
+ if ((8 * sizeof(long) > 3 * PyLong_SHIFT)) {
+ if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) {
+ __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])))
+ } else if ((8 * sizeof(long) >= 4 * PyLong_SHIFT)) {
+ return (long) (((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]));
+ }
+ }
+ break;
+ }
+ }
+#endif
+#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A7
+ if (unlikely(Py_SIZE(x) < 0)) {
+ goto raise_neg_overflow;
+ }
+#else
+ {
+ int result = PyObject_RichCompareBool(x, Py_False, Py_LT);
+ if (unlikely(result < 0))
+ return (long) -1;
+ if (unlikely(result == 1))
+ goto raise_neg_overflow;
+ }
+#endif
+ if ((sizeof(long) <= sizeof(unsigned long))) {
+ __PYX_VERIFY_RETURN_INT_EXC(long, unsigned long, PyLong_AsUnsignedLong(x))
+#ifdef HAVE_LONG_LONG
+ } else if ((sizeof(long) <= sizeof(unsigned PY_LONG_LONG))) {
+ __PYX_VERIFY_RETURN_INT_EXC(long, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x))
+#endif
+ }
+ } else {
+#if CYTHON_USE_PYLONG_INTERNALS
+ if (__Pyx_PyLong_IsCompact(x)) {
+ __PYX_VERIFY_RETURN_INT(long, __Pyx_compact_pylong, __Pyx_PyLong_CompactValue(x))
+ } else {
+ const digit* digits = __Pyx_PyLong_Digits(x);
+ assert(__Pyx_PyLong_DigitCount(x) > 1);
+ switch (__Pyx_PyLong_SignedDigitCount(x)) {
+ case -2:
+ if ((8 * sizeof(long) - 1 > 1 * PyLong_SHIFT)) {
+ if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) {
+ __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])))
+ } else if ((8 * sizeof(long) - 1 > 2 * PyLong_SHIFT)) {
+ return (long) (((long)-1)*(((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0])));
+ }
+ }
+ break;
+ case 2:
+ if ((8 * sizeof(long) > 1 * PyLong_SHIFT)) {
+ if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) {
+ __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])))
+ } else if ((8 * sizeof(long) - 1 > 2 * PyLong_SHIFT)) {
+ return (long) ((((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0])));
+ }
+ }
+ break;
+ case -3:
+ if ((8 * sizeof(long) - 1 > 2 * PyLong_SHIFT)) {
+ if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) {
+ __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])))
+ } else if ((8 * sizeof(long) - 1 > 3 * PyLong_SHIFT)) {
+ return (long) (((long)-1)*(((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])));
+ }
+ }
+ break;
+ case 3:
+ if ((8 * sizeof(long) > 2 * PyLong_SHIFT)) {
+ if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) {
+ __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])))
+ } else if ((8 * sizeof(long) - 1 > 3 * PyLong_SHIFT)) {
+ return (long) ((((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])));
+ }
+ }
+ break;
+ case -4:
+ if ((8 * sizeof(long) - 1 > 3 * PyLong_SHIFT)) {
+ if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) {
+ __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])))
+ } else if ((8 * sizeof(long) - 1 > 4 * PyLong_SHIFT)) {
+ return (long) (((long)-1)*(((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])));
+ }
+ }
+ break;
+ case 4:
+ if ((8 * sizeof(long) > 3 * PyLong_SHIFT)) {
+ if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) {
+ __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])))
+ } else if ((8 * sizeof(long) - 1 > 4 * PyLong_SHIFT)) {
+ return (long) ((((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])));
+ }
+ }
+ break;
+ }
+ }
+#endif
+ if ((sizeof(long) <= sizeof(long))) {
+ __PYX_VERIFY_RETURN_INT_EXC(long, long, PyLong_AsLong(x))
+#ifdef HAVE_LONG_LONG
+ } else if ((sizeof(long) <= sizeof(PY_LONG_LONG))) {
+ __PYX_VERIFY_RETURN_INT_EXC(long, PY_LONG_LONG, PyLong_AsLongLong(x))
+#endif
+ }
+ }
+ {
+ long val;
+ int ret = -1;
+#if PY_VERSION_HEX >= 0x030d00A6 && !CYTHON_COMPILING_IN_LIMITED_API
+ Py_ssize_t bytes_copied = PyLong_AsNativeBytes(
+ x, &val, sizeof(val), Py_ASNATIVEBYTES_NATIVE_ENDIAN | (is_unsigned ? Py_ASNATIVEBYTES_UNSIGNED_BUFFER | Py_ASNATIVEBYTES_REJECT_NEGATIVE : 0));
+ if (unlikely(bytes_copied == -1)) {
+ } else if (unlikely(bytes_copied > (Py_ssize_t) sizeof(val))) {
+ goto raise_overflow;
+ } else {
+ ret = 0;
+ }
+#elif PY_VERSION_HEX < 0x030d0000 && !(CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) || defined(_PyLong_AsByteArray)
+ int one = 1; int is_little = (int)*(unsigned char *)&one;
+ unsigned char *bytes = (unsigned char *)&val;
+ ret = _PyLong_AsByteArray((PyLongObject *)x,
+ bytes, sizeof(val),
+ is_little, !is_unsigned);
+#else
+ PyObject *v;
+ PyObject *stepval = NULL, *mask = NULL, *shift = NULL;
+ int bits, remaining_bits, is_negative = 0;
+ int chunk_size = (sizeof(long) < 8) ? 30 : 62;
+ if (likely(PyLong_CheckExact(x))) {
+ v = __Pyx_NewRef(x);
+ } else {
+ v = PyNumber_Long(x);
+ if (unlikely(!v)) return (long) -1;
+ assert(PyLong_CheckExact(v));
+ }
+ {
+ int result = PyObject_RichCompareBool(v, Py_False, Py_LT);
+ if (unlikely(result < 0)) {
+ Py_DECREF(v);
+ return (long) -1;
+ }
+ is_negative = result == 1;
+ }
+ if (is_unsigned && unlikely(is_negative)) {
+ Py_DECREF(v);
+ goto raise_neg_overflow;
+ } else if (is_negative) {
+ stepval = PyNumber_Invert(v);
+ Py_DECREF(v);
+ if (unlikely(!stepval))
+ return (long) -1;
+ } else {
+ stepval = v;
+ }
+ v = NULL;
+ val = (long) 0;
+ mask = PyLong_FromLong((1L << chunk_size) - 1); if (unlikely(!mask)) goto done;
+ shift = PyLong_FromLong(chunk_size); if (unlikely(!shift)) goto done;
+ for (bits = 0; bits < (int) sizeof(long) * 8 - chunk_size; bits += chunk_size) {
+ PyObject *tmp, *digit;
+ long idigit;
+ digit = PyNumber_And(stepval, mask);
+ if (unlikely(!digit)) goto done;
+ idigit = PyLong_AsLong(digit);
+ Py_DECREF(digit);
+ if (unlikely(idigit < 0)) goto done;
+ val |= ((long) idigit) << bits;
+ tmp = PyNumber_Rshift(stepval, shift);
+ if (unlikely(!tmp)) goto done;
+ Py_DECREF(stepval); stepval = tmp;
+ }
+ Py_DECREF(shift); shift = NULL;
+ Py_DECREF(mask); mask = NULL;
+ {
+ long idigit = PyLong_AsLong(stepval);
+ if (unlikely(idigit < 0)) goto done;
+ remaining_bits = ((int) sizeof(long) * 8) - bits - (is_unsigned ? 0 : 1);
+ if (unlikely(idigit >= (1L << remaining_bits)))
+ goto raise_overflow;
+ val |= ((long) idigit) << bits;
+ }
+ if (!is_unsigned) {
+ if (unlikely(val & (((long) 1) << (sizeof(long) * 8 - 1))))
+ goto raise_overflow;
+ if (is_negative)
+ val = ~val;
+ }
+ ret = 0;
+ done:
+ Py_XDECREF(shift);
+ Py_XDECREF(mask);
+ Py_XDECREF(stepval);
+#endif
+ if (unlikely(ret))
+ return (long) -1;
+ return val;
+ }
+raise_overflow:
+ PyErr_SetString(PyExc_OverflowError,
+ "value too large to convert to long");
+ return (long) -1;
+raise_neg_overflow:
+ PyErr_SetString(PyExc_OverflowError,
+ "can't convert negative value to long");
+ return (long) -1;
+}
+
+/* FastTypeChecks */
+#if CYTHON_COMPILING_IN_CPYTHON
+static int __Pyx_InBases(PyTypeObject *a, PyTypeObject *b) {
+ while (a) {
+ a = __Pyx_PyType_GetSlot(a, tp_base, PyTypeObject*);
+ if (a == b)
+ return 1;
+ }
+ return b == &PyBaseObject_Type;
+}
+static CYTHON_INLINE int __Pyx_IsSubtype(PyTypeObject *a, PyTypeObject *b) {
+ PyObject *mro;
+ if (a == b) return 1;
+ mro = a->tp_mro;
+ if (likely(mro)) {
+ Py_ssize_t i, n;
+ n = PyTuple_GET_SIZE(mro);
+ for (i = 0; i < n; i++) {
+ if (PyTuple_GET_ITEM(mro, i) == (PyObject *)b)
+ return 1;
+ }
+ return 0;
+ }
+ return __Pyx_InBases(a, b);
+}
+static CYTHON_INLINE int __Pyx_IsAnySubtype2(PyTypeObject *cls, PyTypeObject *a, PyTypeObject *b) {
+ PyObject *mro;
+ if (cls == a || cls == b) return 1;
+ mro = cls->tp_mro;
+ if (likely(mro)) {
+ Py_ssize_t i, n;
+ n = PyTuple_GET_SIZE(mro);
+ for (i = 0; i < n; i++) {
+ PyObject *base = PyTuple_GET_ITEM(mro, i);
+ if (base == (PyObject *)a || base == (PyObject *)b)
+ return 1;
+ }
+ return 0;
+ }
+ return __Pyx_InBases(cls, a) || __Pyx_InBases(cls, b);
+}
+static CYTHON_INLINE int __Pyx_inner_PyErr_GivenExceptionMatches2(PyObject *err, PyObject* exc_type1, PyObject *exc_type2) {
+ if (exc_type1) {
+ return __Pyx_IsAnySubtype2((PyTypeObject*)err, (PyTypeObject*)exc_type1, (PyTypeObject*)exc_type2);
+ } else {
+ return __Pyx_IsSubtype((PyTypeObject*)err, (PyTypeObject*)exc_type2);
+ }
+}
+static int __Pyx_PyErr_GivenExceptionMatchesTuple(PyObject *exc_type, PyObject *tuple) {
+ Py_ssize_t i, n;
+ assert(PyExceptionClass_Check(exc_type));
+ n = PyTuple_GET_SIZE(tuple);
+ for (i=0; i= 0x030B00a4
+ _PyErr_StackItem *exc_info = tstate->exc_info;
+ tmp_value = exc_info->exc_value;
+ exc_info->exc_value = *value;
+ if (tmp_value == NULL || tmp_value == Py_None) {
+ Py_XDECREF(tmp_value);
+ tmp_value = NULL;
+ tmp_type = NULL;
+ tmp_tb = NULL;
+ } else {
+ tmp_type = (PyObject*) Py_TYPE(tmp_value);
+ Py_INCREF(tmp_type);
+ #if CYTHON_COMPILING_IN_CPYTHON
+ tmp_tb = ((PyBaseExceptionObject*) tmp_value)->traceback;
+ Py_XINCREF(tmp_tb);
+ #else
+ tmp_tb = PyException_GetTraceback(tmp_value);
+ #endif
+ }
+ #elif CYTHON_USE_EXC_INFO_STACK
+ _PyErr_StackItem *exc_info = tstate->exc_info;
+ tmp_type = exc_info->exc_type;
+ tmp_value = exc_info->exc_value;
+ tmp_tb = exc_info->exc_traceback;
+ exc_info->exc_type = *type;
+ exc_info->exc_value = *value;
+ exc_info->exc_traceback = *tb;
+ #else
+ tmp_type = tstate->exc_type;
+ tmp_value = tstate->exc_value;
+ tmp_tb = tstate->exc_traceback;
+ tstate->exc_type = *type;
+ tstate->exc_value = *value;
+ tstate->exc_traceback = *tb;
+ #endif
+ *type = tmp_type;
+ *value = tmp_value;
+ *tb = tmp_tb;
+}
+#else
+static CYTHON_INLINE void __Pyx_ExceptionSwap(PyObject **type, PyObject **value, PyObject **tb) {
+ PyObject *tmp_type, *tmp_value, *tmp_tb;
+ PyErr_GetExcInfo(&tmp_type, &tmp_value, &tmp_tb);
+ PyErr_SetExcInfo(*type, *value, *tb);
+ *type = tmp_type;
+ *value = tmp_value;
+ *tb = tmp_tb;
+}
+#endif
+
+/* PyObjectCall2Args */
+static CYTHON_INLINE PyObject* __Pyx_PyObject_Call2Args(PyObject* function, PyObject* arg1, PyObject* arg2) {
+ PyObject *args[3] = {NULL, arg1, arg2};
+ return __Pyx_PyObject_FastCall(function, args+1, 2 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET);
+}
+
+/* PyObjectCallMethod1 */
+#if !(CYTHON_VECTORCALL && (__PYX_LIMITED_VERSION_HEX >= 0x030C0000 || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x03090000)))
+static PyObject* __Pyx__PyObject_CallMethod1(PyObject* method, PyObject* arg) {
+ PyObject *result = __Pyx_PyObject_CallOneArg(method, arg);
+ Py_DECREF(method);
+ return result;
+}
+#endif
+static PyObject* __Pyx_PyObject_CallMethod1(PyObject* obj, PyObject* method_name, PyObject* arg) {
+#if CYTHON_VECTORCALL && (__PYX_LIMITED_VERSION_HEX >= 0x030C0000 || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x03090000))
+ PyObject *args[2] = {obj, arg};
+ (void) __Pyx_PyObject_GetMethod;
+ (void) __Pyx_PyObject_CallOneArg;
+ (void) __Pyx_PyObject_Call2Args;
+ return PyObject_VectorcallMethod(method_name, args, 2 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL);
+#else
+ PyObject *method = NULL, *result;
+ int is_method = __Pyx_PyObject_GetMethod(obj, method_name, &method);
+ if (likely(is_method)) {
+ result = __Pyx_PyObject_Call2Args(method, obj, arg);
+ Py_DECREF(method);
+ return result;
+ }
+ if (unlikely(!method)) return NULL;
+ return __Pyx__PyObject_CallMethod1(method, arg);
+#endif
+}
+
+/* ReturnWithStopIteration */
+static void __Pyx__ReturnWithStopIteration(PyObject* value, int async);
+static CYTHON_INLINE void __Pyx_ReturnWithStopIteration(PyObject* value, int async, int iternext) {
+ if (value == Py_None) {
+ if (async || !iternext)
+ PyErr_SetNone(async ? PyExc_StopAsyncIteration : PyExc_StopIteration);
+ return;
+ }
+ __Pyx__ReturnWithStopIteration(value, async);
+}
+static void __Pyx__ReturnWithStopIteration(PyObject* value, int async) {
+#if CYTHON_COMPILING_IN_CPYTHON
+ __Pyx_PyThreadState_declare
+#endif
+ PyObject *exc;
+ PyObject *exc_type = async ? PyExc_StopAsyncIteration : PyExc_StopIteration;
+#if CYTHON_COMPILING_IN_CPYTHON
+ if ((PY_VERSION_HEX >= (0x030C00A6)) || unlikely(PyTuple_Check(value) || PyExceptionInstance_Check(value))) {
+ if (PY_VERSION_HEX >= (0x030e00A1)) {
+ exc = __Pyx_PyObject_CallOneArg(exc_type, value);
+ } else {
+ PyObject *args_tuple = PyTuple_New(1);
+ if (unlikely(!args_tuple)) return;
+ Py_INCREF(value);
+ PyTuple_SET_ITEM(args_tuple, 0, value);
+ exc = PyObject_Call(exc_type, args_tuple, NULL);
+ Py_DECREF(args_tuple);
+ }
+ if (unlikely(!exc)) return;
+ } else {
+ Py_INCREF(value);
+ exc = value;
+ }
+ #if CYTHON_FAST_THREAD_STATE
+ __Pyx_PyThreadState_assign
+ #if CYTHON_USE_EXC_INFO_STACK
+ if (!__pyx_tstate->exc_info->exc_value)
+ #else
+ if (!__pyx_tstate->exc_type)
+ #endif
+ {
+ Py_INCREF(exc_type);
+ __Pyx_ErrRestore(exc_type, exc, NULL);
+ return;
+ }
+ #endif
+#else
+ exc = __Pyx_PyObject_CallOneArg(exc_type, value);
+ if (unlikely(!exc)) return;
+#endif
+ PyErr_SetObject(exc_type, exc);
+ Py_DECREF(exc);
+}
+
+/* CoroutineBase */
+#if !CYTHON_COMPILING_IN_LIMITED_API
+#include
+#if PY_VERSION_HEX >= 0x030b00a6 && !defined(PYPY_VERSION)
+ #ifndef Py_BUILD_CORE
+ #define Py_BUILD_CORE 1
+ #endif
+ #include "internal/pycore_frame.h"
+#endif
+#endif // CYTHON_COMPILING_IN_LIMITED_API
+static CYTHON_INLINE void
+__Pyx_Coroutine_Undelegate(__pyx_CoroutineObject *gen) {
+#if CYTHON_USE_AM_SEND
+ gen->yieldfrom_am_send = NULL;
+#endif
+ Py_CLEAR(gen->yieldfrom);
+}
+static int __Pyx_PyGen__FetchStopIterationValue(PyThreadState *__pyx_tstate, PyObject **pvalue) {
+ PyObject *et, *ev, *tb;
+ PyObject *value = NULL;
+ CYTHON_UNUSED_VAR(__pyx_tstate);
+ __Pyx_ErrFetch(&et, &ev, &tb);
+ if (!et) {
+ Py_XDECREF(tb);
+ Py_XDECREF(ev);
+ Py_INCREF(Py_None);
+ *pvalue = Py_None;
+ return 0;
+ }
+ if (likely(et == PyExc_StopIteration)) {
+ if (!ev) {
+ Py_INCREF(Py_None);
+ value = Py_None;
+ }
+ else if (likely(__Pyx_IS_TYPE(ev, (PyTypeObject*)PyExc_StopIteration))) {
+ #if CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_GRAAL
+ value = PyObject_GetAttr(ev, __pyx_mstate_global->__pyx_n_u_value);
+ if (unlikely(!value)) goto limited_api_failure;
+ #else
+ value = ((PyStopIterationObject *)ev)->value;
+ Py_INCREF(value);
+ #endif
+ Py_DECREF(ev);
+ }
+ else if (unlikely(PyTuple_Check(ev))) {
+ Py_ssize_t tuple_size = __Pyx_PyTuple_GET_SIZE(ev);
+ #if !CYTHON_ASSUME_SAFE_SIZE
+ if (unlikely(tuple_size < 0)) {
+ Py_XDECREF(tb);
+ Py_DECREF(ev);
+ Py_DECREF(et);
+ return -1;
+ }
+ #endif
+ if (tuple_size >= 1) {
+#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS
+ value = PyTuple_GET_ITEM(ev, 0);
+ Py_INCREF(value);
+#elif CYTHON_ASSUME_SAFE_MACROS
+ value = PySequence_ITEM(ev, 0);
+#else
+ value = PySequence_GetItem(ev, 0);
+ if (!value) goto limited_api_failure;
+#endif
+ } else {
+ Py_INCREF(Py_None);
+ value = Py_None;
+ }
+ Py_DECREF(ev);
+ }
+ else if (!__Pyx_TypeCheck(ev, (PyTypeObject*)PyExc_StopIteration)) {
+ value = ev;
+ }
+ if (likely(value)) {
+ Py_XDECREF(tb);
+ Py_DECREF(et);
+ *pvalue = value;
+ return 0;
+ }
+ } else if (!__Pyx_PyErr_GivenExceptionMatches(et, PyExc_StopIteration)) {
+ __Pyx_ErrRestore(et, ev, tb);
+ return -1;
+ }
+ PyErr_NormalizeException(&et, &ev, &tb);
+ if (unlikely(!PyObject_TypeCheck(ev, (PyTypeObject*)PyExc_StopIteration))) {
+ __Pyx_ErrRestore(et, ev, tb);
+ return -1;
+ }
+ Py_XDECREF(tb);
+ Py_DECREF(et);
+#if CYTHON_COMPILING_IN_LIMITED_API
+ value = PyObject_GetAttr(ev, __pyx_mstate_global->__pyx_n_u_value);
+#else
+ value = ((PyStopIterationObject *)ev)->value;
+ Py_INCREF(value);
+#endif
+ Py_DECREF(ev);
+#if CYTHON_COMPILING_IN_LIMITED_API
+ if (unlikely(!value)) return -1;
+#endif
+ *pvalue = value;
+ return 0;
+#if CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_GRAAL || !CYTHON_ASSUME_SAFE_MACROS
+ limited_api_failure:
+ Py_XDECREF(et);
+ Py_XDECREF(tb);
+ Py_XDECREF(ev);
+ return -1;
+#endif
+}
+static CYTHON_INLINE
+__Pyx_PySendResult __Pyx_Coroutine_status_from_result(PyObject **retval) {
+ if (*retval) {
+ return PYGEN_NEXT;
+ } else if (likely(__Pyx_PyGen__FetchStopIterationValue(__Pyx_PyThreadState_Current, retval) == 0)) {
+ return PYGEN_RETURN;
+ } else {
+ return PYGEN_ERROR;
+ }
+}
+static CYTHON_INLINE
+void __Pyx_Coroutine_ExceptionClear(__Pyx_ExcInfoStruct *exc_state) {
+#if PY_VERSION_HEX >= 0x030B00a4
+ Py_CLEAR(exc_state->exc_value);
+#else
+ PyObject *t, *v, *tb;
+ t = exc_state->exc_type;
+ v = exc_state->exc_value;
+ tb = exc_state->exc_traceback;
+ exc_state->exc_type = NULL;
+ exc_state->exc_value = NULL;
+ exc_state->exc_traceback = NULL;
+ Py_XDECREF(t);
+ Py_XDECREF(v);
+ Py_XDECREF(tb);
+#endif
+}
+#define __Pyx_Coroutine_AlreadyRunningError(gen) (__Pyx__Coroutine_AlreadyRunningError(gen), (PyObject*)NULL)
+static void __Pyx__Coroutine_AlreadyRunningError(__pyx_CoroutineObject *gen) {
+ const char *msg;
+ CYTHON_MAYBE_UNUSED_VAR(gen);
+ if ((0)) {
+ #ifdef __Pyx_Coroutine_USED
+ } else if (__Pyx_Coroutine_Check((PyObject*)gen)) {
+ msg = "coroutine already executing";
+ #endif
+ #ifdef __Pyx_AsyncGen_USED
+ } else if (__Pyx_AsyncGen_CheckExact((PyObject*)gen)) {
+ msg = "async generator already executing";
+ #endif
+ } else {
+ msg = "generator already executing";
+ }
+ PyErr_SetString(PyExc_ValueError, msg);
+}
+static void __Pyx_Coroutine_AlreadyTerminatedError(PyObject *gen, PyObject *value, int closing) {
+ CYTHON_MAYBE_UNUSED_VAR(gen);
+ CYTHON_MAYBE_UNUSED_VAR(closing);
+ #ifdef __Pyx_Coroutine_USED
+ if (!closing && __Pyx_Coroutine_Check(gen)) {
+ PyErr_SetString(PyExc_RuntimeError, "cannot reuse already awaited coroutine");
+ } else
+ #endif
+ if (value) {
+ #ifdef __Pyx_AsyncGen_USED
+ if (__Pyx_AsyncGen_CheckExact(gen))
+ PyErr_SetNone(PyExc_StopAsyncIteration);
+ else
+ #endif
+ PyErr_SetNone(PyExc_StopIteration);
+ }
+}
+static
+__Pyx_PySendResult __Pyx_Coroutine_SendEx(__pyx_CoroutineObject *self, PyObject *value, PyObject **result, int closing) {
+ __Pyx_PyThreadState_declare
+ PyThreadState *tstate;
+ __Pyx_ExcInfoStruct *exc_state;
+ PyObject *retval;
+ assert(__Pyx_Coroutine_get_is_running(self)); // Callers should ensure is_running
+ if (unlikely(self->resume_label == -1)) {
+ __Pyx_Coroutine_AlreadyTerminatedError((PyObject*)self, value, closing);
+ return PYGEN_ERROR;
+ }
+#if CYTHON_FAST_THREAD_STATE
+ __Pyx_PyThreadState_assign
+ tstate = __pyx_tstate;
+#else
+ tstate = __Pyx_PyThreadState_Current;
+#endif
+ exc_state = &self->gi_exc_state;
+ if (exc_state->exc_value) {
+ #if CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_PYPY
+ #else
+ PyObject *exc_tb;
+ #if PY_VERSION_HEX >= 0x030B00a4 && !CYTHON_COMPILING_IN_CPYTHON
+ exc_tb = PyException_GetTraceback(exc_state->exc_value);
+ #elif PY_VERSION_HEX >= 0x030B00a4
+ exc_tb = ((PyBaseExceptionObject*) exc_state->exc_value)->traceback;
+ #else
+ exc_tb = exc_state->exc_traceback;
+ #endif
+ if (exc_tb) {
+ PyTracebackObject *tb = (PyTracebackObject *) exc_tb;
+ PyFrameObject *f = tb->tb_frame;
+ assert(f->f_back == NULL);
+ #if PY_VERSION_HEX >= 0x030B00A1
+ f->f_back = PyThreadState_GetFrame(tstate);
+ #else
+ Py_XINCREF(tstate->frame);
+ f->f_back = tstate->frame;
+ #endif
+ #if PY_VERSION_HEX >= 0x030B00a4 && !CYTHON_COMPILING_IN_CPYTHON
+ Py_DECREF(exc_tb);
+ #endif
+ }
+ #endif
+ }
+#if CYTHON_USE_EXC_INFO_STACK
+ exc_state->previous_item = tstate->exc_info;
+ tstate->exc_info = exc_state;
+#else
+ if (exc_state->exc_type) {
+ __Pyx_ExceptionSwap(&exc_state->exc_type, &exc_state->exc_value, &exc_state->exc_traceback);
+ } else {
+ __Pyx_Coroutine_ExceptionClear(exc_state);
+ __Pyx_ExceptionSave(&exc_state->exc_type, &exc_state->exc_value, &exc_state->exc_traceback);
+ }
+#endif
+ retval = self->body(self, tstate, value);
+#if CYTHON_USE_EXC_INFO_STACK
+ exc_state = &self->gi_exc_state;
+ tstate->exc_info = exc_state->previous_item;
+ exc_state->previous_item = NULL;
+ __Pyx_Coroutine_ResetFrameBackpointer(exc_state);
+#endif
+ *result = retval;
+ if (self->resume_label == -1) {
+ return likely(retval) ? PYGEN_RETURN : PYGEN_ERROR;
+ }
+ return PYGEN_NEXT;
+}
+static CYTHON_INLINE void __Pyx_Coroutine_ResetFrameBackpointer(__Pyx_ExcInfoStruct *exc_state) {
+#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API
+ CYTHON_UNUSED_VAR(exc_state);
+#else
+ PyObject *exc_tb;
+ #if PY_VERSION_HEX >= 0x030B00a4
+ if (!exc_state->exc_value) return;
+ exc_tb = PyException_GetTraceback(exc_state->exc_value);
+ #else
+ exc_tb = exc_state->exc_traceback;
+ #endif
+ if (likely(exc_tb)) {
+ PyTracebackObject *tb = (PyTracebackObject *) exc_tb;
+ PyFrameObject *f = tb->tb_frame;
+ Py_CLEAR(f->f_back);
+ #if PY_VERSION_HEX >= 0x030B00a4
+ Py_DECREF(exc_tb);
+ #endif
+ }
+#endif
+}
+#define __Pyx_Coroutine_MethodReturnFromResult(gen, result, retval, iternext)\
+ ((result) == PYGEN_NEXT ? (retval) : __Pyx__Coroutine_MethodReturnFromResult(gen, result, retval, iternext))
+static PyObject *
+__Pyx__Coroutine_MethodReturnFromResult(PyObject* gen, __Pyx_PySendResult result, PyObject *retval, int iternext) {
+ CYTHON_MAYBE_UNUSED_VAR(gen);
+ if (likely(result == PYGEN_RETURN)) {
+ int is_async = 0;
+ #ifdef __Pyx_AsyncGen_USED
+ is_async = __Pyx_AsyncGen_CheckExact(gen);
+ #endif
+ __Pyx_ReturnWithStopIteration(retval, is_async, iternext);
+ Py_XDECREF(retval);
+ }
+ return NULL;
+}
+#if CYTHON_COMPILING_IN_CPYTHON
+static CYTHON_INLINE
+PyObject *__Pyx_PyGen_Send(PyGenObject *gen, PyObject *arg) {
+#if PY_VERSION_HEX <= 0x030A00A1
+ return _PyGen_Send(gen, arg);
+#else
+ PyObject *result;
+ if (PyIter_Send((PyObject*)gen, arg ? arg : Py_None, &result) == PYGEN_RETURN) {
+ if (PyAsyncGen_CheckExact(gen)) {
+ assert(result == Py_None);
+ PyErr_SetNone(PyExc_StopAsyncIteration);
+ }
+ else if (result == Py_None) {
+ PyErr_SetNone(PyExc_StopIteration);
+ }
+ else {
+#if PY_VERSION_HEX < 0x030d00A1
+ _PyGen_SetStopIterationValue(result);
+#else
+ if (!PyTuple_Check(result) && !PyExceptionInstance_Check(result)) {
+ PyErr_SetObject(PyExc_StopIteration, result);
+ } else {
+ PyObject *exc = __Pyx_PyObject_CallOneArg(PyExc_StopIteration, result);
+ if (likely(exc != NULL)) {
+ PyErr_SetObject(PyExc_StopIteration, exc);
+ Py_DECREF(exc);
+ }
+ }
+#endif
+ }
+ Py_DECREF(result);
+ result = NULL;
+ }
+ return result;
+#endif
+}
+#endif
+static CYTHON_INLINE __Pyx_PySendResult
+__Pyx_Coroutine_FinishDelegation(__pyx_CoroutineObject *gen, PyObject** retval) {
+ __Pyx_PySendResult result;
+ PyObject *val = NULL;
+ assert(__Pyx_Coroutine_get_is_running(gen));
+ __Pyx_Coroutine_Undelegate(gen);
+ __Pyx_PyGen__FetchStopIterationValue(__Pyx_PyThreadState_Current, &val);
+ result = __Pyx_Coroutine_SendEx(gen, val, retval, 0);
+ Py_XDECREF(val);
+ return result;
+}
+#if CYTHON_USE_AM_SEND
+static __Pyx_PySendResult
+__Pyx_Coroutine_SendToDelegate(__pyx_CoroutineObject *gen, __Pyx_pyiter_sendfunc gen_am_send, PyObject *value, PyObject **retval) {
+ PyObject *ret = NULL;
+ __Pyx_PySendResult delegate_result, result;
+ assert(__Pyx_Coroutine_get_is_running(gen));
+ delegate_result = gen_am_send(gen->yieldfrom, value, &ret);
+ if (delegate_result == PYGEN_NEXT) {
+ assert (ret != NULL);
+ *retval = ret;
+ return PYGEN_NEXT;
+ }
+ assert (delegate_result != PYGEN_ERROR || ret == NULL);
+ __Pyx_Coroutine_Undelegate(gen);
+ result = __Pyx_Coroutine_SendEx(gen, ret, retval, 0);
+ Py_XDECREF(ret);
+ return result;
+}
+#endif
+static PyObject *__Pyx_Coroutine_Send(PyObject *self, PyObject *value) {
+ PyObject *retval = NULL;
+ __Pyx_PySendResult result = __Pyx_Coroutine_AmSend(self, value, &retval);
+ return __Pyx_Coroutine_MethodReturnFromResult(self, result, retval, 0);
+}
+static __Pyx_PySendResult
+__Pyx_Coroutine_AmSend(PyObject *self, PyObject *value, PyObject **retval) {
+ __Pyx_PySendResult result;
+ __pyx_CoroutineObject *gen = (__pyx_CoroutineObject*) self;
+ if (unlikely(__Pyx_Coroutine_test_and_set_is_running(gen))) {
+ *retval = __Pyx_Coroutine_AlreadyRunningError(gen);
+ return PYGEN_ERROR;
+ }
+ #if CYTHON_USE_AM_SEND
+ if (gen->yieldfrom_am_send) {
+ result = __Pyx_Coroutine_SendToDelegate(gen, gen->yieldfrom_am_send, value, retval);
+ } else
+ #endif
+ if (gen->yieldfrom) {
+ PyObject *yf = gen->yieldfrom;
+ PyObject *ret;
+ #if !CYTHON_USE_AM_SEND
+ #ifdef __Pyx_Generator_USED
+ if (__Pyx_Generator_CheckExact(yf)) {
+ ret = __Pyx_Coroutine_Send(yf, value);
+ } else
+ #endif
+ #ifdef __Pyx_Coroutine_USED
+ if (__Pyx_Coroutine_Check(yf)) {
+ ret = __Pyx_Coroutine_Send(yf, value);
+ } else
+ #endif
+ #ifdef __Pyx_AsyncGen_USED
+ if (__pyx_PyAsyncGenASend_CheckExact(yf)) {
+ ret = __Pyx_async_gen_asend_send(yf, value);
+ } else
+ #endif
+ #if CYTHON_COMPILING_IN_CPYTHON
+ if (PyGen_CheckExact(yf)) {
+ ret = __Pyx_PyGen_Send((PyGenObject*)yf, value == Py_None ? NULL : value);
+ } else
+ if (PyCoro_CheckExact(yf)) {
+ ret = __Pyx_PyGen_Send((PyGenObject*)yf, value == Py_None ? NULL : value);
+ } else
+ #endif
+ #endif
+ {
+ #if !CYTHON_COMPILING_IN_LIMITED_API || __PYX_LIMITED_VERSION_HEX >= 0x03080000
+ if (value == Py_None && PyIter_Check(yf))
+ ret = __Pyx_PyIter_Next_Plain(yf);
+ else
+ #endif
+ ret = __Pyx_PyObject_CallMethod1(yf, __pyx_mstate_global->__pyx_n_u_send, value);
+ }
+ if (likely(ret)) {
+ __Pyx_Coroutine_unset_is_running(gen);
+ *retval = ret;
+ return PYGEN_NEXT;
+ }
+ result = __Pyx_Coroutine_FinishDelegation(gen, retval);
+ } else {
+ result = __Pyx_Coroutine_SendEx(gen, value, retval, 0);
+ }
+ __Pyx_Coroutine_unset_is_running(gen);
+ return result;
+}
+static int __Pyx_Coroutine_CloseIter(__pyx_CoroutineObject *gen, PyObject *yf) {
+ __Pyx_PySendResult result;
+ PyObject *retval = NULL;
+ CYTHON_UNUSED_VAR(gen);
+ assert(__Pyx_Coroutine_get_is_running(gen));
+ #ifdef __Pyx_Generator_USED
+ if (__Pyx_Generator_CheckExact(yf)) {
+ result = __Pyx_Coroutine_Close(yf, &retval);
+ } else
+ #endif
+ #ifdef __Pyx_Coroutine_USED
+ if (__Pyx_Coroutine_Check(yf)) {
+ result = __Pyx_Coroutine_Close(yf, &retval);
+ } else
+ if (__Pyx_CoroutineAwait_CheckExact(yf)) {
+ result = __Pyx_CoroutineAwait_Close((__pyx_CoroutineAwaitObject*)yf);
+ } else
+ #endif
+ #ifdef __Pyx_AsyncGen_USED
+ if (__pyx_PyAsyncGenASend_CheckExact(yf)) {
+ retval = __Pyx_async_gen_asend_close(yf, NULL);
+ result = PYGEN_RETURN;
+ } else
+ if (__pyx_PyAsyncGenAThrow_CheckExact(yf)) {
+ retval = __Pyx_async_gen_athrow_close(yf, NULL);
+ result = PYGEN_RETURN;
+ } else
+ #endif
+ {
+ PyObject *meth;
+ result = PYGEN_RETURN;
+ meth = __Pyx_PyObject_GetAttrStrNoError(yf, __pyx_mstate_global->__pyx_n_u_close);
+ if (unlikely(!meth)) {
+ if (unlikely(PyErr_Occurred())) {
+ PyErr_WriteUnraisable(yf);
+ }
+ } else {
+ retval = __Pyx_PyObject_CallNoArg(meth);
+ Py_DECREF(meth);
+ if (unlikely(!retval)) {
+ result = PYGEN_ERROR;
+ }
+ }
+ }
+ Py_XDECREF(retval);
+ return result == PYGEN_ERROR ? -1 : 0;
+}
+static PyObject *__Pyx_Generator_Next(PyObject *self) {
+ __Pyx_PySendResult result;
+ PyObject *retval = NULL;
+ __pyx_CoroutineObject *gen = (__pyx_CoroutineObject*) self;
+ if (unlikely(__Pyx_Coroutine_test_and_set_is_running(gen))) {
+ return __Pyx_Coroutine_AlreadyRunningError(gen);
+ }
+ #if CYTHON_USE_AM_SEND
+ if (gen->yieldfrom_am_send) {
+ result = __Pyx_Coroutine_SendToDelegate(gen, gen->yieldfrom_am_send, Py_None, &retval);
+ } else
+ #endif
+ if (gen->yieldfrom) {
+ PyObject *yf = gen->yieldfrom;
+ PyObject *ret;
+ #ifdef __Pyx_Generator_USED
+ if (__Pyx_Generator_CheckExact(yf)) {
+ ret = __Pyx_Generator_Next(yf);
+ } else
+ #endif
+ #ifdef __Pyx_Coroutine_USED
+ if (__Pyx_Coroutine_CheckExact(yf)) {
+ ret = __Pyx_Coroutine_Send(yf, Py_None);
+ } else
+ #endif
+ #if CYTHON_COMPILING_IN_CPYTHON && (PY_VERSION_HEX < 0x030A00A3 || !CYTHON_USE_AM_SEND)
+ if (PyGen_CheckExact(yf)) {
+ ret = __Pyx_PyGen_Send((PyGenObject*)yf, NULL);
+ } else
+ #endif
+ ret = __Pyx_PyIter_Next_Plain(yf);
+ if (likely(ret)) {
+ __Pyx_Coroutine_unset_is_running(gen);
+ return ret;
+ }
+ result = __Pyx_Coroutine_FinishDelegation(gen, &retval);
+ } else {
+ result = __Pyx_Coroutine_SendEx(gen, Py_None, &retval, 0);
+ }
+ __Pyx_Coroutine_unset_is_running(gen);
+ return __Pyx_Coroutine_MethodReturnFromResult(self, result, retval, 1);
+}
+static PyObject *__Pyx_Coroutine_Close_Method(PyObject *self, PyObject *arg) {
+ PyObject *retval = NULL;
+ __Pyx_PySendResult result;
+ CYTHON_UNUSED_VAR(arg);
+ result = __Pyx_Coroutine_Close(self, &retval);
+ if (unlikely(result == PYGEN_ERROR))
+ return NULL;
+ Py_XDECREF(retval);
+ Py_RETURN_NONE;
+}
+static __Pyx_PySendResult
+__Pyx_Coroutine_Close(PyObject *self, PyObject **retval) {
+ __pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self;
+ __Pyx_PySendResult result;
+ PyObject *yf;
+ int err = 0;
+ if (unlikely(__Pyx_Coroutine_test_and_set_is_running(gen))) {
+ *retval = __Pyx_Coroutine_AlreadyRunningError(gen);
+ return PYGEN_ERROR;
+ }
+ yf = gen->yieldfrom;
+ if (yf) {
+ Py_INCREF(yf);
+ err = __Pyx_Coroutine_CloseIter(gen, yf);
+ __Pyx_Coroutine_Undelegate(gen);
+ Py_DECREF(yf);
+ }
+ if (err == 0)
+ PyErr_SetNone(PyExc_GeneratorExit);
+ result = __Pyx_Coroutine_SendEx(gen, NULL, retval, 1);
+ if (result == PYGEN_ERROR) {
+ __Pyx_PyThreadState_declare
+ __Pyx_PyThreadState_assign
+ __Pyx_Coroutine_unset_is_running(gen);
+ if (!__Pyx_PyErr_Occurred()) {
+ return PYGEN_RETURN;
+ } else if (likely(__Pyx_PyErr_ExceptionMatches2(PyExc_GeneratorExit, PyExc_StopIteration))) {
+ __Pyx_PyErr_Clear();
+ return PYGEN_RETURN;
+ }
+ return PYGEN_ERROR;
+ } else if (likely(result == PYGEN_RETURN && *retval == Py_None)) {
+ __Pyx_Coroutine_unset_is_running(gen);
+ return PYGEN_RETURN;
+ } else {
+ const char *msg;
+ Py_DECREF(*retval);
+ *retval = NULL;
+ if ((0)) {
+ #ifdef __Pyx_Coroutine_USED
+ } else if (__Pyx_Coroutine_Check(self)) {
+ msg = "coroutine ignored GeneratorExit";
+ #endif
+ #ifdef __Pyx_AsyncGen_USED
+ } else if (__Pyx_AsyncGen_CheckExact(self)) {
+ msg = "async generator ignored GeneratorExit";
+ #endif
+ } else {
+ msg = "generator ignored GeneratorExit";
+ }
+ PyErr_SetString(PyExc_RuntimeError, msg);
+ __Pyx_Coroutine_unset_is_running(gen);
+ return PYGEN_ERROR;
+ }
+}
+static PyObject *__Pyx__Coroutine_Throw(PyObject *self, PyObject *typ, PyObject *val, PyObject *tb,
+ PyObject *args, int close_on_genexit) {
+ __pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self;
+ PyObject *yf;
+ if (unlikely(__Pyx_Coroutine_test_and_set_is_running(gen)))
+ return __Pyx_Coroutine_AlreadyRunningError(gen);
+ yf = gen->yieldfrom;
+ if (yf) {
+ __Pyx_PySendResult result;
+ PyObject *ret;
+ Py_INCREF(yf);
+ if (__Pyx_PyErr_GivenExceptionMatches(typ, PyExc_GeneratorExit) && close_on_genexit) {
+ int err = __Pyx_Coroutine_CloseIter(gen, yf);
+ Py_DECREF(yf);
+ __Pyx_Coroutine_Undelegate(gen);
+ if (err < 0)
+ goto propagate_exception;
+ goto throw_here;
+ }
+ if (0
+ #ifdef __Pyx_Generator_USED
+ || __Pyx_Generator_CheckExact(yf)
+ #endif
+ #ifdef __Pyx_Coroutine_USED
+ || __Pyx_Coroutine_Check(yf)
+ #endif
+ ) {
+ ret = __Pyx__Coroutine_Throw(yf, typ, val, tb, args, close_on_genexit);
+ #ifdef __Pyx_Coroutine_USED
+ } else if (__Pyx_CoroutineAwait_CheckExact(yf)) {
+ ret = __Pyx__Coroutine_Throw(((__pyx_CoroutineAwaitObject*)yf)->coroutine, typ, val, tb, args, close_on_genexit);
+ #endif
+ } else {
+ PyObject *meth = __Pyx_PyObject_GetAttrStrNoError(yf, __pyx_mstate_global->__pyx_n_u_throw);
+ if (unlikely(!meth)) {
+ Py_DECREF(yf);
+ if (unlikely(PyErr_Occurred())) {
+ __Pyx_Coroutine_unset_is_running(gen);
+ return NULL;
+ }
+ __Pyx_Coroutine_Undelegate(gen);
+ goto throw_here;
+ }
+ if (likely(args)) {
+ ret = __Pyx_PyObject_Call(meth, args, NULL);
+ } else {
+ PyObject *cargs[4] = {NULL, typ, val, tb};
+ ret = __Pyx_PyObject_FastCall(meth, cargs+1, 3 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET);
+ }
+ Py_DECREF(meth);
+ }
+ Py_DECREF(yf);
+ if (ret) {
+ __Pyx_Coroutine_unset_is_running(gen);
+ return ret;
+ }
+ result = __Pyx_Coroutine_FinishDelegation(gen, &ret);
+ __Pyx_Coroutine_unset_is_running(gen);
+ return __Pyx_Coroutine_MethodReturnFromResult(self, result, ret, 0);
+ }
+throw_here:
+ __Pyx_Raise(typ, val, tb, NULL);
+propagate_exception:
+ {
+ PyObject *retval = NULL;
+ __Pyx_PySendResult result = __Pyx_Coroutine_SendEx(gen, NULL, &retval, 0);
+ __Pyx_Coroutine_unset_is_running(gen);
+ return __Pyx_Coroutine_MethodReturnFromResult(self, result, retval, 0);
+ }
+}
+static PyObject *__Pyx_Coroutine_Throw(PyObject *self, PyObject *args) {
+ PyObject *typ;
+ PyObject *val = NULL;
+ PyObject *tb = NULL;
+ if (unlikely(!PyArg_UnpackTuple(args, "throw", 1, 3, &typ, &val, &tb)))
+ return NULL;
+ return __Pyx__Coroutine_Throw(self, typ, val, tb, args, 1);
+}
+static CYTHON_INLINE int __Pyx_Coroutine_traverse_excstate(__Pyx_ExcInfoStruct *exc_state, visitproc visit, void *arg) {
+#if PY_VERSION_HEX >= 0x030B00a4
+ Py_VISIT(exc_state->exc_value);
+#else
+ Py_VISIT(exc_state->exc_type);
+ Py_VISIT(exc_state->exc_value);
+ Py_VISIT(exc_state->exc_traceback);
+#endif
+ return 0;
+}
+static int __Pyx_Coroutine_traverse(__pyx_CoroutineObject *gen, visitproc visit, void *arg) {
+ {
+ int e = __Pyx_call_type_traverse((PyObject*)gen, 1, visit, arg);
+ if (e) return e;
+ }
+ Py_VISIT(gen->closure);
+ Py_VISIT(gen->classobj);
+ Py_VISIT(gen->yieldfrom);
+ return __Pyx_Coroutine_traverse_excstate(&gen->gi_exc_state, visit, arg);
+}
+static int __Pyx_Coroutine_clear(PyObject *self) {
+ __pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self;
+ Py_CLEAR(gen->closure);
+ Py_CLEAR(gen->classobj);
+ __Pyx_Coroutine_Undelegate(gen);
+ __Pyx_Coroutine_ExceptionClear(&gen->gi_exc_state);
+#ifdef __Pyx_AsyncGen_USED
+ if (__Pyx_AsyncGen_CheckExact(self)) {
+ Py_CLEAR(((__pyx_PyAsyncGenObject*)gen)->ag_finalizer);
+ }
+#endif
+ Py_CLEAR(gen->gi_code);
+ Py_CLEAR(gen->gi_frame);
+ Py_CLEAR(gen->gi_name);
+ Py_CLEAR(gen->gi_qualname);
+ Py_CLEAR(gen->gi_modulename);
+ return 0;
+}
+static void __Pyx_Coroutine_dealloc(PyObject *self) {
+ __pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self;
+ PyObject_GC_UnTrack(gen);
+ if (gen->gi_weakreflist != NULL)
+ PyObject_ClearWeakRefs(self);
+ if (gen->resume_label >= 0) {
+ PyObject_GC_Track(self);
+#if CYTHON_USE_TP_FINALIZE
+ if (unlikely(PyObject_CallFinalizerFromDealloc(self)))
+#else
+ {
+ destructor del = __Pyx_PyObject_GetSlot(gen, tp_del, destructor);
+ if (del) del(self);
+ }
+ if (unlikely(Py_REFCNT(self) > 0))
+#endif
+ {
+ return;
+ }
+ PyObject_GC_UnTrack(self);
+ }
+#ifdef __Pyx_AsyncGen_USED
+ if (__Pyx_AsyncGen_CheckExact(self)) {
+ /* We have to handle this case for asynchronous generators
+ right here, because this code has to be between UNTRACK
+ and GC_Del. */
+ Py_CLEAR(((__pyx_PyAsyncGenObject*)self)->ag_finalizer);
+ }
+#endif
+ __Pyx_Coroutine_clear(self);
+ __Pyx_PyHeapTypeObject_GC_Del(gen);
+}
+#if CYTHON_USE_TP_FINALIZE
+static void __Pyx_Coroutine_del(PyObject *self) {
+ PyObject *error_type, *error_value, *error_traceback;
+ __pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self;
+ __Pyx_PyThreadState_declare
+ if (gen->resume_label < 0) {
+ return;
+ }
+ __Pyx_PyThreadState_assign
+ __Pyx_ErrFetch(&error_type, &error_value, &error_traceback);
+#ifdef __Pyx_AsyncGen_USED
+ if (__Pyx_AsyncGen_CheckExact(self)) {
+ __pyx_PyAsyncGenObject *agen = (__pyx_PyAsyncGenObject*)self;
+ PyObject *finalizer = agen->ag_finalizer;
+ if (finalizer && !agen->ag_closed) {
+ PyObject *res = __Pyx_PyObject_CallOneArg(finalizer, self);
+ if (unlikely(!res)) {
+ PyErr_WriteUnraisable(self);
+ } else {
+ Py_DECREF(res);
+ }
+ __Pyx_ErrRestore(error_type, error_value, error_traceback);
+ return;
+ }
+ }
+#endif
+ if (unlikely(gen->resume_label == 0 && !error_value)) {
+#ifdef __Pyx_Coroutine_USED
+#ifdef __Pyx_Generator_USED
+ if (!__Pyx_Generator_CheckExact(self))
+#endif
+ {
+ PyObject_GC_UnTrack(self);
+ if (unlikely(PyErr_WarnFormat(PyExc_RuntimeWarning, 1, "coroutine '%.50S' was never awaited", gen->gi_qualname) < 0))
+ PyErr_WriteUnraisable(self);
+ PyObject_GC_Track(self);
+ }
+#endif
+ } else {
+ PyObject *retval = NULL;
+ __Pyx_PySendResult result = __Pyx_Coroutine_Close(self, &retval);
+ if (result == PYGEN_ERROR) {
+ PyErr_WriteUnraisable(self);
+ } else {
+ Py_XDECREF(retval);
+ }
+ }
+ __Pyx_ErrRestore(error_type, error_value, error_traceback);
+}
+#endif
+static PyObject *
+__Pyx_Coroutine_get_name(__pyx_CoroutineObject *self, void *context)
+{
+ PyObject *name = self->gi_name;
+ CYTHON_UNUSED_VAR(context);
+ if (unlikely(!name)) name = Py_None;
+ Py_INCREF(name);
+ return name;
+}
+static int
+__Pyx_Coroutine_set_name(__pyx_CoroutineObject *self, PyObject *value, void *context)
+{
+ CYTHON_UNUSED_VAR(context);
+ if (unlikely(value == NULL || !PyUnicode_Check(value))) {
+ PyErr_SetString(PyExc_TypeError,
+ "__name__ must be set to a string object");
+ return -1;
+ }
+ Py_INCREF(value);
+ __Pyx_Py_XDECREF_SET(self->gi_name, value);
+ return 0;
+}
+static PyObject *
+__Pyx_Coroutine_get_qualname(__pyx_CoroutineObject *self, void *context)
+{
+ PyObject *name = self->gi_qualname;
+ CYTHON_UNUSED_VAR(context);
+ if (unlikely(!name)) name = Py_None;
+ Py_INCREF(name);
+ return name;
+}
+static int
+__Pyx_Coroutine_set_qualname(__pyx_CoroutineObject *self, PyObject *value, void *context)
+{
+ CYTHON_UNUSED_VAR(context);
+ if (unlikely(value == NULL || !PyUnicode_Check(value))) {
+ PyErr_SetString(PyExc_TypeError,
+ "__qualname__ must be set to a string object");
+ return -1;
+ }
+ Py_INCREF(value);
+ __Pyx_Py_XDECREF_SET(self->gi_qualname, value);
+ return 0;
+}
+static PyObject *
+__Pyx__Coroutine_get_frame(__pyx_CoroutineObject *self)
+{
+#if !CYTHON_COMPILING_IN_LIMITED_API
+ PyObject *frame;
+ #if PY_VERSION_HEX >= 0x030d0000
+ Py_BEGIN_CRITICAL_SECTION(self);
+ #endif
+ frame = self->gi_frame;
+ if (!frame) {
+ if (unlikely(!self->gi_code)) {
+ Py_RETURN_NONE;
+ }
+ PyObject *globals = PyDict_New();
+ if (unlikely(!globals)) return NULL;
+ frame = (PyObject *) PyFrame_New(
+ PyThreadState_Get(), /*PyThreadState *tstate,*/
+ (PyCodeObject*) self->gi_code, /*PyCodeObject *code,*/
+ globals, /*PyObject *globals,*/
+ 0 /*PyObject *locals*/
+ );
+ Py_DECREF(globals);
+ if (unlikely(!frame))
+ return NULL;
+ if (unlikely(self->gi_frame)) {
+ Py_DECREF(frame);
+ frame = self->gi_frame;
+ } else {
+ self->gi_frame = frame;
+ }
+ }
+ Py_INCREF(frame);
+ #if PY_VERSION_HEX >= 0x030d0000
+ Py_END_CRITICAL_SECTION();
+ #endif
+ return frame;
+#else
+ CYTHON_UNUSED_VAR(self);
+ Py_RETURN_NONE;
+#endif
+}
+static PyObject *
+__Pyx_Coroutine_get_frame(__pyx_CoroutineObject *self, void *context) {
+ CYTHON_UNUSED_VAR(context);
+ PyObject *frame = self->gi_frame;
+ if (frame)
+ return __Pyx_NewRef(frame);
+ return __Pyx__Coroutine_get_frame(self);
+}
+static __pyx_CoroutineObject *__Pyx__Coroutine_New(
+ PyTypeObject* type, __pyx_coroutine_body_t body, PyObject *code, PyObject *closure,
+ PyObject *name, PyObject *qualname, PyObject *module_name) {
+ __pyx_CoroutineObject *gen = PyObject_GC_New(__pyx_CoroutineObject, type);
+ if (unlikely(!gen))
+ return NULL;
+ return __Pyx__Coroutine_NewInit(gen, body, code, closure, name, qualname, module_name);
+}
+static __pyx_CoroutineObject *__Pyx__Coroutine_NewInit(
+ __pyx_CoroutineObject *gen, __pyx_coroutine_body_t body, PyObject *code, PyObject *closure,
+ PyObject *name, PyObject *qualname, PyObject *module_name) {
+ gen->body = body;
+ gen->closure = closure;
+ Py_XINCREF(closure);
+ gen->is_running = 0;
+ gen->resume_label = 0;
+ gen->classobj = NULL;
+ gen->yieldfrom = NULL;
+ gen->yieldfrom_am_send = NULL;
+ #if PY_VERSION_HEX >= 0x030B00a4 && !CYTHON_COMPILING_IN_LIMITED_API
+ gen->gi_exc_state.exc_value = NULL;
+ #else
+ gen->gi_exc_state.exc_type = NULL;
+ gen->gi_exc_state.exc_value = NULL;
+ gen->gi_exc_state.exc_traceback = NULL;
+ #endif
+#if CYTHON_USE_EXC_INFO_STACK
+ gen->gi_exc_state.previous_item = NULL;
+#endif
+ gen->gi_weakreflist = NULL;
+ Py_XINCREF(qualname);
+ gen->gi_qualname = qualname;
+ Py_XINCREF(name);
+ gen->gi_name = name;
+ Py_XINCREF(module_name);
+ gen->gi_modulename = module_name;
+ Py_XINCREF(code);
+ gen->gi_code = code;
+ gen->gi_frame = NULL;
+ PyObject_GC_Track(gen);
+ return gen;
+}
+static char __Pyx_Coroutine_test_and_set_is_running(__pyx_CoroutineObject *gen) {
+ char result;
+ #if PY_VERSION_HEX >= 0x030d0000 && !CYTHON_COMPILING_IN_LIMITED_API
+ Py_BEGIN_CRITICAL_SECTION(gen);
+ #endif
+ result = gen->is_running;
+ gen->is_running = 1;
+ #if PY_VERSION_HEX >= 0x030d0000 && !CYTHON_COMPILING_IN_LIMITED_API
+ Py_END_CRITICAL_SECTION();
+ #endif
+ return result;
+}
+static void __Pyx_Coroutine_unset_is_running(__pyx_CoroutineObject *gen) {
+ #if PY_VERSION_HEX >= 0x030d0000 && !CYTHON_COMPILING_IN_LIMITED_API
+ Py_BEGIN_CRITICAL_SECTION(gen);
+ #endif
+ assert(gen->is_running);
+ gen->is_running = 0;
+ #if PY_VERSION_HEX >= 0x030d0000 && !CYTHON_COMPILING_IN_LIMITED_API
+ Py_END_CRITICAL_SECTION();
+ #endif
+}
+static char __Pyx_Coroutine_get_is_running(__pyx_CoroutineObject *gen) {
+ char result;
+ #if PY_VERSION_HEX >= 0x030d0000 && !CYTHON_COMPILING_IN_LIMITED_API
+ Py_BEGIN_CRITICAL_SECTION(gen);
+ #endif
+ result = gen->is_running;
+ #if PY_VERSION_HEX >= 0x030d0000 && !CYTHON_COMPILING_IN_LIMITED_API
+ Py_END_CRITICAL_SECTION();
+ #endif
+ return result;
+}
+static PyObject *__Pyx_Coroutine_get_is_running_getter(PyObject *gen, void *closure) {
+ CYTHON_UNUSED_VAR(closure);
+ char result = __Pyx_Coroutine_get_is_running((__pyx_CoroutineObject*)gen);
+ if (result) Py_RETURN_TRUE;
+ else Py_RETURN_FALSE;
+}
+#if __PYX_HAS_PY_AM_SEND == 2
+static void __Pyx_SetBackportTypeAmSend(PyTypeObject *type, __Pyx_PyAsyncMethodsStruct *static_amsend_methods, __Pyx_pyiter_sendfunc am_send) {
+ Py_ssize_t ptr_offset = (char*)(type->tp_as_async) - (char*)type;
+ if (ptr_offset < 0 || ptr_offset > type->tp_basicsize) {
+ return;
+ }
+ memcpy((void*)static_amsend_methods, (void*)(type->tp_as_async), sizeof(*type->tp_as_async));
+ static_amsend_methods->am_send = am_send;
+ type->tp_as_async = __Pyx_SlotTpAsAsync(static_amsend_methods);
+}
+#endif
+static PyObject *__Pyx_Coroutine_fail_reduce_ex(PyObject *self, PyObject *arg) {
+ CYTHON_UNUSED_VAR(arg);
+ __Pyx_TypeName self_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE((PyObject*)self));
+ PyErr_Format(PyExc_TypeError, "cannot pickle '" __Pyx_FMT_TYPENAME "' object",
+ self_type_name);
+ __Pyx_DECREF_TypeName(self_type_name);
+ return NULL;
+}
+
+/* Generator */
+static PyMethodDef __pyx_Generator_methods[] = {
+ {"send", (PyCFunction) __Pyx_Coroutine_Send, METH_O,
+ PyDoc_STR("send(arg) -> send 'arg' into generator,\nreturn next yielded value or raise StopIteration.")},
+ {"throw", (PyCFunction) __Pyx_Coroutine_Throw, METH_VARARGS,
+ PyDoc_STR("throw(typ[,val[,tb]]) -> raise exception in generator,\nreturn next yielded value or raise StopIteration.")},
+ {"close", (PyCFunction) __Pyx_Coroutine_Close_Method, METH_NOARGS,
+ PyDoc_STR("close() -> raise GeneratorExit inside generator.")},
+ {"__reduce_ex__", (PyCFunction) __Pyx_Coroutine_fail_reduce_ex, METH_O, 0},
+ {"__reduce__", (PyCFunction) __Pyx_Coroutine_fail_reduce_ex, METH_NOARGS, 0},
+ {0, 0, 0, 0}
+};
+static PyMemberDef __pyx_Generator_memberlist[] = {
+ {"gi_yieldfrom", T_OBJECT, offsetof(__pyx_CoroutineObject, yieldfrom), READONLY,
+ PyDoc_STR("object being iterated by 'yield from', or None")},
+ {"gi_code", T_OBJECT, offsetof(__pyx_CoroutineObject, gi_code), READONLY, NULL},
+ {"__module__", T_OBJECT, offsetof(__pyx_CoroutineObject, gi_modulename), 0, 0},
+ {"__weaklistoffset__", T_PYSSIZET, offsetof(__pyx_CoroutineObject, gi_weakreflist), READONLY, 0},
+ {0, 0, 0, 0, 0}
+};
+static PyGetSetDef __pyx_Generator_getsets[] = {
+ {"__name__", (getter)__Pyx_Coroutine_get_name, (setter)__Pyx_Coroutine_set_name,
+ PyDoc_STR("name of the generator"), 0},
+ {"__qualname__", (getter)__Pyx_Coroutine_get_qualname, (setter)__Pyx_Coroutine_set_qualname,
+ PyDoc_STR("qualified name of the generator"), 0},
+ {"gi_frame", (getter)__Pyx_Coroutine_get_frame, NULL,
+ PyDoc_STR("Frame of the generator"), 0},
+ {"gi_running", __Pyx_Coroutine_get_is_running_getter, NULL, NULL, NULL},
+ {0, 0, 0, 0, 0}
+};
+static PyType_Slot __pyx_GeneratorType_slots[] = {
+ {Py_tp_dealloc, (void *)__Pyx_Coroutine_dealloc},
+ {Py_tp_traverse, (void *)__Pyx_Coroutine_traverse},
+ {Py_tp_iter, (void *)PyObject_SelfIter},
+ {Py_tp_iternext, (void *)__Pyx_Generator_Next},
+ {Py_tp_methods, (void *)__pyx_Generator_methods},
+ {Py_tp_members, (void *)__pyx_Generator_memberlist},
+ {Py_tp_getset, (void *)__pyx_Generator_getsets},
+ {Py_tp_getattro, (void *) PyObject_GenericGetAttr},
+#if CYTHON_USE_TP_FINALIZE
+ {Py_tp_finalize, (void *)__Pyx_Coroutine_del},
+#endif
+#if __PYX_HAS_PY_AM_SEND == 1
+ {Py_am_send, (void *)__Pyx_Coroutine_AmSend},
+#endif
+ {0, 0},
+};
+static PyType_Spec __pyx_GeneratorType_spec = {
+ __PYX_TYPE_MODULE_PREFIX "generator",
+ sizeof(__pyx_CoroutineObject),
+ 0,
+#if PY_VERSION_HEX >= 0x030A0000
+ Py_TPFLAGS_IMMUTABLETYPE |
+#endif
+ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_FINALIZE | __Pyx_TPFLAGS_HAVE_AM_SEND,
+ __pyx_GeneratorType_slots
+};
+#if __PYX_HAS_PY_AM_SEND == 2
+static __Pyx_PyAsyncMethodsStruct __pyx_Generator_as_async;
+#endif
+static int __pyx_Generator_init(PyObject *module) {
+ __pyx_mstatetype *mstate = __Pyx_PyModule_GetState(module);
+ mstate->__pyx_GeneratorType = __Pyx_FetchCommonTypeFromSpec(
+ mstate->__pyx_CommonTypesMetaclassType, module, &__pyx_GeneratorType_spec, NULL);
+ if (unlikely(!mstate->__pyx_GeneratorType)) {
+ return -1;
+ }
+#if __PYX_HAS_PY_AM_SEND == 2
+ __Pyx_SetBackportTypeAmSend(mstate->__pyx_GeneratorType, &__pyx_Generator_as_async, &__Pyx_Coroutine_AmSend);
+#endif
+ return 0;
+}
+static PyObject *__Pyx_Generator_GetInlinedResult(PyObject *self) {
+ __pyx_CoroutineObject *gen = (__pyx_CoroutineObject*) self;
+ PyObject *retval = NULL;
+ if (unlikely(__Pyx_Coroutine_test_and_set_is_running(gen))) {
+ return __Pyx_Coroutine_AlreadyRunningError(gen);
+ }
+ __Pyx_PySendResult result = __Pyx_Coroutine_SendEx(gen, Py_None, &retval, 0);
+ __Pyx_Coroutine_unset_is_running(gen);
+ (void) result;
+ assert (result == PYGEN_RETURN || result == PYGEN_ERROR);
+ assert ((result == PYGEN_RETURN && retval != NULL) || (result == PYGEN_ERROR && retval == NULL));
+ return retval;
+}
+
+/* GetRuntimeVersion */
+static unsigned long __Pyx_get_runtime_version(void) {
+#if __PYX_LIMITED_VERSION_HEX >= 0x030b0000
+ return Py_Version & ~0xFFUL;
+#else
+ static unsigned long __Pyx_cached_runtime_version = 0;
+ if (__Pyx_cached_runtime_version == 0) {
+ const char* rt_version = Py_GetVersion();
+ unsigned long version = 0;
+ unsigned long factor = 0x01000000UL;
+ unsigned int digit = 0;
+ int i = 0;
+ while (factor) {
+ while ('0' <= rt_version[i] && rt_version[i] <= '9') {
+ digit = digit * 10 + (unsigned int) (rt_version[i] - '0');
+ ++i;
+ }
+ version += factor * digit;
+ if (rt_version[i] != '.')
+ break;
+ digit = 0;
+ factor >>= 8;
+ ++i;
+ }
+ __Pyx_cached_runtime_version = version;
+ }
+ return __Pyx_cached_runtime_version;
+#endif
+}
+
+/* CheckBinaryVersion */
+static int __Pyx_check_binary_version(unsigned long ct_version, unsigned long rt_version, int allow_newer) {
+ const unsigned long MAJOR_MINOR = 0xFFFF0000UL;
+ if ((rt_version & MAJOR_MINOR) == (ct_version & MAJOR_MINOR))
+ return 0;
+ if (likely(allow_newer && (rt_version & MAJOR_MINOR) > (ct_version & MAJOR_MINOR)))
+ return 1;
+ {
+ char message[200];
+ PyOS_snprintf(message, sizeof(message),
+ "compile time Python version %d.%d "
+ "of module '%.100s' "
+ "%s "
+ "runtime version %d.%d",
+ (int) (ct_version >> 24), (int) ((ct_version >> 16) & 0xFF),
+ __Pyx_MODULE_NAME,
+ (allow_newer) ? "was newer than" : "does not match",
+ (int) (rt_version >> 24), (int) ((rt_version >> 16) & 0xFF)
+ );
+ return PyErr_WarnEx(NULL, message, 1);
+ }
+}
+
+/* NewCodeObj */
+#if CYTHON_COMPILING_IN_LIMITED_API
+ static PyObject* __Pyx__PyCode_New(int a, int p, int k, int l, int s, int f,
+ PyObject *code, PyObject *c, PyObject* n, PyObject *v,
+ PyObject *fv, PyObject *cell, PyObject* fn,
+ PyObject *name, int fline, PyObject *lnos) {
+ PyObject *exception_table = NULL;
+ PyObject *types_module=NULL, *code_type=NULL, *result=NULL;
+ #if __PYX_LIMITED_VERSION_HEX < 0x030b0000
+ PyObject *version_info;
+ PyObject *py_minor_version = NULL;
+ #endif
+ long minor_version = 0;
+ PyObject *type, *value, *traceback;
+ PyErr_Fetch(&type, &value, &traceback);
+ #if __PYX_LIMITED_VERSION_HEX >= 0x030b0000
+ minor_version = 11;
+ #else
+ if (!(version_info = PySys_GetObject("version_info"))) goto end;
+ if (!(py_minor_version = PySequence_GetItem(version_info, 1))) goto end;
+ minor_version = PyLong_AsLong(py_minor_version);
+ Py_DECREF(py_minor_version);
+ if (minor_version == -1 && PyErr_Occurred()) goto end;
+ #endif
+ if (!(types_module = PyImport_ImportModule("types"))) goto end;
+ if (!(code_type = PyObject_GetAttrString(types_module, "CodeType"))) goto end;
+ if (minor_version <= 7) {
+ (void)p;
+ result = PyObject_CallFunction(code_type, "iiiiiOOOOOOiOOO", a, k, l, s, f, code,
+ c, n, v, fn, name, fline, lnos, fv, cell);
+ } else if (minor_version <= 10) {
+ result = PyObject_CallFunction(code_type, "iiiiiiOOOOOOiOOO", a,p, k, l, s, f, code,
+ c, n, v, fn, name, fline, lnos, fv, cell);
+ } else {
+ if (!(exception_table = PyBytes_FromStringAndSize(NULL, 0))) goto end;
+ result = PyObject_CallFunction(code_type, "iiiiiiOOOOOOOiOOOO", a,p, k, l, s, f, code,
+ c, n, v, fn, name, name, fline, lnos, exception_table, fv, cell);
+ }
+ end:
+ Py_XDECREF(code_type);
+ Py_XDECREF(exception_table);
+ Py_XDECREF(types_module);
+ if (type) {
+ PyErr_Restore(type, value, traceback);
+ }
+ return result;
+ }
+#elif PY_VERSION_HEX >= 0x030B0000
+ static PyCodeObject* __Pyx__PyCode_New(int a, int p, int k, int l, int s, int f,
+ PyObject *code, PyObject *c, PyObject* n, PyObject *v,
+ PyObject *fv, PyObject *cell, PyObject* fn,
+ PyObject *name, int fline, PyObject *lnos) {
+ PyCodeObject *result;
+ result =
+ #if PY_VERSION_HEX >= 0x030C0000
+ PyUnstable_Code_NewWithPosOnlyArgs
+ #else
+ PyCode_NewWithPosOnlyArgs
+ #endif
+ (a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, name, fline, lnos, __pyx_mstate_global->__pyx_empty_bytes);
+ #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030c00A1
+ if (likely(result))
+ result->_co_firsttraceable = 0;
+ #endif
+ return result;
+ }
+#elif PY_VERSION_HEX >= 0x030800B2 && !CYTHON_COMPILING_IN_PYPY
+ #define __Pyx__PyCode_New(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos)\
+ PyCode_NewWithPosOnlyArgs(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos)
+#else
+ #define __Pyx__PyCode_New(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos)\
+ PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos)
+#endif
+static PyObject* __Pyx_PyCode_New(
+ const __Pyx_PyCode_New_function_description descr,
+ PyObject * const *varnames,
+ PyObject *filename,
+ PyObject *funcname,
+ const char *line_table,
+ PyObject *tuple_dedup_map
+) {
+ PyObject *code_obj = NULL, *varnames_tuple_dedup = NULL, *code_bytes = NULL, *line_table_bytes = NULL;
+ Py_ssize_t var_count = (Py_ssize_t) descr.nlocals;
+ PyObject *varnames_tuple = PyTuple_New(var_count);
+ if (unlikely(!varnames_tuple)) return NULL;
+ for (Py_ssize_t i=0; i < var_count; i++) {
+ Py_INCREF(varnames[i]);
+ if (__Pyx_PyTuple_SET_ITEM(varnames_tuple, i, varnames[i]) != (0)) goto done;
+ }
+ #if CYTHON_COMPILING_IN_LIMITED_API
+ varnames_tuple_dedup = PyDict_GetItem(tuple_dedup_map, varnames_tuple);
+ if (!varnames_tuple_dedup) {
+ if (unlikely(PyDict_SetItem(tuple_dedup_map, varnames_tuple, varnames_tuple) < 0)) goto done;
+ varnames_tuple_dedup = varnames_tuple;
+ }
+ #else
+ varnames_tuple_dedup = PyDict_SetDefault(tuple_dedup_map, varnames_tuple, varnames_tuple);
+ if (unlikely(!varnames_tuple_dedup)) goto done;
+ #endif
+ #if CYTHON_AVOID_BORROWED_REFS
+ Py_INCREF(varnames_tuple_dedup);
+ #endif
+ if (__PYX_LIMITED_VERSION_HEX >= (0x030b0000) && line_table != NULL
+ && !CYTHON_COMPILING_IN_GRAAL) {
+ line_table_bytes = PyBytes_FromStringAndSize(line_table, descr.line_table_length);
+ if (unlikely(!line_table_bytes)) goto done;
+ Py_ssize_t code_len = (descr.line_table_length * 2 + 4) & ~3;
+ code_bytes = PyBytes_FromStringAndSize(NULL, code_len);
+ if (unlikely(!code_bytes)) goto done;
+ char* c_code_bytes = PyBytes_AsString(code_bytes);
+ if (unlikely(!c_code_bytes)) goto done;
+ memset(c_code_bytes, 0, (size_t) code_len);
+ }
+ code_obj = (PyObject*) __Pyx__PyCode_New(
+ (int) descr.argcount,
+ (int) descr.num_posonly_args,
+ (int) descr.num_kwonly_args,
+ (int) descr.nlocals,
+ 0,
+ (int) descr.flags,
+ code_bytes ? code_bytes : __pyx_mstate_global->__pyx_empty_bytes,
+ __pyx_mstate_global->__pyx_empty_tuple,
+ __pyx_mstate_global->__pyx_empty_tuple,
+ varnames_tuple_dedup,
+ __pyx_mstate_global->__pyx_empty_tuple,
+ __pyx_mstate_global->__pyx_empty_tuple,
+ filename,
+ funcname,
+ (int) descr.first_line,
+ (__PYX_LIMITED_VERSION_HEX >= (0x030b0000) && line_table_bytes) ? line_table_bytes : __pyx_mstate_global->__pyx_empty_bytes
+ );
+done:
+ Py_XDECREF(code_bytes);
+ Py_XDECREF(line_table_bytes);
+ #if CYTHON_AVOID_BORROWED_REFS
+ Py_XDECREF(varnames_tuple_dedup);
+ #endif
+ Py_DECREF(varnames_tuple);
+ return code_obj;
+}
+
+/* InitStrings */
+static int __Pyx_InitStrings(__Pyx_StringTabEntry const *t, PyObject **target, const char* const* encoding_names) {
+ while (t->s) {
+ PyObject *str;
+ if (t->is_unicode) {
+ if (t->intern) {
+ str = PyUnicode_InternFromString(t->s);
+ } else if (t->encoding) {
+ str = PyUnicode_Decode(t->s, t->n - 1, encoding_names[t->encoding], NULL);
+ } else {
+ str = PyUnicode_FromStringAndSize(t->s, t->n - 1);
+ }
+ } else {
+ str = PyBytes_FromStringAndSize(t->s, t->n - 1);
+ }
+ if (!str)
+ return -1;
+ *target = str;
+ if (PyObject_Hash(str) == -1)
+ return -1;
+ ++t;
+ ++target;
+ }
+ return 0;
+}
+
+#include
+static CYTHON_INLINE Py_ssize_t __Pyx_ssize_strlen(const char *s) {
+ size_t len = strlen(s);
+ if (unlikely(len > (size_t) PY_SSIZE_T_MAX)) {
+ PyErr_SetString(PyExc_OverflowError, "byte string is too long");
+ return -1;
+ }
+ return (Py_ssize_t) len;
+}
+static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char* c_str) {
+ Py_ssize_t len = __Pyx_ssize_strlen(c_str);
+ if (unlikely(len < 0)) return NULL;
+ return __Pyx_PyUnicode_FromStringAndSize(c_str, len);
+}
+static CYTHON_INLINE PyObject* __Pyx_PyByteArray_FromString(const char* c_str) {
+ Py_ssize_t len = __Pyx_ssize_strlen(c_str);
+ if (unlikely(len < 0)) return NULL;
+ return PyByteArray_FromStringAndSize(c_str, len);
+}
+static CYTHON_INLINE const char* __Pyx_PyObject_AsString(PyObject* o) {
+ Py_ssize_t ignore;
+ return __Pyx_PyObject_AsStringAndSize(o, &ignore);
+}
+#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_UTF8
+static CYTHON_INLINE const char* __Pyx_PyUnicode_AsStringAndSize(PyObject* o, Py_ssize_t *length) {
+ if (unlikely(__Pyx_PyUnicode_READY(o) == -1)) return NULL;
+#if CYTHON_COMPILING_IN_LIMITED_API
+ {
+ const char* result;
+ Py_ssize_t unicode_length;
+ CYTHON_MAYBE_UNUSED_VAR(unicode_length); // only for __PYX_DEFAULT_STRING_ENCODING_IS_ASCII
+ #if __PYX_LIMITED_VERSION_HEX < 0x030A0000
+ if (unlikely(PyArg_Parse(o, "s#", &result, length) < 0)) return NULL;
+ #else
+ result = PyUnicode_AsUTF8AndSize(o, length);
+ #endif
+ #if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII
+ unicode_length = PyUnicode_GetLength(o);
+ if (unlikely(unicode_length < 0)) return NULL;
+ if (unlikely(unicode_length != *length)) {
+ PyUnicode_AsASCIIString(o);
+ return NULL;
+ }
+ #endif
+ return result;
+ }
+#else
+#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII
+ if (likely(PyUnicode_IS_ASCII(o))) {
+ *length = PyUnicode_GET_LENGTH(o);
+ return PyUnicode_AsUTF8(o);
+ } else {
+ PyUnicode_AsASCIIString(o);
+ return NULL;
+ }
+#else
+ return PyUnicode_AsUTF8AndSize(o, length);
+#endif
+#endif
+}
+#endif
+static CYTHON_INLINE const char* __Pyx_PyObject_AsStringAndSize(PyObject* o, Py_ssize_t *length) {
+#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_UTF8
+ if (PyUnicode_Check(o)) {
+ return __Pyx_PyUnicode_AsStringAndSize(o, length);
+ } else
+#endif
+ if (PyByteArray_Check(o)) {
+#if (CYTHON_ASSUME_SAFE_SIZE && CYTHON_ASSUME_SAFE_MACROS) || (CYTHON_COMPILING_IN_PYPY && (defined(PyByteArray_AS_STRING) && defined(PyByteArray_GET_SIZE)))
+ *length = PyByteArray_GET_SIZE(o);
+ return PyByteArray_AS_STRING(o);
+#else
+ *length = PyByteArray_Size(o);
+ if (*length == -1) return NULL;
+ return PyByteArray_AsString(o);
+#endif
+ } else
+ {
+ char* result;
+ int r = PyBytes_AsStringAndSize(o, &result, length);
+ if (unlikely(r < 0)) {
+ return NULL;
+ } else {
+ return result;
+ }
+ }
+}
+static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject* x) {
+ int is_true = x == Py_True;
+ if (is_true | (x == Py_False) | (x == Py_None)) return is_true;
+ else return PyObject_IsTrue(x);
+}
+static CYTHON_INLINE int __Pyx_PyObject_IsTrueAndDecref(PyObject* x) {
+ int retval;
+ if (unlikely(!x)) return -1;
+ retval = __Pyx_PyObject_IsTrue(x);
+ Py_DECREF(x);
+ return retval;
+}
+static PyObject* __Pyx_PyNumber_LongWrongResultType(PyObject* result) {
+ __Pyx_TypeName result_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(result));
+ if (PyLong_Check(result)) {
+ if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1,
+ "__int__ returned non-int (type " __Pyx_FMT_TYPENAME "). "
+ "The ability to return an instance of a strict subclass of int is deprecated, "
+ "and may be removed in a future version of Python.",
+ result_type_name)) {
+ __Pyx_DECREF_TypeName(result_type_name);
+ Py_DECREF(result);
+ return NULL;
+ }
+ __Pyx_DECREF_TypeName(result_type_name);
+ return result;
+ }
+ PyErr_Format(PyExc_TypeError,
+ "__int__ returned non-int (type " __Pyx_FMT_TYPENAME ")",
+ result_type_name);
+ __Pyx_DECREF_TypeName(result_type_name);
+ Py_DECREF(result);
+ return NULL;
+}
+static CYTHON_INLINE PyObject* __Pyx_PyNumber_Long(PyObject* x) {
+#if CYTHON_USE_TYPE_SLOTS
+ PyNumberMethods *m;
+#endif
+ PyObject *res = NULL;
+ if (likely(PyLong_Check(x)))
+ return __Pyx_NewRef(x);
+#if CYTHON_USE_TYPE_SLOTS
+ m = Py_TYPE(x)->tp_as_number;
+ if (likely(m && m->nb_int)) {
+ res = m->nb_int(x);
+ }
+#else
+ if (!PyBytes_CheckExact(x) && !PyUnicode_CheckExact(x)) {
+ res = PyNumber_Long(x);
+ }
+#endif
+ if (likely(res)) {
+ if (unlikely(!PyLong_CheckExact(res))) {
+ return __Pyx_PyNumber_LongWrongResultType(res);
+ }
+ }
+ else if (!PyErr_Occurred()) {
+ PyErr_SetString(PyExc_TypeError,
+ "an integer is required");
+ }
+ return res;
+}
+static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject* b) {
+ Py_ssize_t ival;
+ PyObject *x;
+ if (likely(PyLong_CheckExact(b))) {
+ #if CYTHON_USE_PYLONG_INTERNALS
+ if (likely(__Pyx_PyLong_IsCompact(b))) {
+ return __Pyx_PyLong_CompactValue(b);
+ } else {
+ const digit* digits = __Pyx_PyLong_Digits(b);
+ const Py_ssize_t size = __Pyx_PyLong_SignedDigitCount(b);
+ switch (size) {
+ case 2:
+ if (8 * sizeof(Py_ssize_t) > 2 * PyLong_SHIFT) {
+ return (Py_ssize_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]));
+ }
+ break;
+ case -2:
+ if (8 * sizeof(Py_ssize_t) > 2 * PyLong_SHIFT) {
+ return -(Py_ssize_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]));
+ }
+ break;
+ case 3:
+ if (8 * sizeof(Py_ssize_t) > 3 * PyLong_SHIFT) {
+ return (Py_ssize_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]));
+ }
+ break;
+ case -3:
+ if (8 * sizeof(Py_ssize_t) > 3 * PyLong_SHIFT) {
+ return -(Py_ssize_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]));
+ }
+ break;
+ case 4:
+ if (8 * sizeof(Py_ssize_t) > 4 * PyLong_SHIFT) {
+ return (Py_ssize_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]));
+ }
+ break;
+ case -4:
+ if (8 * sizeof(Py_ssize_t) > 4 * PyLong_SHIFT) {
+ return -(Py_ssize_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]));
+ }
+ break;
+ }
+ }
+ #endif
+ return PyLong_AsSsize_t(b);
+ }
+ x = PyNumber_Index(b);
+ if (!x) return -1;
+ ival = PyLong_AsSsize_t(x);
+ Py_DECREF(x);
+ return ival;
+}
+static CYTHON_INLINE Py_hash_t __Pyx_PyIndex_AsHash_t(PyObject* o) {
+ if (sizeof(Py_hash_t) == sizeof(Py_ssize_t)) {
+ return (Py_hash_t) __Pyx_PyIndex_AsSsize_t(o);
+ } else {
+ Py_ssize_t ival;
+ PyObject *x;
+ x = PyNumber_Index(o);
+ if (!x) return -1;
+ ival = PyLong_AsLong(x);
+ Py_DECREF(x);
+ return ival;
+ }
+}
+static CYTHON_INLINE PyObject *__Pyx_Owned_Py_None(int b) {
+ CYTHON_UNUSED_VAR(b);
+ return __Pyx_NewRef(Py_None);
+}
+static CYTHON_INLINE PyObject * __Pyx_PyBool_FromLong(long b) {
+ return b ? __Pyx_NewRef(Py_True) : __Pyx_NewRef(Py_False);
+}
+static CYTHON_INLINE PyObject * __Pyx_PyLong_FromSize_t(size_t ival) {
+ return PyLong_FromSize_t(ival);
+}
+
+
+/* MultiPhaseInitModuleState */
+#if CYTHON_PEP489_MULTI_PHASE_INIT && CYTHON_USE_MODULE_STATE
+#ifndef CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE
+#if (CYTHON_COMPILING_IN_LIMITED_API || PY_VERSION_HEX >= 0x030C0000)
+ #define CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE 1
+#else
+ #define CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE 0
+#endif
+#endif
+#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE && !CYTHON_ATOMICS
+#error "Module state with PEP489 requires atomics. Currently that's one of\
+ C11, C++11, gcc atomic intrinsics or MSVC atomic intrinsics"
+#endif
+#if !CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE
+#define __Pyx_ModuleStateLookup_Lock()
+#define __Pyx_ModuleStateLookup_Unlock()
+#elif !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d0000
+static PyMutex __Pyx_ModuleStateLookup_mutex = {0};
+#define __Pyx_ModuleStateLookup_Lock() PyMutex_Lock(&__Pyx_ModuleStateLookup_mutex)
+#define __Pyx_ModuleStateLookup_Unlock() PyMutex_Unlock(&__Pyx_ModuleStateLookup_mutex)
+#elif defined(__cplusplus) && __cplusplus >= 201103L
+#include
+static std::mutex __Pyx_ModuleStateLookup_mutex;
+#define __Pyx_ModuleStateLookup_Lock() __Pyx_ModuleStateLookup_mutex.lock()
+#define __Pyx_ModuleStateLookup_Unlock() __Pyx_ModuleStateLookup_mutex.unlock()
+#elif defined(__STDC_VERSION__) && (__STDC_VERSION__ > 201112L) && !defined(__STDC_NO_THREADS__)
+#include
+static mtx_t __Pyx_ModuleStateLookup_mutex;
+static once_flag __Pyx_ModuleStateLookup_mutex_once_flag = ONCE_FLAG_INIT;
+static void __Pyx_ModuleStateLookup_initialize_mutex(void) {
+ mtx_init(&__Pyx_ModuleStateLookup_mutex, mtx_plain);
+}
+#define __Pyx_ModuleStateLookup_Lock()\
+ call_once(&__Pyx_ModuleStateLookup_mutex_once_flag, __Pyx_ModuleStateLookup_initialize_mutex);\
+ mtx_lock(&__Pyx_ModuleStateLookup_mutex)
+#define __Pyx_ModuleStateLookup_Unlock() mtx_unlock(&__Pyx_ModuleStateLookup_mutex)
+#elif defined(HAVE_PTHREAD_H)
+#include
+static pthread_mutex_t __Pyx_ModuleStateLookup_mutex = PTHREAD_MUTEX_INITIALIZER;
+#define __Pyx_ModuleStateLookup_Lock() pthread_mutex_lock(&__Pyx_ModuleStateLookup_mutex)
+#define __Pyx_ModuleStateLookup_Unlock() pthread_mutex_unlock(&__Pyx_ModuleStateLookup_mutex)
+#elif defined(_WIN32)
+#include // synchapi.h on its own doesn't work
+static SRWLOCK __Pyx_ModuleStateLookup_mutex = SRWLOCK_INIT;
+#define __Pyx_ModuleStateLookup_Lock() AcquireSRWLockExclusive(&__Pyx_ModuleStateLookup_mutex)
+#define __Pyx_ModuleStateLookup_Unlock() ReleaseSRWLockExclusive(&__Pyx_ModuleStateLookup_mutex)
+#else
+#error "No suitable lock available for CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE.\
+ Requires C standard >= C11, or C++ standard >= C++11,\
+ or pthreads, or the Windows 32 API, or Python >= 3.13."
+#endif
+typedef struct {
+ int64_t id;
+ PyObject *module;
+} __Pyx_InterpreterIdAndModule;
+typedef struct {
+ char interpreter_id_as_index;
+ Py_ssize_t count;
+ Py_ssize_t allocated;
+ __Pyx_InterpreterIdAndModule table[1];
+} __Pyx_ModuleStateLookupData;
+#define __PYX_MODULE_STATE_LOOKUP_SMALL_SIZE 32
+#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE
+static __pyx_atomic_int_type __Pyx_ModuleStateLookup_read_counter = 0;
+#endif
+#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE
+static __pyx_atomic_ptr_type __Pyx_ModuleStateLookup_data = 0;
+#else
+static __Pyx_ModuleStateLookupData* __Pyx_ModuleStateLookup_data = NULL;
+#endif
+static __Pyx_InterpreterIdAndModule* __Pyx_State_FindModuleStateLookupTableLowerBound(
+ __Pyx_InterpreterIdAndModule* table,
+ Py_ssize_t count,
+ int64_t interpreterId) {
+ __Pyx_InterpreterIdAndModule* begin = table;
+ __Pyx_InterpreterIdAndModule* end = begin + count;
+ if (begin->id == interpreterId) {
+ return begin;
+ }
+ while ((end - begin) > __PYX_MODULE_STATE_LOOKUP_SMALL_SIZE) {
+ __Pyx_InterpreterIdAndModule* halfway = begin + (end - begin)/2;
+ if (halfway->id == interpreterId) {
+ return halfway;
+ }
+ if (halfway->id < interpreterId) {
+ begin = halfway;
+ } else {
+ end = halfway;
+ }
+ }
+ for (; begin < end; ++begin) {
+ if (begin->id >= interpreterId) return begin;
+ }
+ return begin;
+}
+static PyObject *__Pyx_State_FindModule(CYTHON_UNUSED void* dummy) {
+ int64_t interpreter_id = PyInterpreterState_GetID(__Pyx_PyInterpreterState_Get());
+ if (interpreter_id == -1) return NULL;
+#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE
+ __Pyx_ModuleStateLookupData* data = (__Pyx_ModuleStateLookupData*)__pyx_atomic_pointer_load_relaxed(&__Pyx_ModuleStateLookup_data);
+ {
+ __pyx_atomic_incr_acq_rel(&__Pyx_ModuleStateLookup_read_counter);
+ if (likely(data)) {
+ __Pyx_ModuleStateLookupData* new_data = (__Pyx_ModuleStateLookupData*)__pyx_atomic_pointer_load_acquire(&__Pyx_ModuleStateLookup_data);
+ if (likely(data == new_data)) {
+ goto read_finished;
+ }
+ }
+ __pyx_atomic_decr_acq_rel(&__Pyx_ModuleStateLookup_read_counter);
+ __Pyx_ModuleStateLookup_Lock();
+ __pyx_atomic_incr_relaxed(&__Pyx_ModuleStateLookup_read_counter);
+ data = (__Pyx_ModuleStateLookupData*)__pyx_atomic_pointer_load_relaxed(&__Pyx_ModuleStateLookup_data);
+ __Pyx_ModuleStateLookup_Unlock();
+ }
+ read_finished:;
+#else
+ __Pyx_ModuleStateLookupData* data = __Pyx_ModuleStateLookup_data;
+#endif
+ __Pyx_InterpreterIdAndModule* found = NULL;
+ if (unlikely(!data)) goto end;
+ if (data->interpreter_id_as_index) {
+ if (interpreter_id < data->count) {
+ found = data->table+interpreter_id;
+ }
+ } else {
+ found = __Pyx_State_FindModuleStateLookupTableLowerBound(
+ data->table, data->count, interpreter_id);
+ }
+ end:
+ {
+ PyObject *result=NULL;
+ if (found && found->id == interpreter_id) {
+ result = found->module;
+ }
+#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE
+ __pyx_atomic_decr_acq_rel(&__Pyx_ModuleStateLookup_read_counter);
+#endif
+ return result;
+ }
+}
+#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE
+static void __Pyx_ModuleStateLookup_wait_until_no_readers(void) {
+ while (__pyx_atomic_load(&__Pyx_ModuleStateLookup_read_counter) != 0);
+}
+#else
+#define __Pyx_ModuleStateLookup_wait_until_no_readers()
+#endif
+static int __Pyx_State_AddModuleInterpIdAsIndex(__Pyx_ModuleStateLookupData **old_data, PyObject* module, int64_t interpreter_id) {
+ Py_ssize_t to_allocate = (*old_data)->allocated;
+ while (to_allocate <= interpreter_id) {
+ if (to_allocate == 0) to_allocate = 1;
+ else to_allocate *= 2;
+ }
+ __Pyx_ModuleStateLookupData *new_data = *old_data;
+ if (to_allocate != (*old_data)->allocated) {
+ new_data = (__Pyx_ModuleStateLookupData *)realloc(
+ *old_data,
+ sizeof(__Pyx_ModuleStateLookupData)+(to_allocate-1)*sizeof(__Pyx_InterpreterIdAndModule));
+ if (!new_data) {
+ PyErr_NoMemory();
+ return -1;
+ }
+ for (Py_ssize_t i = new_data->allocated; i < to_allocate; ++i) {
+ new_data->table[i].id = i;
+ new_data->table[i].module = NULL;
+ }
+ new_data->allocated = to_allocate;
+ }
+ new_data->table[interpreter_id].module = module;
+ if (new_data->count < interpreter_id+1) {
+ new_data->count = interpreter_id+1;
+ }
+ *old_data = new_data;
+ return 0;
+}
+static void __Pyx_State_ConvertFromInterpIdAsIndex(__Pyx_ModuleStateLookupData *data) {
+ __Pyx_InterpreterIdAndModule *read = data->table;
+ __Pyx_InterpreterIdAndModule *write = data->table;
+ __Pyx_InterpreterIdAndModule *end = read + data->count;
+ for (; readmodule) {
+ write->id = read->id;
+ write->module = read->module;
+ ++write;
+ }
+ }
+ data->count = write - data->table;
+ for (; writeid = 0;
+ write->module = NULL;
+ }
+ data->interpreter_id_as_index = 0;
+}
+static int __Pyx_State_AddModule(PyObject* module, CYTHON_UNUSED void* dummy) {
+ int64_t interpreter_id = PyInterpreterState_GetID(__Pyx_PyInterpreterState_Get());
+ if (interpreter_id == -1) return -1;
+ int result = 0;
+ __Pyx_ModuleStateLookup_Lock();
+#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE
+ __Pyx_ModuleStateLookupData *old_data = (__Pyx_ModuleStateLookupData *)
+ __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, 0);
+#else
+ __Pyx_ModuleStateLookupData *old_data = __Pyx_ModuleStateLookup_data;
+#endif
+ __Pyx_ModuleStateLookupData *new_data = old_data;
+ if (!new_data) {
+ new_data = (__Pyx_ModuleStateLookupData *)calloc(1, sizeof(__Pyx_ModuleStateLookupData));
+ if (!new_data) {
+ result = -1;
+ PyErr_NoMemory();
+ goto end;
+ }
+ new_data->allocated = 1;
+ new_data->interpreter_id_as_index = 1;
+ }
+ __Pyx_ModuleStateLookup_wait_until_no_readers();
+ if (new_data->interpreter_id_as_index) {
+ if (interpreter_id < __PYX_MODULE_STATE_LOOKUP_SMALL_SIZE) {
+ result = __Pyx_State_AddModuleInterpIdAsIndex(&new_data, module, interpreter_id);
+ goto end;
+ }
+ __Pyx_State_ConvertFromInterpIdAsIndex(new_data);
+ }
+ {
+ Py_ssize_t insert_at = 0;
+ {
+ __Pyx_InterpreterIdAndModule* lower_bound = __Pyx_State_FindModuleStateLookupTableLowerBound(
+ new_data->table, new_data->count, interpreter_id);
+ assert(lower_bound);
+ insert_at = lower_bound - new_data->table;
+ if (unlikely(insert_at < new_data->count && lower_bound->id == interpreter_id)) {
+ lower_bound->module = module;
+ goto end; // already in table, nothing more to do
+ }
+ }
+ if (new_data->count+1 >= new_data->allocated) {
+ Py_ssize_t to_allocate = (new_data->count+1)*2;
+ new_data =
+ (__Pyx_ModuleStateLookupData*)realloc(
+ new_data,
+ sizeof(__Pyx_ModuleStateLookupData) +
+ (to_allocate-1)*sizeof(__Pyx_InterpreterIdAndModule));
+ if (!new_data) {
+ result = -1;
+ new_data = old_data;
+ PyErr_NoMemory();
+ goto end;
+ }
+ new_data->allocated = to_allocate;
+ }
+ ++new_data->count;
+ int64_t last_id = interpreter_id;
+ PyObject *last_module = module;
+ for (Py_ssize_t i=insert_at; icount; ++i) {
+ int64_t current_id = new_data->table[i].id;
+ new_data->table[i].id = last_id;
+ last_id = current_id;
+ PyObject *current_module = new_data->table[i].module;
+ new_data->table[i].module = last_module;
+ last_module = current_module;
+ }
+ }
+ end:
+#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE
+ __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, new_data);
+#else
+ __Pyx_ModuleStateLookup_data = new_data;
+#endif
+ __Pyx_ModuleStateLookup_Unlock();
+ return result;
+}
+static int __Pyx_State_RemoveModule(CYTHON_UNUSED void* dummy) {
+ int64_t interpreter_id = PyInterpreterState_GetID(__Pyx_PyInterpreterState_Get());
+ if (interpreter_id == -1) return -1;
+ __Pyx_ModuleStateLookup_Lock();
+#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE
+ __Pyx_ModuleStateLookupData *data = (__Pyx_ModuleStateLookupData *)
+ __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, 0);
+#else
+ __Pyx_ModuleStateLookupData *data = __Pyx_ModuleStateLookup_data;
+#endif
+ if (data->interpreter_id_as_index) {
+ if (interpreter_id < data->count) {
+ data->table[interpreter_id].module = NULL;
+ }
+ goto done;
+ }
+ {
+ __Pyx_ModuleStateLookup_wait_until_no_readers();
+ __Pyx_InterpreterIdAndModule* lower_bound = __Pyx_State_FindModuleStateLookupTableLowerBound(
+ data->table, data->count, interpreter_id);
+ if (!lower_bound) goto done;
+ if (lower_bound->id != interpreter_id) goto done;
+ __Pyx_InterpreterIdAndModule *end = data->table+data->count;
+ for (;lower_boundid = (lower_bound+1)->id;
+ lower_bound->module = (lower_bound+1)->module;
+ }
+ }
+ --data->count;
+ if (data->count == 0) {
+ free(data);
+ data = NULL;
+ }
+ done:
+#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE
+ __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, data);
+#else
+ __Pyx_ModuleStateLookup_data = data;
+#endif
+ __Pyx_ModuleStateLookup_Unlock();
+ return 0;
+}
+#endif
+
+/* #### Code section: utility_code_pragmas_end ### */
+#ifdef _MSC_VER
+#pragma warning( pop )
+#endif
+
+
+
+/* #### Code section: end ### */
+#endif /* Py_PYTHON_H */
diff --git a/venv/lib/python3.10/site-packages/fontTools/cu2qu/cu2qu.py b/venv/lib/python3.10/site-packages/fontTools/cu2qu/cu2qu.py
new file mode 100644
index 0000000000000000000000000000000000000000..150c03fb4a06d7d5e875a9450c958693c601a6fa
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/fontTools/cu2qu/cu2qu.py
@@ -0,0 +1,563 @@
+# cython: language_level=3
+# distutils: define_macros=CYTHON_TRACE_NOGIL=1
+
+# Copyright 2015 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+try:
+ import cython
+except (AttributeError, ImportError):
+ # if cython not installed, use mock module with no-op decorators and types
+ from fontTools.misc import cython
+COMPILED = cython.compiled
+
+import math
+
+from .errors import Error as Cu2QuError, ApproxNotFoundError
+
+
+__all__ = ["curve_to_quadratic", "curves_to_quadratic"]
+
+MAX_N = 100
+
+NAN = float("NaN")
+
+
+@cython.cfunc
+@cython.inline
+@cython.returns(cython.double)
+@cython.locals(v1=cython.complex, v2=cython.complex, result=cython.double)
+def dot(v1, v2):
+ """Return the dot product of two vectors.
+
+ Args:
+ v1 (complex): First vector.
+ v2 (complex): Second vector.
+
+ Returns:
+ double: Dot product.
+ """
+ result = (v1 * v2.conjugate()).real
+ # When vectors are perpendicular (i.e. dot product is 0), the above expression may
+ # yield slightly different results when running in pure Python vs C/Cython,
+ # both of which are correct within IEEE-754 floating-point precision.
+ # It's probably due to the different order of operations and roundings in each
+ # implementation. Because we are using the result in a denominator and catching
+ # ZeroDivisionError (see `calc_intersect`), it's best to normalize the result here.
+ if abs(result) < 1e-15:
+ result = 0.0
+ return result
+
+
+@cython.cfunc
+@cython.locals(z=cython.complex, den=cython.double)
+@cython.locals(zr=cython.double, zi=cython.double)
+def _complex_div_by_real(z, den):
+ """Divide complex by real using Python's method (two separate divisions).
+
+ This ensures bit-exact compatibility with Python's complex division,
+ avoiding C's multiply-by-reciprocal optimization that can cause 1 ULP differences
+ on some platforms/compilers (e.g. clang on macOS arm64).
+
+ https://github.com/fonttools/fonttools/issues/3928
+ """
+ zr = z.real
+ zi = z.imag
+ return complex(zr / den, zi / den)
+
+
+@cython.cfunc
+@cython.inline
+@cython.locals(a=cython.complex, b=cython.complex, c=cython.complex, d=cython.complex)
+@cython.locals(
+ _1=cython.complex, _2=cython.complex, _3=cython.complex, _4=cython.complex
+)
+def calc_cubic_points(a, b, c, d):
+ _1 = d
+ _2 = _complex_div_by_real(c, 3.0) + d
+ _3 = _complex_div_by_real(b + c, 3.0) + _2
+ _4 = a + d + c + b
+ return _1, _2, _3, _4
+
+
+@cython.cfunc
+@cython.inline
+@cython.locals(
+ p0=cython.complex, p1=cython.complex, p2=cython.complex, p3=cython.complex
+)
+@cython.locals(a=cython.complex, b=cython.complex, c=cython.complex, d=cython.complex)
+def calc_cubic_parameters(p0, p1, p2, p3):
+ c = (p1 - p0) * 3.0
+ b = (p2 - p1) * 3.0 - c
+ d = p0
+ a = p3 - d - c - b
+ return a, b, c, d
+
+
+@cython.cfunc
+@cython.inline
+@cython.locals(
+ p0=cython.complex, p1=cython.complex, p2=cython.complex, p3=cython.complex
+)
+def split_cubic_into_n_iter(p0, p1, p2, p3, n):
+ """Split a cubic Bezier into n equal parts.
+
+ Splits the curve into `n` equal parts by curve time.
+ (t=0..1/n, t=1/n..2/n, ...)
+
+ Args:
+ p0 (complex): Start point of curve.
+ p1 (complex): First handle of curve.
+ p2 (complex): Second handle of curve.
+ p3 (complex): End point of curve.
+
+ Returns:
+ An iterator yielding the control points (four complex values) of the
+ subcurves.
+ """
+ # Hand-coded special-cases
+ if n == 2:
+ return iter(split_cubic_into_two(p0, p1, p2, p3))
+ if n == 3:
+ return iter(split_cubic_into_three(p0, p1, p2, p3))
+ if n == 4:
+ a, b = split_cubic_into_two(p0, p1, p2, p3)
+ return iter(
+ split_cubic_into_two(a[0], a[1], a[2], a[3])
+ + split_cubic_into_two(b[0], b[1], b[2], b[3])
+ )
+ if n == 6:
+ a, b = split_cubic_into_two(p0, p1, p2, p3)
+ return iter(
+ split_cubic_into_three(a[0], a[1], a[2], a[3])
+ + split_cubic_into_three(b[0], b[1], b[2], b[3])
+ )
+
+ return _split_cubic_into_n_gen(p0, p1, p2, p3, n)
+
+
+@cython.locals(
+ p0=cython.complex,
+ p1=cython.complex,
+ p2=cython.complex,
+ p3=cython.complex,
+ n=cython.int,
+)
+@cython.locals(a=cython.complex, b=cython.complex, c=cython.complex, d=cython.complex)
+@cython.locals(
+ dt=cython.double, delta_2=cython.double, delta_3=cython.double, i=cython.int
+)
+@cython.locals(
+ a1=cython.complex, b1=cython.complex, c1=cython.complex, d1=cython.complex
+)
+def _split_cubic_into_n_gen(p0, p1, p2, p3, n):
+ a, b, c, d = calc_cubic_parameters(p0, p1, p2, p3)
+ dt = 1 / n
+ delta_2 = dt * dt
+ delta_3 = dt * delta_2
+ for i in range(n):
+ t1 = i * dt
+ t1_2 = t1 * t1
+ # calc new a, b, c and d
+ a1 = a * delta_3
+ b1 = (3 * a * t1 + b) * delta_2
+ c1 = (2 * b * t1 + c + 3 * a * t1_2) * dt
+ d1 = a * t1 * t1_2 + b * t1_2 + c * t1 + d
+ yield calc_cubic_points(a1, b1, c1, d1)
+
+
+@cython.cfunc
+@cython.inline
+@cython.locals(
+ p0=cython.complex, p1=cython.complex, p2=cython.complex, p3=cython.complex
+)
+@cython.locals(mid=cython.complex, deriv3=cython.complex)
+def split_cubic_into_two(p0, p1, p2, p3):
+ """Split a cubic Bezier into two equal parts.
+
+ Splits the curve into two equal parts at t = 0.5
+
+ Args:
+ p0 (complex): Start point of curve.
+ p1 (complex): First handle of curve.
+ p2 (complex): Second handle of curve.
+ p3 (complex): End point of curve.
+
+ Returns:
+ tuple: Two cubic Beziers (each expressed as a tuple of four complex
+ values).
+ """
+ mid = (p0 + 3 * (p1 + p2) + p3) * 0.125
+ deriv3 = (p3 + p2 - p1 - p0) * 0.125
+ return (
+ (p0, (p0 + p1) * 0.5, mid - deriv3, mid),
+ (mid, mid + deriv3, (p2 + p3) * 0.5, p3),
+ )
+
+
+@cython.cfunc
+@cython.inline
+@cython.locals(
+ p0=cython.complex,
+ p1=cython.complex,
+ p2=cython.complex,
+ p3=cython.complex,
+)
+@cython.locals(
+ mid1=cython.complex,
+ deriv1=cython.complex,
+ mid2=cython.complex,
+ deriv2=cython.complex,
+)
+def split_cubic_into_three(p0, p1, p2, p3):
+ """Split a cubic Bezier into three equal parts.
+
+ Splits the curve into three equal parts at t = 1/3 and t = 2/3
+
+ Args:
+ p0 (complex): Start point of curve.
+ p1 (complex): First handle of curve.
+ p2 (complex): Second handle of curve.
+ p3 (complex): End point of curve.
+
+ Returns:
+ tuple: Three cubic Beziers (each expressed as a tuple of four complex
+ values).
+ """
+ mid1 = (8 * p0 + 12 * p1 + 6 * p2 + p3) * (1 / 27)
+ deriv1 = (p3 + 3 * p2 - 4 * p0) * (1 / 27)
+ mid2 = (p0 + 6 * p1 + 12 * p2 + 8 * p3) * (1 / 27)
+ deriv2 = (4 * p3 - 3 * p1 - p0) * (1 / 27)
+ return (
+ (p0, (2 * p0 + p1) / 3.0, mid1 - deriv1, mid1),
+ (mid1, mid1 + deriv1, mid2 - deriv2, mid2),
+ (mid2, mid2 + deriv2, (p2 + 2 * p3) / 3.0, p3),
+ )
+
+
+@cython.cfunc
+@cython.inline
+@cython.returns(cython.complex)
+@cython.locals(
+ t=cython.double,
+ p0=cython.complex,
+ p1=cython.complex,
+ p2=cython.complex,
+ p3=cython.complex,
+)
+@cython.locals(_p1=cython.complex, _p2=cython.complex)
+def cubic_approx_control(t, p0, p1, p2, p3):
+ """Approximate a cubic Bezier using a quadratic one.
+
+ Args:
+ t (double): Position of control point.
+ p0 (complex): Start point of curve.
+ p1 (complex): First handle of curve.
+ p2 (complex): Second handle of curve.
+ p3 (complex): End point of curve.
+
+ Returns:
+ complex: Location of candidate control point on quadratic curve.
+ """
+ _p1 = p0 + (p1 - p0) * 1.5
+ _p2 = p3 + (p2 - p3) * 1.5
+ return _p1 + (_p2 - _p1) * t
+
+
+@cython.cfunc
+@cython.inline
+@cython.returns(cython.complex)
+@cython.locals(a=cython.complex, b=cython.complex, c=cython.complex, d=cython.complex)
+@cython.locals(ab=cython.complex, cd=cython.complex, p=cython.complex, h=cython.double)
+def calc_intersect(a, b, c, d):
+ """Calculate the intersection of two lines.
+
+ Args:
+ a (complex): Start point of first line.
+ b (complex): End point of first line.
+ c (complex): Start point of second line.
+ d (complex): End point of second line.
+
+ Returns:
+ complex: Location of intersection if one present, ``complex(NaN,NaN)``
+ if no intersection was found.
+ """
+ ab = b - a
+ cd = d - c
+ p = ab * 1j
+ try:
+ h = dot(p, a - c) / dot(p, cd)
+ except ZeroDivisionError:
+ # if 3 or 4 points are equal, we do have an intersection despite the zero-div:
+ # return one of the off-curves so that the algorithm can attempt a one-curve
+ # solution if it's within tolerance:
+ # https://github.com/linebender/kurbo/pull/484
+ if b == c and (a == b or c == d):
+ return b
+ return complex(NAN, NAN)
+ return c + cd * h
+
+
+@cython.cfunc
+@cython.returns(cython.int)
+@cython.locals(
+ tolerance=cython.double,
+ p0=cython.complex,
+ p1=cython.complex,
+ p2=cython.complex,
+ p3=cython.complex,
+)
+@cython.locals(mid=cython.complex, deriv3=cython.complex)
+def cubic_farthest_fit_inside(p0, p1, p2, p3, tolerance):
+ """Check if a cubic Bezier lies within a given distance of the origin.
+
+ "Origin" means *the* origin (0,0), not the start of the curve. Note that no
+ checks are made on the start and end positions of the curve; this function
+ only checks the inside of the curve.
+
+ Args:
+ p0 (complex): Start point of curve.
+ p1 (complex): First handle of curve.
+ p2 (complex): Second handle of curve.
+ p3 (complex): End point of curve.
+ tolerance (double): Distance from origin.
+
+ Returns:
+ bool: True if the cubic Bezier ``p`` entirely lies within a distance
+ ``tolerance`` of the origin, False otherwise.
+ """
+ # First check p2 then p1, as p2 has higher error early on.
+ if abs(p2) <= tolerance and abs(p1) <= tolerance:
+ return True
+
+ # Split.
+ mid = (p0 + 3 * (p1 + p2) + p3) * 0.125
+ if abs(mid) > tolerance:
+ return False
+ deriv3 = (p3 + p2 - p1 - p0) * 0.125
+ return cubic_farthest_fit_inside(
+ p0, (p0 + p1) * 0.5, mid - deriv3, mid, tolerance
+ ) and cubic_farthest_fit_inside(mid, mid + deriv3, (p2 + p3) * 0.5, p3, tolerance)
+
+
+@cython.cfunc
+@cython.inline
+@cython.locals(tolerance=cython.double)
+@cython.locals(
+ q1=cython.complex,
+ c0=cython.complex,
+ c1=cython.complex,
+ c2=cython.complex,
+ c3=cython.complex,
+)
+def cubic_approx_quadratic(cubic, tolerance):
+ """Approximate a cubic Bezier with a single quadratic within a given tolerance.
+
+ Args:
+ cubic (sequence): Four complex numbers representing control points of
+ the cubic Bezier curve.
+ tolerance (double): Permitted deviation from the original curve.
+
+ Returns:
+ Three complex numbers representing control points of the quadratic
+ curve if it fits within the given tolerance, or ``None`` if no suitable
+ curve could be calculated.
+ """
+
+ q1 = calc_intersect(cubic[0], cubic[1], cubic[2], cubic[3])
+ if math.isnan(q1.imag):
+ return None
+ c0 = cubic[0]
+ c3 = cubic[3]
+ c1 = c0 + (q1 - c0) * (2 / 3)
+ c2 = c3 + (q1 - c3) * (2 / 3)
+ if not cubic_farthest_fit_inside(0, c1 - cubic[1], c2 - cubic[2], 0, tolerance):
+ return None
+ return c0, q1, c3
+
+
+@cython.cfunc
+@cython.locals(n=cython.int, tolerance=cython.double)
+@cython.locals(i=cython.int)
+@cython.locals(all_quadratic=cython.int)
+@cython.locals(
+ c0=cython.complex, c1=cython.complex, c2=cython.complex, c3=cython.complex
+)
+@cython.locals(
+ q0=cython.complex,
+ q1=cython.complex,
+ next_q1=cython.complex,
+ q2=cython.complex,
+ d1=cython.complex,
+)
+def cubic_approx_spline(cubic, n, tolerance, all_quadratic):
+ """Approximate a cubic Bezier curve with a spline of n quadratics.
+
+ Args:
+ cubic (sequence): Four complex numbers representing control points of
+ the cubic Bezier curve.
+ n (int): Number of quadratic Bezier curves in the spline.
+ tolerance (double): Permitted deviation from the original curve.
+
+ Returns:
+ A list of ``n+2`` complex numbers, representing control points of the
+ quadratic spline if it fits within the given tolerance, or ``None`` if
+ no suitable spline could be calculated.
+ """
+
+ if n == 1:
+ return cubic_approx_quadratic(cubic, tolerance)
+ if n == 2 and all_quadratic == False:
+ return cubic
+
+ cubics = split_cubic_into_n_iter(cubic[0], cubic[1], cubic[2], cubic[3], n)
+
+ # calculate the spline of quadratics and check errors at the same time.
+ next_cubic = next(cubics)
+ next_q1 = cubic_approx_control(
+ 0, next_cubic[0], next_cubic[1], next_cubic[2], next_cubic[3]
+ )
+ q2 = cubic[0]
+ d1 = 0j
+ spline = [cubic[0], next_q1]
+ for i in range(1, n + 1):
+ # Current cubic to convert
+ c0, c1, c2, c3 = next_cubic
+
+ # Current quadratic approximation of current cubic
+ q0 = q2
+ q1 = next_q1
+ if i < n:
+ next_cubic = next(cubics)
+ next_q1 = cubic_approx_control(
+ i / (n - 1), next_cubic[0], next_cubic[1], next_cubic[2], next_cubic[3]
+ )
+ spline.append(next_q1)
+ q2 = (q1 + next_q1) * 0.5
+ else:
+ q2 = c3
+
+ # End-point deltas
+ d0 = d1
+ d1 = q2 - c3
+
+ if abs(d1) > tolerance or not cubic_farthest_fit_inside(
+ d0,
+ q0 + (q1 - q0) * (2 / 3) - c1,
+ q2 + (q1 - q2) * (2 / 3) - c2,
+ d1,
+ tolerance,
+ ):
+ return None
+ spline.append(cubic[3])
+
+ return spline
+
+
+@cython.locals(max_err=cython.double)
+@cython.locals(n=cython.int)
+@cython.locals(all_quadratic=cython.int)
+def curve_to_quadratic(curve, max_err, all_quadratic=True):
+ """Approximate a cubic Bezier curve with a spline of n quadratics.
+
+ Args:
+ cubic (sequence): Four 2D tuples representing control points of
+ the cubic Bezier curve.
+ max_err (double): Permitted deviation from the original curve.
+ all_quadratic (bool): If True (default) returned value is a
+ quadratic spline. If False, it's either a single quadratic
+ curve or a single cubic curve.
+
+ Returns:
+ If all_quadratic is True: A list of 2D tuples, representing
+ control points of the quadratic spline if it fits within the
+ given tolerance, or ``None`` if no suitable spline could be
+ calculated.
+
+ If all_quadratic is False: Either a quadratic curve (if length
+ of output is 3), or a cubic curve (if length of output is 4).
+ """
+
+ curve = [complex(*p) for p in curve]
+
+ for n in range(1, MAX_N + 1):
+ spline = cubic_approx_spline(curve, n, max_err, all_quadratic)
+ if spline is not None:
+ # done. go home
+ return [(s.real, s.imag) for s in spline]
+
+ raise ApproxNotFoundError(curve)
+
+
+@cython.locals(l=cython.int, last_i=cython.int, i=cython.int)
+@cython.locals(all_quadratic=cython.int)
+def curves_to_quadratic(curves, max_errors, all_quadratic=True):
+ """Return quadratic Bezier splines approximating the input cubic Beziers.
+
+ Args:
+ curves: A sequence of *n* curves, each curve being a sequence of four
+ 2D tuples.
+ max_errors: A sequence of *n* floats representing the maximum permissible
+ deviation from each of the cubic Bezier curves.
+ all_quadratic (bool): If True (default) returned values are a
+ quadratic spline. If False, they are either a single quadratic
+ curve or a single cubic curve.
+
+ Example::
+
+ >>> curves_to_quadratic( [
+ ... [ (50,50), (100,100), (150,100), (200,50) ],
+ ... [ (75,50), (120,100), (150,75), (200,60) ]
+ ... ], [1,1] )
+ [[(50.0, 50.0), (75.0, 75.0), (125.0, 91.66666666666666), (175.0, 75.0), (200.0, 50.0)], [(75.0, 50.0), (97.5, 75.0), (135.41666666666666, 82.08333333333333), (175.0, 67.5), (200.0, 60.0)]]
+
+ The returned splines have "implied oncurve points" suitable for use in
+ TrueType ``glif`` outlines - i.e. in the first spline returned above,
+ the first quadratic segment runs from (50,50) to
+ ( (75 + 125)/2 , (120 + 91.666..)/2 ) = (100, 83.333...).
+
+ Returns:
+ If all_quadratic is True, a list of splines, each spline being a list
+ of 2D tuples.
+
+ If all_quadratic is False, a list of curves, each curve being a quadratic
+ (length 3), or cubic (length 4).
+
+ Raises:
+ fontTools.cu2qu.Errors.ApproxNotFoundError: if no suitable approximation
+ can be found for all curves with the given parameters.
+ """
+
+ curves = [[complex(*p) for p in curve] for curve in curves]
+ assert len(max_errors) == len(curves)
+
+ l = len(curves)
+ splines = [None] * l
+ last_i = i = 0
+ n = 1
+ while True:
+ spline = cubic_approx_spline(curves[i], n, max_errors[i], all_quadratic)
+ if spline is None:
+ if n == MAX_N:
+ break
+ n += 1
+ last_i = i
+ continue
+ splines[i] = spline
+ i = (i + 1) % l
+ if i == last_i:
+ # done. go home
+ return [[(s.real, s.imag) for s in spline] for spline in splines]
+
+ raise ApproxNotFoundError(curves)
diff --git a/venv/lib/python3.10/site-packages/fontTools/cu2qu/errors.py b/venv/lib/python3.10/site-packages/fontTools/cu2qu/errors.py
new file mode 100644
index 0000000000000000000000000000000000000000..fa3dc42937131c5db54890dde8f519b15f5d0ff1
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/fontTools/cu2qu/errors.py
@@ -0,0 +1,77 @@
+# Copyright 2016 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+class Error(Exception):
+ """Base Cu2Qu exception class for all other errors."""
+
+
+class ApproxNotFoundError(Error):
+ def __init__(self, curve):
+ message = "no approximation found: %s" % curve
+ super().__init__(message)
+ self.curve = curve
+
+
+class UnequalZipLengthsError(Error):
+ pass
+
+
+class IncompatibleGlyphsError(Error):
+ def __init__(self, glyphs):
+ assert len(glyphs) > 1
+ self.glyphs = glyphs
+ names = set(repr(g.name) for g in glyphs)
+ if len(names) > 1:
+ self.combined_name = "{%s}" % ", ".join(sorted(names))
+ else:
+ self.combined_name = names.pop()
+
+ def __repr__(self):
+ return "<%s %s>" % (type(self).__name__, self.combined_name)
+
+
+class IncompatibleSegmentNumberError(IncompatibleGlyphsError):
+ def __str__(self):
+ return "Glyphs named %s have different number of segments" % (
+ self.combined_name
+ )
+
+
+class IncompatibleSegmentTypesError(IncompatibleGlyphsError):
+ def __init__(self, glyphs, segments):
+ IncompatibleGlyphsError.__init__(self, glyphs)
+ self.segments = segments
+
+ def __str__(self):
+ lines = []
+ ndigits = len(str(max(self.segments)))
+ for i, tags in sorted(self.segments.items()):
+ lines.append(
+ "%s: (%s)" % (str(i).rjust(ndigits), ", ".join(repr(t) for t in tags))
+ )
+ return "Glyphs named %s have incompatible segment types:\n %s" % (
+ self.combined_name,
+ "\n ".join(lines),
+ )
+
+
+class IncompatibleFontsError(Error):
+ def __init__(self, glyph_errors):
+ self.glyph_errors = glyph_errors
+
+ def __str__(self):
+ return "fonts contains incompatible glyphs: %s" % (
+ ", ".join(repr(g) for g in sorted(self.glyph_errors.keys()))
+ )
diff --git a/venv/lib/python3.10/site-packages/fontTools/cu2qu/ufo.py b/venv/lib/python3.10/site-packages/fontTools/cu2qu/ufo.py
new file mode 100644
index 0000000000000000000000000000000000000000..7a6dbc67f8a5db26a309bfe52678be1bf3cf5f19
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/fontTools/cu2qu/ufo.py
@@ -0,0 +1,349 @@
+# Copyright 2015 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+"""Converts cubic bezier curves to quadratic splines.
+
+Conversion is performed such that the quadratic splines keep the same end-curve
+tangents as the original cubics. The approach is iterative, increasing the
+number of segments for a spline until the error gets below a bound.
+
+Respective curves from multiple fonts will be converted at once to ensure that
+the resulting splines are interpolation-compatible.
+"""
+
+import logging
+from fontTools.pens.basePen import AbstractPen
+from fontTools.pens.pointPen import PointToSegmentPen
+from fontTools.pens.reverseContourPen import ReverseContourPen
+
+from . import curves_to_quadratic
+from .errors import (
+ UnequalZipLengthsError,
+ IncompatibleSegmentNumberError,
+ IncompatibleSegmentTypesError,
+ IncompatibleGlyphsError,
+ IncompatibleFontsError,
+)
+
+
+__all__ = ["fonts_to_quadratic", "font_to_quadratic"]
+
+# The default approximation error below is a relative value (1/1000 of the EM square).
+# Later on, we convert it to absolute font units by multiplying it by a font's UPEM
+# (see fonts_to_quadratic).
+DEFAULT_MAX_ERR = 0.001
+CURVE_TYPE_LIB_KEY = "com.github.googlei18n.cu2qu.curve_type"
+
+logger = logging.getLogger(__name__)
+
+
+_zip = zip
+
+
+def zip(*args):
+ """Ensure each argument to zip has the same length. Also make sure a list is
+ returned for python 2/3 compatibility.
+ """
+
+ if len(set(len(a) for a in args)) != 1:
+ raise UnequalZipLengthsError(*args)
+ return list(_zip(*args))
+
+
+class GetSegmentsPen(AbstractPen):
+ """Pen to collect segments into lists of points for conversion.
+
+ Curves always include their initial on-curve point, so some points are
+ duplicated between segments.
+ """
+
+ def __init__(self):
+ self._last_pt = None
+ self.segments = []
+
+ def _add_segment(self, tag, *args):
+ if tag in ["move", "line", "qcurve", "curve"]:
+ self._last_pt = args[-1]
+ self.segments.append((tag, args))
+
+ def moveTo(self, pt):
+ self._add_segment("move", pt)
+
+ def lineTo(self, pt):
+ self._add_segment("line", pt)
+
+ def qCurveTo(self, *points):
+ self._add_segment("qcurve", self._last_pt, *points)
+
+ def curveTo(self, *points):
+ self._add_segment("curve", self._last_pt, *points)
+
+ def closePath(self):
+ self._add_segment("close")
+
+ def endPath(self):
+ self._add_segment("end")
+
+ def addComponent(self, glyphName, transformation):
+ pass
+
+
+def _get_segments(glyph):
+ """Get a glyph's segments as extracted by GetSegmentsPen."""
+
+ pen = GetSegmentsPen()
+ # glyph.draw(pen)
+ # We can't simply draw the glyph with the pen, but we must initialize the
+ # PointToSegmentPen explicitly with outputImpliedClosingLine=True.
+ # By default PointToSegmentPen does not outputImpliedClosingLine -- unless
+ # last and first point on closed contour are duplicated. Because we are
+ # converting multiple glyphs at the same time, we want to make sure
+ # this function returns the same number of segments, whether or not
+ # the last and first point overlap.
+ # https://github.com/googlefonts/fontmake/issues/572
+ # https://github.com/fonttools/fonttools/pull/1720
+ pointPen = PointToSegmentPen(pen, outputImpliedClosingLine=True)
+ glyph.drawPoints(pointPen)
+ return pen.segments
+
+
+def _set_segments(glyph, segments, reverse_direction):
+ """Draw segments as extracted by GetSegmentsPen back to a glyph."""
+
+ glyph.clearContours()
+ pen = glyph.getPen()
+ if reverse_direction:
+ pen = ReverseContourPen(pen)
+ for tag, args in segments:
+ if tag == "move":
+ pen.moveTo(*args)
+ elif tag == "line":
+ pen.lineTo(*args)
+ elif tag == "curve":
+ pen.curveTo(*args[1:])
+ elif tag == "qcurve":
+ pen.qCurveTo(*args[1:])
+ elif tag == "close":
+ pen.closePath()
+ elif tag == "end":
+ pen.endPath()
+ else:
+ raise AssertionError('Unhandled segment type "%s"' % tag)
+
+
+def _segments_to_quadratic(segments, max_err, stats, all_quadratic=True):
+ """Return quadratic approximations of cubic segments."""
+
+ assert all(s[0] == "curve" for s in segments), "Non-cubic given to convert"
+
+ new_points = curves_to_quadratic([s[1] for s in segments], max_err, all_quadratic)
+ n = len(new_points[0])
+ assert all(len(s) == n for s in new_points[1:]), "Converted incompatibly"
+
+ spline_length = str(n - 2)
+ stats[spline_length] = stats.get(spline_length, 0) + 1
+
+ if all_quadratic or n == 3:
+ return [("qcurve", p) for p in new_points]
+ else:
+ return [("curve", p) for p in new_points]
+
+
+def _glyphs_to_quadratic(glyphs, max_err, reverse_direction, stats, all_quadratic=True):
+ """Do the actual conversion of a set of compatible glyphs, after arguments
+ have been set up.
+
+ Return True if the glyphs were modified, else return False.
+ """
+
+ try:
+ segments_by_location = zip(*[_get_segments(g) for g in glyphs])
+ except UnequalZipLengthsError:
+ raise IncompatibleSegmentNumberError(glyphs)
+ if not any(segments_by_location):
+ return False
+
+ # always modify input glyphs if reverse_direction is True
+ glyphs_modified = reverse_direction
+
+ new_segments_by_location = []
+ incompatible = {}
+ for i, segments in enumerate(segments_by_location):
+ tag = segments[0][0]
+ if not all(s[0] == tag for s in segments[1:]):
+ incompatible[i] = [s[0] for s in segments]
+ elif tag == "curve":
+ new_segments = _segments_to_quadratic(
+ segments, max_err, stats, all_quadratic
+ )
+ if all_quadratic or new_segments != segments:
+ glyphs_modified = True
+ segments = new_segments
+ new_segments_by_location.append(segments)
+
+ if glyphs_modified:
+ new_segments_by_glyph = zip(*new_segments_by_location)
+ for glyph, new_segments in zip(glyphs, new_segments_by_glyph):
+ _set_segments(glyph, new_segments, reverse_direction)
+
+ if incompatible:
+ raise IncompatibleSegmentTypesError(glyphs, segments=incompatible)
+ return glyphs_modified
+
+
+def glyphs_to_quadratic(
+ glyphs, max_err=None, reverse_direction=False, stats=None, all_quadratic=True
+):
+ """Convert the curves of a set of compatible of glyphs to quadratic.
+
+ All curves will be converted to quadratic at once, ensuring interpolation
+ compatibility. If this is not required, calling glyphs_to_quadratic with one
+ glyph at a time may yield slightly more optimized results.
+
+ Return True if glyphs were modified, else return False.
+
+ Raises IncompatibleGlyphsError if glyphs have non-interpolatable outlines.
+ """
+ if stats is None:
+ stats = {}
+
+ if not max_err:
+ # assume 1000 is the default UPEM
+ max_err = DEFAULT_MAX_ERR * 1000
+
+ if isinstance(max_err, (list, tuple)):
+ max_errors = max_err
+ else:
+ max_errors = [max_err] * len(glyphs)
+ assert len(max_errors) == len(glyphs)
+
+ return _glyphs_to_quadratic(
+ glyphs, max_errors, reverse_direction, stats, all_quadratic
+ )
+
+
+def fonts_to_quadratic(
+ fonts,
+ max_err_em=None,
+ max_err=None,
+ reverse_direction=False,
+ stats=None,
+ dump_stats=False,
+ remember_curve_type=True,
+ all_quadratic=True,
+):
+ """Convert the curves of a collection of fonts to quadratic.
+
+ All curves will be converted to quadratic at once, ensuring interpolation
+ compatibility. If this is not required, calling fonts_to_quadratic with one
+ font at a time may yield slightly more optimized results.
+
+ Return the set of modified glyph names if any, else return an empty set.
+
+ By default, cu2qu stores the curve type in the fonts' lib, under a private
+ key "com.github.googlei18n.cu2qu.curve_type", and will not try to convert
+ them again if the curve type is already set to "quadratic".
+ Setting 'remember_curve_type' to False disables this optimization.
+
+ Raises IncompatibleFontsError if same-named glyphs from different fonts
+ have non-interpolatable outlines.
+ """
+
+ if remember_curve_type:
+ curve_types = {f.lib.get(CURVE_TYPE_LIB_KEY, "cubic") for f in fonts}
+ if len(curve_types) == 1:
+ curve_type = next(iter(curve_types))
+ if curve_type in ("quadratic", "mixed"):
+ logger.info("Curves already converted to quadratic")
+ return False
+ elif curve_type == "cubic":
+ pass # keep converting
+ else:
+ raise NotImplementedError(curve_type)
+ elif len(curve_types) > 1:
+ # going to crash later if they do differ
+ logger.warning("fonts may contain different curve types")
+
+ if stats is None:
+ stats = {}
+
+ if max_err_em and max_err:
+ raise TypeError("Only one of max_err and max_err_em can be specified.")
+ if not (max_err_em or max_err):
+ max_err_em = DEFAULT_MAX_ERR
+
+ if isinstance(max_err, (list, tuple)):
+ assert len(max_err) == len(fonts)
+ max_errors = max_err
+ elif max_err:
+ max_errors = [max_err] * len(fonts)
+
+ if isinstance(max_err_em, (list, tuple)):
+ assert len(fonts) == len(max_err_em)
+ max_errors = [f.info.unitsPerEm * e for f, e in zip(fonts, max_err_em)]
+ elif max_err_em:
+ max_errors = [f.info.unitsPerEm * max_err_em for f in fonts]
+
+ modified = set()
+ glyph_errors = {}
+ for name in set().union(*(f.keys() for f in fonts)):
+ glyphs = []
+ cur_max_errors = []
+ for font, error in zip(fonts, max_errors):
+ if name in font:
+ glyphs.append(font[name])
+ cur_max_errors.append(error)
+ try:
+ if _glyphs_to_quadratic(
+ glyphs, cur_max_errors, reverse_direction, stats, all_quadratic
+ ):
+ modified.add(name)
+ except IncompatibleGlyphsError as exc:
+ logger.error(exc)
+ glyph_errors[name] = exc
+
+ if glyph_errors:
+ raise IncompatibleFontsError(glyph_errors)
+
+ if modified and dump_stats:
+ spline_lengths = sorted(stats.keys())
+ logger.info(
+ "New spline lengths: %s"
+ % (", ".join("%s: %d" % (l, stats[l]) for l in spline_lengths))
+ )
+
+ if remember_curve_type:
+ for font in fonts:
+ curve_type = font.lib.get(CURVE_TYPE_LIB_KEY, "cubic")
+ new_curve_type = "quadratic" if all_quadratic else "mixed"
+ if curve_type != new_curve_type:
+ font.lib[CURVE_TYPE_LIB_KEY] = new_curve_type
+ return modified
+
+
+def glyph_to_quadratic(glyph, **kwargs):
+ """Convenience wrapper around glyphs_to_quadratic, for just one glyph.
+ Return True if the glyph was modified, else return False.
+ """
+
+ return glyphs_to_quadratic([glyph], **kwargs)
+
+
+def font_to_quadratic(font, **kwargs):
+ """Convenience wrapper around fonts_to_quadratic, for just one font.
+ Return the set of modified glyph names if any, else return empty set.
+ """
+
+ return fonts_to_quadratic([font], **kwargs)
diff --git a/venv/lib/python3.10/site-packages/fontTools/designspaceLib/__init__.py b/venv/lib/python3.10/site-packages/fontTools/designspaceLib/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..661f3405da108f88f0cadd4286d0053c141de284
--- /dev/null
+++ b/venv/lib/python3.10/site-packages/fontTools/designspaceLib/__init__.py
@@ -0,0 +1,3338 @@
+"""
+ designSpaceDocument
+
+ - Read and write designspace files
+"""
+
+from __future__ import annotations
+
+import collections
+import copy
+import itertools
+import math
+import os
+import posixpath
+from io import BytesIO, StringIO
+from textwrap import indent
+from typing import Any, Dict, List, MutableMapping, Optional, Tuple, Union, cast
+
+from fontTools.misc import etree as ET
+from fontTools.misc import plistlib
+from fontTools.misc.loggingTools import LogMixin
+from fontTools.misc.textTools import tobytes, tostr
+
+
+__all__ = [
+ "AxisDescriptor",
+ "AxisLabelDescriptor",
+ "AxisMappingDescriptor",
+ "BaseDocReader",
+ "BaseDocWriter",
+ "DesignSpaceDocument",
+ "DesignSpaceDocumentError",
+ "DiscreteAxisDescriptor",
+ "InstanceDescriptor",
+ "LocationLabelDescriptor",
+ "RangeAxisSubsetDescriptor",
+ "RuleDescriptor",
+ "SourceDescriptor",
+ "ValueAxisSubsetDescriptor",
+ "VariableFontDescriptor",
+]
+
+# ElementTree allows to find namespace-prefixed elements, but not attributes
+# so we have to do it ourselves for 'xml:lang'
+XML_NS = "{http://www.w3.org/XML/1998/namespace}"
+XML_LANG = XML_NS + "lang"
+
+
+def posix(path):
+ """Normalize paths using forward slash to work also on Windows."""
+ new_path = posixpath.join(*path.split(os.path.sep))
+ if path.startswith("/"):
+ # The above transformation loses absolute paths
+ new_path = "/" + new_path
+ elif path.startswith(r"\\"):
+ # The above transformation loses leading slashes of UNC path mounts
+ new_path = "//" + new_path
+ return new_path
+
+
+def posixpath_property(private_name):
+ """Generate a propery that holds a path always using forward slashes."""
+
+ def getter(self):
+ # Normal getter
+ return getattr(self, private_name)
+
+ def setter(self, value):
+ # The setter rewrites paths using forward slashes
+ if value is not None:
+ value = posix(value)
+ setattr(self, private_name, value)
+
+ return property(getter, setter)
+
+
+class DesignSpaceDocumentError(Exception):
+ def __init__(self, msg, obj=None):
+ self.msg = msg
+ self.obj = obj
+
+ def __str__(self):
+ return str(self.msg) + (": %r" % self.obj if self.obj is not None else "")
+
+
+class AsDictMixin(object):
+ def asdict(self):
+ d = {}
+ for attr, value in self.__dict__.items():
+ if attr.startswith("_"):
+ continue
+ if hasattr(value, "asdict"):
+ value = value.asdict()
+ elif isinstance(value, list):
+ value = [v.asdict() if hasattr(v, "asdict") else v for v in value]
+ d[attr] = value
+ return d
+
+
+class SimpleDescriptor(AsDictMixin):
+ """Containers for a bunch of attributes"""
+
+ # XXX this is ugly. The 'print' is inappropriate here, and instead of
+ # assert, it should simply return True/False
+ def compare(self, other):
+ # test if this object contains the same data as the other
+ for attr in self._attrs:
+ try:
+ assert getattr(self, attr) == getattr(other, attr)
+ except AssertionError:
+ print(
+ "failed attribute",
+ attr,
+ getattr(self, attr),
+ "!=",
+ getattr(other, attr),
+ )
+
+ def __repr__(self):
+ attrs = [f"{a}={repr(getattr(self, a))}," for a in self._attrs]
+ attrs = indent("\n".join(attrs), " ")
+ return f"{self.__class__.__name__}(\n{attrs}\n)"
+
+
+class SourceDescriptor(SimpleDescriptor):
+ """Simple container for data related to the source
+
+ .. code:: python
+
+ doc = DesignSpaceDocument()
+ s1 = SourceDescriptor()
+ s1.path = masterPath1
+ s1.name = "master.ufo1"
+ s1.font = defcon.Font("master.ufo1")
+ s1.location = dict(weight=0)
+ s1.familyName = "MasterFamilyName"
+ s1.styleName = "MasterStyleNameOne"
+ s1.localisedFamilyName = dict(fr="Caractère")
+ s1.mutedGlyphNames.append("A")
+ s1.mutedGlyphNames.append("Z")
+ doc.addSource(s1)
+
+ """
+
+ flavor = "source"
+ _attrs = [
+ "filename",
+ "path",
+ "name",
+ "layerName",
+ "location",
+ "copyLib",
+ "copyGroups",
+ "copyFeatures",
+ "muteKerning",
+ "muteInfo",
+ "mutedGlyphNames",
+ "familyName",
+ "styleName",
+ "localisedFamilyName",
+ ]
+
+ filename = posixpath_property("_filename")
+ path = posixpath_property("_path")
+
+ def __init__(
+ self,
+ *,
+ filename=None,
+ path=None,
+ font=None,
+ name=None,
+ location=None,
+ designLocation=None,
+ layerName=None,
+ familyName=None,
+ styleName=None,
+ localisedFamilyName=None,
+ copyLib=False,
+ copyInfo=False,
+ copyGroups=False,
+ copyFeatures=False,
+ muteKerning=False,
+ muteInfo=False,
+ mutedGlyphNames=None,
+ ):
+ self.filename = filename
+ """string. A relative path to the source file, **as it is in the document**.
+
+ MutatorMath + VarLib.
+ """
+ self.path = path
+ """The absolute path, calculated from filename."""
+
+ self.font = font
+ """Any Python object. Optional. Points to a representation of this
+ source font that is loaded in memory, as a Python object (e.g. a
+ ``defcon.Font`` or a ``fontTools.ttFont.TTFont``).
+
+ The default document reader will not fill-in this attribute, and the
+ default writer will not use this attribute. It is up to the user of
+ ``designspaceLib`` to either load the resource identified by
+ ``filename`` and store it in this field, or write the contents of
+ this field to the disk and make ```filename`` point to that.
+ """
+
+ self.name = name
+ """string. Optional. Unique identifier name for this source.
+
+ MutatorMath + varLib.
+ """
+
+ self.designLocation = (
+ designLocation if designLocation is not None else location or {}
+ )
+ """dict. Axis values for this source, in design space coordinates.
+
+ MutatorMath + varLib.
+
+ This may be only part of the full design location.
+ See :meth:`getFullDesignLocation()`
+
+ .. versionadded:: 5.0
+ """
+
+ self.layerName = layerName
+ """string. The name of the layer in the source to look for
+ outline data. Default ``None`` which means ``foreground``.
+ """
+ self.familyName = familyName
+ """string. Family name of this source. Though this data
+ can be extracted from the font, it can be efficient to have it right
+ here.
+
+ varLib.
+ """
+ self.styleName = styleName
+ """string. Style name of this source. Though this data
+ can be extracted from the font, it can be efficient to have it right
+ here.
+
+ varLib.
+ """
+ self.localisedFamilyName = localisedFamilyName or {}
+ """dict. A dictionary of localised family name strings, keyed by
+ language code.
+
+ If present, will be used to build localized names for all instances.
+
+ .. versionadded:: 5.0
+ """
+
+ self.copyLib = copyLib
+ """bool. Indicates if the contents of the font.lib need to
+ be copied to the instances.
+
+ MutatorMath.
+
+ .. deprecated:: 5.0
+ """
+ self.copyInfo = copyInfo
+ """bool. Indicates if the non-interpolating font.info needs
+ to be copied to the instances.
+
+ MutatorMath.
+
+ .. deprecated:: 5.0
+ """
+ self.copyGroups = copyGroups
+ """bool. Indicates if the groups need to be copied to the
+ instances.
+
+ MutatorMath.
+
+ .. deprecated:: 5.0
+ """
+ self.copyFeatures = copyFeatures
+ """bool. Indicates if the feature text needs to be
+ copied to the instances.
+
+ MutatorMath.
+
+ .. deprecated:: 5.0
+ """
+ self.muteKerning = muteKerning
+ """bool. Indicates if the kerning data from this source
+ needs to be muted (i.e. not be part of the calculations).
+
+ MutatorMath only.
+ """
+ self.muteInfo = muteInfo
+ """bool. Indicated if the interpolating font.info data for
+ this source needs to be muted.
+
+ MutatorMath only.
+ """
+ self.mutedGlyphNames = mutedGlyphNames or []
+ """list. Glyphnames that need to be muted in the
+ instances.
+
+ MutatorMath only.
+ """
+
+ @property
+ def location(self):
+ """dict. Axis values for this source, in design space coordinates.
+
+ MutatorMath + varLib.
+
+ .. deprecated:: 5.0
+ Use the more explicit alias for this property :attr:`designLocation`.
+ """
+ return self.designLocation
+
+ @location.setter
+ def location(self, location: Optional[SimpleLocationDict]):
+ self.designLocation = location or {}
+
+ def setFamilyName(self, familyName, languageCode="en"):
+ """Setter for :attr:`localisedFamilyName`
+
+ .. versionadded:: 5.0
+ """
+ self.localisedFamilyName[languageCode] = tostr(familyName)
+
+ def getFamilyName(self, languageCode="en"):
+ """Getter for :attr:`localisedFamilyName`
+
+ .. versionadded:: 5.0
+ """
+ return self.localisedFamilyName.get(languageCode)
+
+ def getFullDesignLocation(self, doc: "DesignSpaceDocument") -> SimpleLocationDict:
+ """Get the complete design location of this source, from its
+ :attr:`designLocation` and the document's axis defaults.
+
+ .. versionadded:: 5.0
+ """
+ result: SimpleLocationDict = {}
+ for axis in doc.axes:
+ if axis.name in self.designLocation:
+ result[axis.name] = self.designLocation[axis.name]
+ else:
+ result[axis.name] = axis.map_forward(axis.default)
+ return result
+
+
+class RuleDescriptor(SimpleDescriptor):
+ """Represents the rule descriptor element: a set of glyph substitutions to
+ trigger conditionally in some parts of the designspace.
+
+ .. code:: python
+
+ r1 = RuleDescriptor()
+ r1.name = "unique.rule.name"
+ r1.conditionSets.append([dict(name="weight", minimum=-10, maximum=10), dict(...)])
+ r1.conditionSets.append([dict(...), dict(...)])
+ r1.subs.append(("a", "a.alt"))
+
+ .. code:: xml
+
+
+
+
+
+
+
+
+
+
+
+
+
+ """
+
+ _attrs = ["name", "conditionSets", "subs"] # what do we need here
+
+ def __init__(self, *, name=None, conditionSets=None, subs=None):
+ self.name = name
+ """string. Unique name for this rule. Can be used to reference this rule data."""
+ # list of lists of dict(name='aaaa', minimum=0, maximum=1000)
+ self.conditionSets = conditionSets or []
+ """a list of conditionsets.
+
+ - Each conditionset is a list of conditions.
+ - Each condition is a dict with ``name``, ``minimum`` and ``maximum`` keys.
+ """
+ # list of substitutions stored as tuples of glyphnames ("a", "a.alt")
+ self.subs = subs or []
+ """list of substitutions.
+
+ - Each substitution is stored as tuples of glyphnames, e.g. ("a", "a.alt").
+ - Note: By default, rules are applied first, before other text
+ shaping/OpenType layout, as they are part of the
+ `Required Variation Alternates OpenType feature `_.
+ See ref:`rules-element` § Attributes.
+ """
+
+
+def evaluateRule(rule, location):
+ """Return True if any of the rule's conditionsets matches the given location."""
+ return any(evaluateConditions(c, location) for c in rule.conditionSets)
+
+
+def evaluateConditions(conditions, location):
+ """Return True if all the conditions matches the given location.
+
+ - If a condition has no minimum, check for < maximum.
+ - If a condition has no maximum, check for > minimum.
+ """
+ for cd in conditions:
+ value = location[cd["name"]]
+ if cd.get("minimum") is None:
+ if value > cd["maximum"]:
+ return False
+ elif cd.get("maximum") is None:
+ if cd["minimum"] > value:
+ return False
+ elif not cd["minimum"] <= value <= cd["maximum"]:
+ return False
+ return True
+
+
+def processRules(rules, location, glyphNames):
+ """Apply these rules at this location to these glyphnames.
+
+ Return a new list of glyphNames with substitutions applied.
+
+ - rule order matters
+ """
+ newNames = []
+ for rule in rules:
+ if evaluateRule(rule, location):
+ for name in glyphNames:
+ swap = False
+ for a, b in rule.subs:
+ if name == a:
+ swap = True
+ break
+ if swap:
+ newNames.append(b)
+ else:
+ newNames.append(name)
+ glyphNames = newNames
+ newNames = []
+ return glyphNames
+
+
+AnisotropicLocationDict = Dict[str, Union[float, Tuple[float, float]]]
+SimpleLocationDict = Dict[str, float]
+
+
+class AxisMappingDescriptor(SimpleDescriptor):
+ """Represents the axis mapping element: mapping an input location
+ to an output location in the designspace.
+
+ .. code:: python
+
+ m1 = AxisMappingDescriptor()
+ m1.inputLocation = {"weight": 900, "width": 150}
+ m1.outputLocation = {"weight": 870}
+
+ .. code:: xml
+
+
+
+
+
+
+
+
+
+
+ """
+
+ _attrs = ["inputLocation", "outputLocation"]
+
+ def __init__(
+ self,
+ *,
+ inputLocation=None,
+ outputLocation=None,
+ description=None,
+ groupDescription=None,
+ ):
+ self.inputLocation: SimpleLocationDict = inputLocation or {}
+ """dict. Axis values for the input of the mapping, in design space coordinates.
+
+ varLib.
+
+ .. versionadded:: 5.1
+ """
+ self.outputLocation: SimpleLocationDict = outputLocation or {}
+ """dict. Axis values for the output of the mapping, in design space coordinates.
+
+ varLib.
+
+ .. versionadded:: 5.1
+ """
+ self.description = description
+ """string. A description of the mapping.
+
+ varLib.
+
+ .. versionadded:: 5.2
+ """
+ self.groupDescription = groupDescription
+ """string. A description of the group of mappings.
+
+ varLib.
+
+ .. versionadded:: 5.2
+ """
+
+
+class InstanceDescriptor(SimpleDescriptor):
+ """Simple container for data related to the instance
+
+
+ .. code:: python
+
+ i2 = InstanceDescriptor()
+ i2.path = instancePath2
+ i2.familyName = "InstanceFamilyName"
+ i2.styleName = "InstanceStyleName"
+ i2.name = "instance.ufo2"
+ # anisotropic location
+ i2.designLocation = dict(weight=500, width=(400,300))
+ i2.postScriptFontName = "InstancePostscriptName"
+ i2.styleMapFamilyName = "InstanceStyleMapFamilyName"
+ i2.styleMapStyleName = "InstanceStyleMapStyleName"
+ i2.lib['com.coolDesignspaceApp.specimenText'] = 'Hamburgerwhatever'
+ doc.addInstance(i2)
+ """
+
+ flavor = "instance"
+ _defaultLanguageCode = "en"
+ _attrs = [
+ "filename",
+ "path",
+ "name",
+ "locationLabel",
+ "designLocation",
+ "userLocation",
+ "familyName",
+ "styleName",
+ "postScriptFontName",
+ "styleMapFamilyName",
+ "styleMapStyleName",
+ "localisedFamilyName",
+ "localisedStyleName",
+ "localisedStyleMapFamilyName",
+ "localisedStyleMapStyleName",
+ "glyphs",
+ "kerning",
+ "info",
+ "lib",
+ ]
+
+ filename = posixpath_property("_filename")
+ path = posixpath_property("_path")
+
+ def __init__(
+ self,
+ *,
+ filename=None,
+ path=None,
+ font=None,
+ name=None,
+ location=None,
+ locationLabel=None,
+ designLocation=None,
+ userLocation=None,
+ familyName=None,
+ styleName=None,
+ postScriptFontName=None,
+ styleMapFamilyName=None,
+ styleMapStyleName=None,
+ localisedFamilyName=None,
+ localisedStyleName=None,
+ localisedStyleMapFamilyName=None,
+ localisedStyleMapStyleName=None,
+ glyphs=None,
+ kerning=True,
+ info=True,
+ lib=None,
+ ):
+ self.filename = filename
+ """string. Relative path to the instance file, **as it is
+ in the document**. The file may or may not exist.
+
+ MutatorMath + VarLib.
+ """
+ self.path = path
+ """string. Absolute path to the instance file, calculated from
+ the document path and the string in the filename attr. The file may
+ or may not exist.
+
+ MutatorMath.
+ """
+ self.font = font
+ """Same as :attr:`SourceDescriptor.font`
+
+ .. seealso:: :attr:`SourceDescriptor.font`
+ """
+ self.name = name
+ """string. Unique identifier name of the instance, used to
+ identify it if it needs to be referenced from elsewhere in the
+ document.
+ """
+ self.locationLabel = locationLabel
+ """Name of a :class:`LocationLabelDescriptor`. If
+ provided, the instance should have the same location as the
+ LocationLabel.
+
+ .. seealso::
+ :meth:`getFullDesignLocation`
+ :meth:`getFullUserLocation`
+
+ .. versionadded:: 5.0
+ """
+ self.designLocation: AnisotropicLocationDict = (
+ designLocation if designLocation is not None else (location or {})
+ )
+ """dict. Axis values for this instance, in design space coordinates.
+
+ MutatorMath + varLib.
+
+ .. seealso:: This may be only part of the full location. See:
+ :meth:`getFullDesignLocation`
+ :meth:`getFullUserLocation`
+
+ .. versionadded:: 5.0
+ """
+ self.userLocation: SimpleLocationDict = userLocation or {}
+ """dict. Axis values for this instance, in user space coordinates.
+
+ MutatorMath + varLib.
+
+ .. seealso:: This may be only part of the full location. See:
+ :meth:`getFullDesignLocation`
+ :meth:`getFullUserLocation`
+
+ .. versionadded:: 5.0
+ """
+ self.familyName = familyName
+ """string. Family name of this instance.
+
+ MutatorMath + varLib.
+ """
+ self.styleName = styleName
+ """string. Style name of this instance.
+
+ MutatorMath + varLib.
+ """
+ self.postScriptFontName = postScriptFontName
+ """string. Postscript fontname for this instance.
+
+ MutatorMath + varLib.
+ """
+ self.styleMapFamilyName = styleMapFamilyName
+ """string. StyleMap familyname for this instance.
+
+ MutatorMath + varLib.
+ """
+ self.styleMapStyleName = styleMapStyleName
+ """string. StyleMap stylename for this instance.
+
+ MutatorMath + varLib.
+ """
+ self.localisedFamilyName = localisedFamilyName or {}
+ """dict. A dictionary of localised family name
+ strings, keyed by language code.
+ """
+ self.localisedStyleName = localisedStyleName or {}
+ """dict. A dictionary of localised stylename
+ strings, keyed by language code.
+ """
+ self.localisedStyleMapFamilyName = localisedStyleMapFamilyName or {}
+ """A dictionary of localised style map
+ familyname strings, keyed by language code.
+ """
+ self.localisedStyleMapStyleName = localisedStyleMapStyleName or {}
+ """A dictionary of localised style map
+ stylename strings, keyed by language code.
+ """
+ self.glyphs = glyphs or {}
+ """dict for special master definitions for glyphs. If glyphs
+ need special masters (to record the results of executed rules for
+ example).
+
+ MutatorMath.
+
+ .. deprecated:: 5.0
+ Use rules or sparse sources instead.
+ """
+ self.kerning = kerning
+ """ bool. Indicates if this instance needs its kerning
+ calculated.
+
+ MutatorMath.
+
+ .. deprecated:: 5.0
+ """
+ self.info = info
+ """bool. Indicated if this instance needs the interpolating
+ font.info calculated.
+
+ .. deprecated:: 5.0
+ """
+
+ self.lib = lib or {}
+ """Custom data associated with this instance."""
+
+ @property
+ def location(self):
+ """dict. Axis values for this instance.
+
+ MutatorMath + varLib.
+
+ .. deprecated:: 5.0
+ Use the more explicit alias for this property :attr:`designLocation`.
+ """
+ return self.designLocation
+
+ @location.setter
+ def location(self, location: Optional[AnisotropicLocationDict]):
+ self.designLocation = location or {}
+
+ def setStyleName(self, styleName, languageCode="en"):
+ """These methods give easier access to the localised names."""
+ self.localisedStyleName[languageCode] = tostr(styleName)
+
+ def getStyleName(self, languageCode="en"):
+ return self.localisedStyleName.get(languageCode)
+
+ def setFamilyName(self, familyName, languageCode="en"):
+ self.localisedFamilyName[languageCode] = tostr(familyName)
+
+ def getFamilyName(self, languageCode="en"):
+ return self.localisedFamilyName.get(languageCode)
+
+ def setStyleMapStyleName(self, styleMapStyleName, languageCode="en"):
+ self.localisedStyleMapStyleName[languageCode] = tostr(styleMapStyleName)
+
+ def getStyleMapStyleName(self, languageCode="en"):
+ return self.localisedStyleMapStyleName.get(languageCode)
+
+ def setStyleMapFamilyName(self, styleMapFamilyName, languageCode="en"):
+ self.localisedStyleMapFamilyName[languageCode] = tostr(styleMapFamilyName)
+
+ def getStyleMapFamilyName(self, languageCode="en"):
+ return self.localisedStyleMapFamilyName.get(languageCode)
+
+ def clearLocation(self, axisName: Optional[str] = None):
+ """Clear all location-related fields. Ensures that
+ :attr:``designLocation`` and :attr:``userLocation`` are dictionaries
+ (possibly empty if clearing everything).
+
+ In order to update the location of this instance wholesale, a user
+ should first clear all the fields, then change the field(s) for which
+ they have data.
+
+ .. code:: python
+
+ instance.clearLocation()
+ instance.designLocation = {'Weight': (34, 36.5), 'Width': 100}
+ instance.userLocation = {'Opsz': 16}
+
+ In order to update a single axis location, the user should only clear
+ that axis, then edit the values:
+
+ .. code:: python
+
+ instance.clearLocation('Weight')
+ instance.designLocation['Weight'] = (34, 36.5)
+
+ Args:
+ axisName: if provided, only clear the location for that axis.
+
+ .. versionadded:: 5.0
+ """
+ self.locationLabel = None
+ if axisName is None:
+ self.designLocation = {}
+ self.userLocation = {}
+ else:
+ if self.designLocation is None:
+ self.designLocation = {}
+ if axisName in self.designLocation:
+ del self.designLocation[axisName]
+ if self.userLocation is None:
+ self.userLocation = {}
+ if axisName in self.userLocation:
+ del self.userLocation[axisName]
+
+ def getLocationLabelDescriptor(
+ self, doc: "DesignSpaceDocument"
+ ) -> Optional[LocationLabelDescriptor]:
+ """Get the :class:`LocationLabelDescriptor` instance that matches
+ this instances's :attr:`locationLabel`.
+
+ Raises if the named label can't be found.
+
+ .. versionadded:: 5.0
+ """
+ if self.locationLabel is None:
+ return None
+ label = doc.getLocationLabel(self.locationLabel)
+ if label is None:
+ raise DesignSpaceDocumentError(
+ "InstanceDescriptor.getLocationLabelDescriptor(): "
+ f"unknown location label `{self.locationLabel}` in instance `{self.name}`."
+ )
+ return label
+
+ def getFullDesignLocation(
+ self, doc: "DesignSpaceDocument"
+ ) -> AnisotropicLocationDict:
+ """Get the complete design location of this instance, by combining data
+ from the various location fields, default axis values and mappings, and
+ top-level location labels.
+
+ The source of truth for this instance's location is determined for each
+ axis independently by taking the first not-None field in this list:
+
+ - ``locationLabel``: the location along this axis is the same as the
+ matching STAT format 4 label. No anisotropy.
+ - ``designLocation[axisName]``: the explicit design location along this
+ axis, possibly anisotropic.
+ - ``userLocation[axisName]``: the explicit user location along this
+ axis. No anisotropy.
+ - ``axis.default``: default axis value. No anisotropy.
+
+ .. versionadded:: 5.0
+ """
+ label = self.getLocationLabelDescriptor(doc)
+ if label is not None:
+ return doc.map_forward(label.userLocation) # type: ignore
+ result: AnisotropicLocationDict = {}
+ for axis in doc.axes:
+ if axis.name in self.designLocation:
+ result[axis.name] = self.designLocation[axis.name]
+ elif axis.name in self.userLocation:
+ result[axis.name] = axis.map_forward(self.userLocation[axis.name])
+ else:
+ result[axis.name] = axis.map_forward(axis.default)
+ return result
+
+ def getFullUserLocation(self, doc: "DesignSpaceDocument") -> SimpleLocationDict:
+ """Get the complete user location for this instance.
+
+ .. seealso:: :meth:`getFullDesignLocation`
+
+ .. versionadded:: 5.0
+ """
+ return doc.map_backward(self.getFullDesignLocation(doc))
+
+
+def tagForAxisName(name):
+ # try to find or make a tag name for this axis name
+ names = {
+ "weight": ("wght", dict(en="Weight")),
+ "width": ("wdth", dict(en="Width")),
+ "optical": ("opsz", dict(en="Optical Size")),
+ "slant": ("slnt", dict(en="Slant")),
+ "italic": ("ital", dict(en="Italic")),
+ }
+ if name.lower() in names:
+ return names[name.lower()]
+ if len(name) < 4:
+ tag = name + "*" * (4 - len(name))
+ else:
+ tag = name[:4]
+ return tag, dict(en=name)
+
+
+class AbstractAxisDescriptor(SimpleDescriptor):
+ flavor = "axis"
+
+ def __init__(
+ self,
+ *,
+ tag=None,
+ name=None,
+ labelNames=None,
+ hidden=False,
+ map=None,
+ axisOrdering=None,
+ axisLabels=None,
+ ):
+ # opentype tag for this axis
+ self.tag = tag
+ """string. Four letter tag for this axis. Some might be
+ registered at the `OpenType
+ specification `__.
+ Privately-defined axis tags must begin with an uppercase letter and
+ use only uppercase letters or digits.
+ """
+ # name of the axis used in locations
+ self.name = name
+ """string. Name of the axis as it is used in the location dicts.
+
+ MutatorMath + varLib.
+ """
+ # names for UI purposes, if this is not a standard axis,
+ self.labelNames = labelNames or {}
+ """dict. When defining a non-registered axis, it will be
+ necessary to define user-facing readable names for the axis. Keyed by
+ xml:lang code. Values are required to be ``unicode`` strings, even if
+ they only contain ASCII characters.
+ """
+ self.hidden = hidden
+ """bool. Whether this axis should be hidden in user interfaces.
+ """
+ self.map = map or []
+ """list of input / output values that can describe a warp of user space
+ to design space coordinates. If no map values are present, it is assumed
+ user space is the same as design space, as in [(minimum, minimum),
+ (maximum, maximum)].
+
+ varLib.
+ """
+ self.axisOrdering = axisOrdering
+ """STAT table field ``axisOrdering``.
+
+ See: `OTSpec STAT Axis Record `_
+
+ .. versionadded:: 5.0
+ """
+ self.axisLabels: List[AxisLabelDescriptor] = axisLabels or []
+ """STAT table entries for Axis Value Tables format 1, 2, 3.
+
+ See: `OTSpec STAT Axis Value Tables `_
+
+ .. versionadded:: 5.0
+ """
+
+
+class AxisDescriptor(AbstractAxisDescriptor):
+ """Simple container for the axis data.
+
+ Add more localisations?
+
+ .. code:: python
+
+ a1 = AxisDescriptor()
+ a1.minimum = 1
+ a1.maximum = 1000
+ a1.default = 400
+ a1.name = "weight"
+ a1.tag = "wght"
+ a1.labelNames['fa-IR'] = "قطر"
+ a1.labelNames['en'] = "Wéíght"
+ a1.map = [(1.0, 10.0), (400.0, 66.0), (1000.0, 990.0)]
+ a1.axisOrdering = 1
+ a1.axisLabels = [
+ AxisLabelDescriptor(name="Regular", userValue=400, elidable=True)
+ ]
+ doc.addAxis(a1)
+ """
+
+ _attrs = [
+ "tag",
+ "name",
+ "maximum",
+ "minimum",
+ "default",
+ "map",
+ "axisOrdering",
+ "axisLabels",
+ ]
+
+ def __init__(
+ self,
+ *,
+ tag=None,
+ name=None,
+ labelNames=None,
+ minimum=None,
+ default=None,
+ maximum=None,
+ hidden=False,
+ map=None,
+ axisOrdering=None,
+ axisLabels=None,
+ ):
+ super().__init__(
+ tag=tag,
+ name=name,
+ labelNames=labelNames,
+ hidden=hidden,
+ map=map,
+ axisOrdering=axisOrdering,
+ axisLabels=axisLabels,
+ )
+ self.minimum = minimum
+ """number. The minimum value for this axis in user space.
+
+ MutatorMath + varLib.
+ """
+ self.maximum = maximum
+ """number. The maximum value for this axis in user space.
+
+ MutatorMath + varLib.
+ """
+ self.default = default
+ """number. The default value for this axis, i.e. when a new location is
+ created, this is the value this axis will get in user space.
+
+ MutatorMath + varLib.
+ """
+
+ def serialize(self):
+ # output to a dict, used in testing
+ return dict(
+ tag=self.tag,
+ name=self.name,
+ labelNames=self.labelNames,
+ maximum=self.maximum,
+ minimum=self.minimum,
+ default=self.default,
+ hidden=self.hidden,
+ map=self.map,
+ axisOrdering=self.axisOrdering,
+ axisLabels=self.axisLabels,
+ )
+
+ def map_forward(self, v):
+ """Maps value from axis mapping's input (user) to output (design)."""
+ from fontTools.varLib.models import piecewiseLinearMap
+
+ if not self.map:
+ return v
+ return piecewiseLinearMap(v, {k: v for k, v in self.map})
+
+ def map_backward(self, v):
+ """Maps value from axis mapping's output (design) to input (user)."""
+ from fontTools.varLib.models import piecewiseLinearMap
+
+ if isinstance(v, tuple):
+ v = v[0]
+ if not self.map:
+ return v
+ return piecewiseLinearMap(v, {v: k for k, v in self.map})
+
+
+class DiscreteAxisDescriptor(AbstractAxisDescriptor):
+ """Container for discrete axis data.
+
+ Use this for axes that do not interpolate. The main difference from a
+ continuous axis is that a continuous axis has a ``minimum`` and ``maximum``,
+ while a discrete axis has a list of ``values``.
+
+ Example: an Italic axis with 2 stops, Roman and Italic, that are not
+ compatible. The axis still allows to bind together the full font family,
+ which is useful for the STAT table, however it can't become a variation
+ axis in a VF.
+
+ .. code:: python
+
+ a2 = DiscreteAxisDescriptor()
+ a2.values = [0, 1]
+ a2.default = 0
+ a2.name = "Italic"
+ a2.tag = "ITAL"
+ a2.labelNames['fr'] = "Italique"
+ a2.map = [(0, 0), (1, -11)]
+ a2.axisOrdering = 2
+ a2.axisLabels = [
+ AxisLabelDescriptor(name="Roman", userValue=0, elidable=True)
+ ]
+ doc.addAxis(a2)
+
+ .. versionadded:: 5.0
+ """
+
+ flavor = "axis"
+ _attrs = ("tag", "name", "values", "default", "map", "axisOrdering", "axisLabels")
+
+ def __init__(
+ self,
+ *,
+ tag=None,
+ name=None,
+ labelNames=None,
+ values=None,
+ default=None,
+ hidden=False,
+ map=None,
+ axisOrdering=None,
+ axisLabels=None,
+ ):
+ super().__init__(
+ tag=tag,
+ name=name,
+ labelNames=labelNames,
+ hidden=hidden,
+ map=map,
+ axisOrdering=axisOrdering,
+ axisLabels=axisLabels,
+ )
+ self.default: float = default
+ """The default value for this axis, i.e. when a new location is
+ created, this is the value this axis will get in user space.
+
+ However, this default value is less important than in continuous axes:
+
+ - it doesn't define the "neutral" version of outlines from which
+ deltas would apply, as this axis does not interpolate.
+ - it doesn't provide the reference glyph set for the designspace, as
+ fonts at each value can have different glyph sets.
+ """
+ self.values: List[float] = values or []
+ """List of possible values for this axis. Contrary to continuous axes,
+ only the values in this list can be taken by the axis, nothing in-between.
+ """
+
+ def map_forward(self, value):
+ """Maps value from axis mapping's input to output.
+
+ Returns value unchanged if no mapping entry is found.
+
+ Note: for discrete axes, each value must have its mapping entry, if
+ you intend that value to be mapped.
+ """
+ return next((v for k, v in self.map if k == value), value)
+
+ def map_backward(self, value):
+ """Maps value from axis mapping's output to input.
+
+ Returns value unchanged if no mapping entry is found.
+
+ Note: for discrete axes, each value must have its mapping entry, if
+ you intend that value to be mapped.
+ """
+ if isinstance(value, tuple):
+ value = value[0]
+ return next((k for k, v in self.map if v == value), value)
+
+
+class AxisLabelDescriptor(SimpleDescriptor):
+ """Container for axis label data.
+
+ Analogue of OpenType's STAT data for a single axis (formats 1, 2 and 3).
+ All values are user values.
+ See: `OTSpec STAT Axis value table, format 1, 2, 3 `_
+
+ The STAT format of the Axis value depends on which field are filled-in,
+ see :meth:`getFormat`
+
+ .. versionadded:: 5.0
+ """
+
+ flavor = "label"
+ _attrs = (
+ "userMinimum",
+ "userValue",
+ "userMaximum",
+ "name",
+ "elidable",
+ "olderSibling",
+ "linkedUserValue",
+ "labelNames",
+ )
+
+ def __init__(
+ self,
+ *,
+ name,
+ userValue,
+ userMinimum=None,
+ userMaximum=None,
+ elidable=False,
+ olderSibling=False,
+ linkedUserValue=None,
+ labelNames=None,
+ ):
+ self.userMinimum: Optional[float] = userMinimum
+ """STAT field ``rangeMinValue`` (format 2)."""
+ self.userValue: float = userValue
+ """STAT field ``value`` (format 1, 3) or ``nominalValue`` (format 2)."""
+ self.userMaximum: Optional[float] = userMaximum
+ """STAT field ``rangeMaxValue`` (format 2)."""
+ self.name: str = name
+ """Label for this axis location, STAT field ``valueNameID``."""
+ self.elidable: bool = elidable
+ """STAT flag ``ELIDABLE_AXIS_VALUE_NAME``.
+
+ See: `OTSpec STAT Flags `_
+ """
+ self.olderSibling: bool = olderSibling
+ """STAT flag ``OLDER_SIBLING_FONT_ATTRIBUTE``.
+
+ See: `OTSpec STAT Flags `_
+ """
+ self.linkedUserValue: Optional[float] = linkedUserValue
+ """STAT field ``linkedValue`` (format 3)."""
+ self.labelNames: MutableMapping[str, str] = labelNames or {}
+ """User-facing translations of this location's label. Keyed by
+ ``xml:lang`` code.
+ """
+
+ def getFormat(self) -> int:
+ """Determine which format of STAT Axis value to use to encode this label.
+
+ =========== ========= =========== =========== ===============
+ STAT Format userValue userMinimum userMaximum linkedUserValue
+ =========== ========= =========== =========== ===============
+ 1 ✅ ❌ ❌ ❌
+ 2 ✅ ✅ ✅ ❌
+ 3 ✅ ❌ ❌ ✅
+ =========== ========= =========== =========== ===============
+ """
+ if self.linkedUserValue is not None:
+ return 3
+ if self.userMinimum is not None or self.userMaximum is not None:
+ return 2
+ return 1
+
+ @property
+ def defaultName(self) -> str:
+ """Return the English name from :attr:`labelNames` or the :attr:`name`."""
+ return self.labelNames.get("en") or self.name
+
+
+class LocationLabelDescriptor(SimpleDescriptor):
+ """Container for location label data.
+
+ Analogue of OpenType's STAT data for a free-floating location (format 4).
+ All values are user values.
+
+ See: `OTSpec STAT Axis value table, format 4 `_
+
+ .. versionadded:: 5.0
+ """
+
+ flavor = "label"
+ _attrs = ("name", "elidable", "olderSibling", "userLocation", "labelNames")
+
+ def __init__(
+ self,
+ *,
+ name,
+ userLocation,
+ elidable=False,
+ olderSibling=False,
+ labelNames=None,
+ ):
+ self.name: str = name
+ """Label for this named location, STAT field ``valueNameID``."""
+ self.userLocation: SimpleLocationDict = userLocation or {}
+ """Location in user coordinates along each axis.
+
+ If an axis is not mentioned, it is assumed to be at its default location.
+
+ .. seealso:: This may be only part of the full location. See:
+ :meth:`getFullUserLocation`
+ """
+ self.elidable: bool = elidable
+ """STAT flag ``ELIDABLE_AXIS_VALUE_NAME``.
+
+ See: `OTSpec STAT Flags `_
+ """
+ self.olderSibling: bool = olderSibling
+ """STAT flag ``OLDER_SIBLING_FONT_ATTRIBUTE``.
+
+ See: `OTSpec STAT Flags `_
+ """
+ self.labelNames: Dict[str, str] = labelNames or {}
+ """User-facing translations of this location's label. Keyed by
+ xml:lang code.
+ """
+
+ @property
+ def defaultName(self) -> str:
+ """Return the English name from :attr:`labelNames` or the :attr:`name`."""
+ return self.labelNames.get("en") or self.name
+
+ def getFullUserLocation(self, doc: "DesignSpaceDocument") -> SimpleLocationDict:
+ """Get the complete user location of this label, by combining data
+ from the explicit user location and default axis values.
+
+ .. versionadded:: 5.0
+ """
+ return {
+ axis.name: self.userLocation.get(axis.name, axis.default)
+ for axis in doc.axes
+ }
+
+
+class VariableFontDescriptor(SimpleDescriptor):
+ """Container for variable fonts, sub-spaces of the Designspace.
+
+ Use-cases:
+
+ - From a single DesignSpace with discrete axes, define 1 variable font
+ per value on the discrete axes. Before version 5, you would have needed
+ 1 DesignSpace per such variable font, and a lot of data duplication.
+ - From a big variable font with many axes, define subsets of that variable
+ font that only include some axes and freeze other axes at a given location.
+
+ .. versionadded:: 5.0
+ """
+
+ flavor = "variable-font"
+ _attrs = ("filename", "axisSubsets", "lib")
+
+ filename = posixpath_property("_filename")
+
+ def __init__(self, *, name, filename=None, axisSubsets=None, lib=None):
+ self.name: str = name
+ """string, required. Name of this variable to identify it during the
+ build process and from other parts of the document, and also as a
+ filename in case the filename property is empty.
+
+ VarLib.
+ """
+ self.filename: str = filename
+ """string, optional. Relative path to the variable font file, **as it is
+ in the document**. The file may or may not exist.
+
+ If not specified, the :attr:`name` will be used as a basename for the file.
+ """
+ self.axisSubsets: List[
+ Union[RangeAxisSubsetDescriptor, ValueAxisSubsetDescriptor]
+ ] = (axisSubsets or [])
+ """Axis subsets to include in this variable font.
+
+ If an axis is not mentioned, assume that we only want the default
+ location of that axis (same as a :class:`ValueAxisSubsetDescriptor`).
+ """
+ self.lib: MutableMapping[str, Any] = lib or {}
+ """Custom data associated with this variable font."""
+
+
+class RangeAxisSubsetDescriptor(SimpleDescriptor):
+ """Subset of a continuous axis to include in a variable font.
+
+ .. versionadded:: 5.0
+ """
+
+ flavor = "axis-subset"
+ _attrs = ("name", "userMinimum", "userDefault", "userMaximum")
+
+ def __init__(
+ self, *, name, userMinimum=-math.inf, userDefault=None, userMaximum=math.inf
+ ):
+ self.name: str = name
+ """Name of the :class:`AxisDescriptor` to subset."""
+ self.userMinimum: float = userMinimum
+ """New minimum value of the axis in the target variable font.
+ If not specified, assume the same minimum value as the full axis.
+ (default = ``-math.inf``)
+ """
+ self.userDefault: Optional[float] = userDefault
+ """New default value of the axis in the target variable font.
+ If not specified, assume the same default value as the full axis.
+ (default = ``None``)
+ """
+ self.userMaximum: float = userMaximum
+ """New maximum value of the axis in the target variable font.
+ If not specified, assume the same maximum value as the full axis.
+ (default = ``math.inf``)
+ """
+
+
+class ValueAxisSubsetDescriptor(SimpleDescriptor):
+ """Single value of a discrete or continuous axis to use in a variable font.
+
+ .. versionadded:: 5.0
+ """
+
+ flavor = "axis-subset"
+ _attrs = ("name", "userValue")
+
+ def __init__(self, *, name, userValue):
+ self.name: str = name
+ """Name of the :class:`AxisDescriptor` or :class:`DiscreteAxisDescriptor`
+ to "snapshot" or "freeze".
+ """
+ self.userValue: float = userValue
+ """Value in user coordinates at which to freeze the given axis."""
+
+
+class BaseDocWriter(object):
+ _whiteSpace = " "
+ axisDescriptorClass = AxisDescriptor
+ discreteAxisDescriptorClass = DiscreteAxisDescriptor
+ axisLabelDescriptorClass = AxisLabelDescriptor
+ axisMappingDescriptorClass = AxisMappingDescriptor
+ locationLabelDescriptorClass = LocationLabelDescriptor
+ ruleDescriptorClass = RuleDescriptor
+ sourceDescriptorClass = SourceDescriptor
+ variableFontDescriptorClass = VariableFontDescriptor
+ valueAxisSubsetDescriptorClass = ValueAxisSubsetDescriptor
+ rangeAxisSubsetDescriptorClass = RangeAxisSubsetDescriptor
+ instanceDescriptorClass = InstanceDescriptor
+
+ @classmethod
+ def getAxisDecriptor(cls):
+ return cls.axisDescriptorClass()
+
+ @classmethod
+ def getAxisMappingDescriptor(cls):
+ return cls.axisMappingDescriptorClass()
+
+ @classmethod
+ def getSourceDescriptor(cls):
+ return cls.sourceDescriptorClass()
+
+ @classmethod
+ def getInstanceDescriptor(cls):
+ return cls.instanceDescriptorClass()
+
+ @classmethod
+ def getRuleDescriptor(cls):
+ return cls.ruleDescriptorClass()
+
+ def __init__(self, documentPath, documentObject: DesignSpaceDocument):
+ self.path = documentPath
+ self.documentObject = documentObject
+ self.effectiveFormatTuple = self._getEffectiveFormatTuple()
+ self.root = ET.Element("designspace")
+
+ def write(self, pretty=True, encoding="UTF-8", xml_declaration=True):
+ self.root.attrib["format"] = ".".join(str(i) for i in self.effectiveFormatTuple)
+
+ if (
+ self.documentObject.axes
+ or self.documentObject.axisMappings
+ or self.documentObject.elidedFallbackName is not None
+ ):
+ axesElement = ET.Element("axes")
+ if self.documentObject.elidedFallbackName is not None:
+ axesElement.attrib["elidedfallbackname"] = (
+ self.documentObject.elidedFallbackName
+ )
+ self.root.append(axesElement)
+ for axisObject in self.documentObject.axes:
+ self._addAxis(axisObject)
+
+ if self.documentObject.axisMappings:
+ mappingsElement = None
+ lastGroup = object()
+ for mappingObject in self.documentObject.axisMappings:
+ if getattr(mappingObject, "groupDescription", None) != lastGroup:
+ if mappingsElement is not None:
+ self.root.findall(".axes")[0].append(mappingsElement)
+ lastGroup = getattr(mappingObject, "groupDescription", None)
+ mappingsElement = ET.Element("mappings")
+ if lastGroup is not None:
+ mappingsElement.attrib["description"] = lastGroup
+ self._addAxisMapping(mappingsElement, mappingObject)
+ if mappingsElement is not None:
+ self.root.findall(".axes")[0].append(mappingsElement)
+
+ if self.documentObject.locationLabels:
+ labelsElement = ET.Element("labels")
+ for labelObject in self.documentObject.locationLabels:
+ self._addLocationLabel(labelsElement, labelObject)
+ self.root.append(labelsElement)
+
+ if self.documentObject.rules:
+ if getattr(self.documentObject, "rulesProcessingLast", False):
+ attributes = {"processing": "last"}
+ else:
+ attributes = {}
+ self.root.append(ET.Element("rules", attributes))
+ for ruleObject in self.documentObject.rules:
+ self._addRule(ruleObject)
+
+ if self.documentObject.sources:
+ self.root.append(ET.Element("sources"))
+ for sourceObject in self.documentObject.sources:
+ self._addSource(sourceObject)
+
+ if self.documentObject.variableFonts:
+ variableFontsElement = ET.Element("variable-fonts")
+ for variableFont in self.documentObject.variableFonts:
+ self._addVariableFont(variableFontsElement, variableFont)
+ self.root.append(variableFontsElement)
+
+ if self.documentObject.instances:
+ self.root.append(ET.Element("instances"))
+ for instanceObject in self.documentObject.instances:
+ self._addInstance(instanceObject)
+
+ if self.documentObject.lib:
+ self._addLib(self.root, self.documentObject.lib, 2)
+
+ tree = ET.ElementTree(self.root)
+ tree.write(
+ self.path,
+ encoding=encoding,
+ method="xml",
+ xml_declaration=xml_declaration,
+ pretty_print=pretty,
+ )
+
+ def _getEffectiveFormatTuple(self):
+ """Try to use the version specified in the document, or a sufficiently
+ recent version to be able to encode what the document contains.
+ """
+ minVersion = self.documentObject.formatTuple
+ if (
+ any(
+ hasattr(axis, "values")
+ or axis.axisOrdering is not None
+ or axis.axisLabels
+ for axis in self.documentObject.axes
+ )
+ or self.documentObject.locationLabels
+ or any(source.localisedFamilyName for source in self.documentObject.sources)
+ or self.documentObject.variableFonts
+ or any(
+ instance.locationLabel or instance.userLocation
+ for instance in self.documentObject.instances
+ )
+ ):
+ if minVersion < (5, 0):
+ minVersion = (5, 0)
+ if self.documentObject.axisMappings:
+ if minVersion < (5, 1):
+ minVersion = (5, 1)
+ return minVersion
+
+ def _makeLocationElement(self, locationObject, name=None):
+ """Convert Location dict to a locationElement."""
+ locElement = ET.Element("location")
+ if name is not None:
+ locElement.attrib["name"] = name
+ validatedLocation = self.documentObject.newDefaultLocation()
+ for axisName, axisValue in locationObject.items():
+ if axisName in validatedLocation:
+ # only accept values we know
+ validatedLocation[axisName] = axisValue
+ for dimensionName, dimensionValue in validatedLocation.items():
+ dimElement = ET.Element("dimension")
+ dimElement.attrib["name"] = dimensionName
+ if type(dimensionValue) == tuple:
+ dimElement.attrib["xvalue"] = self.intOrFloat(dimensionValue[0])
+ dimElement.attrib["yvalue"] = self.intOrFloat(dimensionValue[1])
+ else:
+ dimElement.attrib["xvalue"] = self.intOrFloat(dimensionValue)
+ locElement.append(dimElement)
+ return locElement, validatedLocation
+
+ def intOrFloat(self, num):
+ if int(num) == num:
+ return "%d" % num
+ return ("%f" % num).rstrip("0").rstrip(".")
+
+ def _addRule(self, ruleObject):
+ # if none of the conditions have minimum or maximum values, do not add the rule.
+ ruleElement = ET.Element("rule")
+ if ruleObject.name is not None:
+ ruleElement.attrib["name"] = ruleObject.name
+ for conditions in ruleObject.conditionSets:
+ conditionsetElement = ET.Element("conditionset")
+ for cond in conditions:
+ if cond.get("minimum") is None and cond.get("maximum") is None:
+ # neither is defined, don't add this condition
+ continue
+ conditionElement = ET.Element("condition")
+ conditionElement.attrib["name"] = cond.get("name")
+ if cond.get("minimum") is not None:
+ conditionElement.attrib["minimum"] = self.intOrFloat(
+ cond.get("minimum")
+ )
+ if cond.get("maximum") is not None:
+ conditionElement.attrib["maximum"] = self.intOrFloat(
+ cond.get("maximum")
+ )
+ conditionsetElement.append(conditionElement)
+ if len(conditionsetElement):
+ ruleElement.append(conditionsetElement)
+ for sub in ruleObject.subs:
+ subElement = ET.Element("sub")
+ subElement.attrib["name"] = sub[0]
+ subElement.attrib["with"] = sub[1]
+ ruleElement.append(subElement)
+ if len(ruleElement):
+ self.root.findall(".rules")[0].append(ruleElement)
+
+ def _addAxis(self, axisObject):
+ axisElement = ET.Element("axis")
+ axisElement.attrib["tag"] = axisObject.tag
+ axisElement.attrib["name"] = axisObject.name
+ self._addLabelNames(axisElement, axisObject.labelNames)
+ if axisObject.map:
+ for inputValue, outputValue in axisObject.map:
+ mapElement = ET.Element("map")
+ mapElement.attrib["input"] = self.intOrFloat(inputValue)
+ mapElement.attrib["output"] = self.intOrFloat(outputValue)
+ axisElement.append(mapElement)
+ if axisObject.axisOrdering is not None or axisObject.axisLabels:
+ labelsElement = ET.Element("labels")
+ if axisObject.axisOrdering is not None:
+ labelsElement.attrib["ordering"] = str(axisObject.axisOrdering)
+ for label in axisObject.axisLabels:
+ self._addAxisLabel(labelsElement, label)
+ axisElement.append(labelsElement)
+ if hasattr(axisObject, "minimum"):
+ axisElement.attrib["minimum"] = self.intOrFloat(axisObject.minimum)
+ axisElement.attrib["maximum"] = self.intOrFloat(axisObject.maximum)
+ elif hasattr(axisObject, "values"):
+ axisElement.attrib["values"] = " ".join(
+ self.intOrFloat(v) for v in axisObject.values
+ )
+ axisElement.attrib["default"] = self.intOrFloat(axisObject.default)
+ if axisObject.hidden:
+ axisElement.attrib["hidden"] = "1"
+ self.root.findall(".axes")[0].append(axisElement)
+
+ def _addAxisMapping(self, mappingsElement, mappingObject):
+ mappingElement = ET.Element("mapping")
+ if getattr(mappingObject, "description", None) is not None:
+ mappingElement.attrib["description"] = mappingObject.description
+ for what in ("inputLocation", "outputLocation"):
+ whatObject = getattr(mappingObject, what, None)
+ if whatObject is None:
+ continue
+ whatElement = ET.Element(what[:-8])
+ mappingElement.append(whatElement)
+
+ for name, value in whatObject.items():
+ dimensionElement = ET.Element("dimension")
+ dimensionElement.attrib["name"] = name
+ dimensionElement.attrib["xvalue"] = self.intOrFloat(value)
+ whatElement.append(dimensionElement)
+
+ mappingsElement.append(mappingElement)
+
+ def _addAxisLabel(
+ self, axisElement: ET.Element, label: AxisLabelDescriptor
+ ) -> None:
+ labelElement = ET.Element("label")
+ labelElement.attrib["uservalue"] = self.intOrFloat(label.userValue)
+ if label.userMinimum is not None:
+ labelElement.attrib["userminimum"] = self.intOrFloat(label.userMinimum)
+ if label.userMaximum is not None:
+ labelElement.attrib["usermaximum"] = self.intOrFloat(label.userMaximum)
+ labelElement.attrib["name"] = label.name
+ if label.elidable:
+ labelElement.attrib["elidable"] = "true"
+ if label.olderSibling:
+ labelElement.attrib["oldersibling"] = "true"
+ if label.linkedUserValue is not None:
+ labelElement.attrib["linkeduservalue"] = self.intOrFloat(
+ label.linkedUserValue
+ )
+ self._addLabelNames(labelElement, label.labelNames)
+ axisElement.append(labelElement)
+
+ def _addLabelNames(self, parentElement, labelNames):
+ for languageCode, labelName in sorted(labelNames.items()):
+ languageElement = ET.Element("labelname")
+ languageElement.attrib[XML_LANG] = languageCode
+ languageElement.text = labelName
+ parentElement.append(languageElement)
+
+ def _addLocationLabel(
+ self, parentElement: ET.Element, label: LocationLabelDescriptor
+ ) -> None:
+ labelElement = ET.Element("label")
+ labelElement.attrib["name"] = label.name
+ if label.elidable:
+ labelElement.attrib["elidable"] = "true"
+ if label.olderSibling:
+ labelElement.attrib["oldersibling"] = "true"
+ self._addLabelNames(labelElement, label.labelNames)
+ self._addLocationElement(labelElement, userLocation=label.userLocation)
+ parentElement.append(labelElement)
+
+ def _addLocationElement(
+ self,
+ parentElement,
+ *,
+ designLocation: AnisotropicLocationDict = None,
+ userLocation: SimpleLocationDict = None,
+ ):
+ locElement = ET.Element("location")
+ for axis in self.documentObject.axes:
+ if designLocation is not None and axis.name in designLocation:
+ dimElement = ET.Element("dimension")
+ dimElement.attrib["name"] = axis.name
+ value = designLocation[axis.name]
+ if isinstance(value, tuple):
+ dimElement.attrib["xvalue"] = self.intOrFloat(value[0])
+ dimElement.attrib["yvalue"] = self.intOrFloat(value[1])
+ else:
+ dimElement.attrib["xvalue"] = self.intOrFloat(value)
+ locElement.append(dimElement)
+ elif userLocation is not None and axis.name in userLocation:
+ dimElement = ET.Element("dimension")
+ dimElement.attrib["name"] = axis.name
+ value = userLocation[axis.name]
+ dimElement.attrib["uservalue"] = self.intOrFloat(value)
+ locElement.append(dimElement)
+ if len(locElement) > 0:
+ parentElement.append(locElement)
+
+ def _addInstance(self, instanceObject):
+ instanceElement = ET.Element("instance")
+ if instanceObject.name is not None:
+ instanceElement.attrib["name"] = instanceObject.name
+ if instanceObject.locationLabel is not None:
+ instanceElement.attrib["location"] = instanceObject.locationLabel
+ if instanceObject.familyName is not None:
+ instanceElement.attrib["familyname"] = instanceObject.familyName
+ if instanceObject.styleName is not None:
+ instanceElement.attrib["stylename"] = instanceObject.styleName
+ # add localisations
+ if instanceObject.localisedStyleName:
+ languageCodes = list(instanceObject.localisedStyleName.keys())
+ languageCodes.sort()
+ for code in languageCodes:
+ if code == "en":
+ continue # already stored in the element attribute
+ localisedStyleNameElement = ET.Element("stylename")
+ localisedStyleNameElement.attrib[XML_LANG] = code
+ localisedStyleNameElement.text = instanceObject.getStyleName(code)
+ instanceElement.append(localisedStyleNameElement)
+ if instanceObject.localisedFamilyName:
+ languageCodes = list(instanceObject.localisedFamilyName.keys())
+ languageCodes.sort()
+ for code in languageCodes:
+ if code == "en":
+ continue # already stored in the element attribute
+ localisedFamilyNameElement = ET.Element("familyname")
+ localisedFamilyNameElement.attrib[XML_LANG] = code
+ localisedFamilyNameElement.text = instanceObject.getFamilyName(code)
+ instanceElement.append(localisedFamilyNameElement)
+ if instanceObject.localisedStyleMapStyleName:
+ languageCodes = list(instanceObject.localisedStyleMapStyleName.keys())
+ languageCodes.sort()
+ for code in languageCodes:
+ if code == "en":
+ continue
+ localisedStyleMapStyleNameElement = ET.Element("stylemapstylename")
+ localisedStyleMapStyleNameElement.attrib[XML_LANG] = code
+ localisedStyleMapStyleNameElement.text = (
+ instanceObject.getStyleMapStyleName(code)
+ )
+ instanceElement.append(localisedStyleMapStyleNameElement)
+ if instanceObject.localisedStyleMapFamilyName:
+ languageCodes = list(instanceObject.localisedStyleMapFamilyName.keys())
+ languageCodes.sort()
+ for code in languageCodes:
+ if code == "en":
+ continue
+ localisedStyleMapFamilyNameElement = ET.Element("stylemapfamilyname")
+ localisedStyleMapFamilyNameElement.attrib[XML_LANG] = code
+ localisedStyleMapFamilyNameElement.text = (
+ instanceObject.getStyleMapFamilyName(code)
+ )
+ instanceElement.append(localisedStyleMapFamilyNameElement)
+
+ if self.effectiveFormatTuple >= (5, 0):
+ if instanceObject.locationLabel is None:
+ self._addLocationElement(
+ instanceElement,
+ designLocation=instanceObject.designLocation,
+ userLocation=instanceObject.userLocation,
+ )
+ else:
+ # Pre-version 5.0 code was validating and filling in the location
+ # dict while writing it out, as preserved below.
+ if instanceObject.location is not None:
+ locationElement, instanceObject.location = self._makeLocationElement(
+ instanceObject.location
+ )
+ instanceElement.append(locationElement)
+ if instanceObject.filename is not None:
+ instanceElement.attrib["filename"] = instanceObject.filename
+ if instanceObject.postScriptFontName is not None:
+ instanceElement.attrib["postscriptfontname"] = (
+ instanceObject.postScriptFontName
+ )
+ if instanceObject.styleMapFamilyName is not None:
+ instanceElement.attrib["stylemapfamilyname"] = (
+ instanceObject.styleMapFamilyName
+ )
+ if instanceObject.styleMapStyleName is not None:
+ instanceElement.attrib["stylemapstylename"] = (
+ instanceObject.styleMapStyleName
+ )
+ if self.effectiveFormatTuple < (5, 0):
+ # Deprecated members as of version 5.0
+ if instanceObject.glyphs:
+ if instanceElement.findall(".glyphs") == []:
+ glyphsElement = ET.Element("glyphs")
+ instanceElement.append(glyphsElement)
+ glyphsElement = instanceElement.findall(".glyphs")[0]
+ for glyphName, data in sorted(instanceObject.glyphs.items()):
+ glyphElement = self._writeGlyphElement(
+ instanceElement, instanceObject, glyphName, data
+ )
+ glyphsElement.append(glyphElement)
+ if instanceObject.kerning:
+ kerningElement = ET.Element("kerning")
+ instanceElement.append(kerningElement)
+ if instanceObject.info:
+ infoElement = ET.Element("info")
+ instanceElement.append(infoElement)
+ self._addLib(instanceElement, instanceObject.lib, 4)
+ self.root.findall(".instances")[0].append(instanceElement)
+
+ def _addSource(self, sourceObject):
+ sourceElement = ET.Element("source")
+ if sourceObject.filename is not None:
+ sourceElement.attrib["filename"] = sourceObject.filename
+ if sourceObject.name is not None:
+ if sourceObject.name.find("temp_master") != 0:
+ # do not save temporary source names
+ sourceElement.attrib["name"] = sourceObject.name
+ if sourceObject.familyName is not None:
+ sourceElement.attrib["familyname"] = sourceObject.familyName
+ if sourceObject.styleName is not None:
+ sourceElement.attrib["stylename"] = sourceObject.styleName
+ if sourceObject.layerName is not None:
+ sourceElement.attrib["layer"] = sourceObject.layerName
+ if sourceObject.localisedFamilyName:
+ languageCodes = list(sourceObject.localisedFamilyName.keys())
+ languageCodes.sort()
+ for code in languageCodes:
+ if code == "en":
+ continue # already stored in the element attribute
+ localisedFamilyNameElement = ET.Element("familyname")
+ localisedFamilyNameElement.attrib[XML_LANG] = code
+ localisedFamilyNameElement.text = sourceObject.getFamilyName(code)
+ sourceElement.append(localisedFamilyNameElement)
+ if sourceObject.copyLib:
+ libElement = ET.Element("lib")
+ libElement.attrib["copy"] = "1"
+ sourceElement.append(libElement)
+ if sourceObject.copyGroups:
+ groupsElement = ET.Element("groups")
+ groupsElement.attrib["copy"] = "1"
+ sourceElement.append(groupsElement)
+ if sourceObject.copyFeatures:
+ featuresElement = ET.Element("features")
+ featuresElement.attrib["copy"] = "1"
+ sourceElement.append(featuresElement)
+ if sourceObject.copyInfo or sourceObject.muteInfo:
+ infoElement = ET.Element("info")
+ if sourceObject.copyInfo:
+ infoElement.attrib["copy"] = "1"
+ if sourceObject.muteInfo:
+ infoElement.attrib["mute"] = "1"
+ sourceElement.append(infoElement)
+ if sourceObject.muteKerning:
+ kerningElement = ET.Element("kerning")
+ kerningElement.attrib["mute"] = "1"
+ sourceElement.append(kerningElement)
+ if sourceObject.mutedGlyphNames:
+ for name in sourceObject.mutedGlyphNames:
+ glyphElement = ET.Element("glyph")
+ glyphElement.attrib["name"] = name
+ glyphElement.attrib["mute"] = "1"
+ sourceElement.append(glyphElement)
+ if self.effectiveFormatTuple >= (5, 0):
+ self._addLocationElement(
+ sourceElement, designLocation=sourceObject.location
+ )
+ else:
+ # Pre-version 5.0 code was validating and filling in the location
+ # dict while writing it out, as preserved below.
+ locationElement, sourceObject.location = self._makeLocationElement(
+ sourceObject.location
+ )
+ sourceElement.append(locationElement)
+ self.root.findall(".sources")[0].append(sourceElement)
+
+ def _addVariableFont(
+ self, parentElement: ET.Element, vf: VariableFontDescriptor
+ ) -> None:
+ vfElement = ET.Element("variable-font")
+ vfElement.attrib["name"] = vf.name
+ if vf.filename is not None:
+ vfElement.attrib["filename"] = vf.filename
+ if vf.axisSubsets:
+ subsetsElement = ET.Element("axis-subsets")
+ for subset in vf.axisSubsets:
+ subsetElement = ET.Element("axis-subset")
+ subsetElement.attrib["name"] = subset.name
+ # Mypy doesn't support narrowing union types via hasattr()
+ # https://mypy.readthedocs.io/en/stable/type_narrowing.html
+ # TODO(Python 3.10): use TypeGuard
+ if hasattr(subset, "userMinimum"):
+ subset = cast(RangeAxisSubsetDescriptor, subset)
+ if subset.userMinimum != -math.inf:
+ subsetElement.attrib["userminimum"] = self.intOrFloat(
+ subset.userMinimum
+ )
+ if subset.userMaximum != math.inf:
+ subsetElement.attrib["usermaximum"] = self.intOrFloat(
+ subset.userMaximum
+ )
+ if subset.userDefault is not None:
+ subsetElement.attrib["userdefault"] = self.intOrFloat(
+ subset.userDefault
+ )
+ elif hasattr(subset, "userValue"):
+ subset = cast(ValueAxisSubsetDescriptor, subset)
+ subsetElement.attrib["uservalue"] = self.intOrFloat(
+ subset.userValue
+ )
+ subsetsElement.append(subsetElement)
+ vfElement.append(subsetsElement)
+ self._addLib(vfElement, vf.lib, 4)
+ parentElement.append(vfElement)
+
+ def _addLib(self, parentElement: ET.Element, data: Any, indent_level: int) -> None:
+ if not data:
+ return
+ libElement = ET.Element("lib")
+ libElement.append(plistlib.totree(data, indent_level=indent_level))
+ parentElement.append(libElement)
+
+ def _writeGlyphElement(self, instanceElement, instanceObject, glyphName, data):
+ glyphElement = ET.Element("glyph")
+ if data.get("mute"):
+ glyphElement.attrib["mute"] = "1"
+ if data.get("unicodes") is not None:
+ glyphElement.attrib["unicode"] = " ".join(
+ [hex(u) for u in data.get("unicodes")]
+ )
+ if data.get("instanceLocation") is not None:
+ locationElement, data["instanceLocation"] = self._makeLocationElement(
+ data.get("instanceLocation")
+ )
+ glyphElement.append(locationElement)
+ if glyphName is not None:
+ glyphElement.attrib["name"] = glyphName
+ if data.get("note") is not None:
+ noteElement = ET.Element("note")
+ noteElement.text = data.get("note")
+ glyphElement.append(noteElement)
+ if data.get("masters") is not None:
+ mastersElement = ET.Element("masters")
+ for m in data.get("masters"):
+ masterElement = ET.Element("master")
+ if m.get("glyphName") is not None:
+ masterElement.attrib["glyphname"] = m.get("glyphName")
+ if m.get("font") is not None:
+ masterElement.attrib["source"] = m.get("font")
+ if m.get("location") is not None:
+ locationElement, m["location"] = self._makeLocationElement(
+ m.get("location")
+ )
+ masterElement.append(locationElement)
+ mastersElement.append(masterElement)
+ glyphElement.append(mastersElement)
+ return glyphElement
+
+
+class BaseDocReader(LogMixin):
+ axisDescriptorClass = AxisDescriptor
+ discreteAxisDescriptorClass = DiscreteAxisDescriptor
+ axisLabelDescriptorClass = AxisLabelDescriptor
+ axisMappingDescriptorClass = AxisMappingDescriptor
+ locationLabelDescriptorClass = LocationLabelDescriptor
+ ruleDescriptorClass = RuleDescriptor
+ sourceDescriptorClass = SourceDescriptor
+ variableFontsDescriptorClass = VariableFontDescriptor
+ valueAxisSubsetDescriptorClass = ValueAxisSubsetDescriptor
+ rangeAxisSubsetDescriptorClass = RangeAxisSubsetDescriptor
+ instanceDescriptorClass = InstanceDescriptor
+
+ def __init__(self, documentPath, documentObject):
+ self.path = documentPath
+ self.documentObject = documentObject
+ tree = ET.parse(self.path)
+ self.root = tree.getroot()
+ self.documentObject.formatVersion = self.root.attrib.get("format", "3.0")
+ self._axes = []
+ self.rules = []
+ self.sources = []
+ self.instances = []
+ self.axisDefaults = {}
+ self._strictAxisNames = True
+
+ @classmethod
+ def fromstring(cls, string, documentObject):
+ f = BytesIO(tobytes(string, encoding="utf-8"))
+ self = cls(f, documentObject)
+ self.path = None
+ return self
+
+ def read(self):
+ self.readAxes()
+ self.readLabels()
+ self.readRules()
+ self.readVariableFonts()
+ self.readSources()
+ self.readInstances()
+ self.readLib()
+
+ def readRules(self):
+ # we also need to read any conditions that are outside of a condition set.
+ rules = []
+ rulesElement = self.root.find(".rules")
+ if rulesElement is not None:
+ processingValue = rulesElement.attrib.get("processing", "first")
+ if processingValue not in {"first", "last"}:
+ raise DesignSpaceDocumentError(
+ " processing attribute value is not valid: %r, "
+ "expected 'first' or 'last'" % processingValue
+ )
+ self.documentObject.rulesProcessingLast = processingValue == "last"
+ for ruleElement in self.root.findall(".rules/rule"):
+ ruleObject = self.ruleDescriptorClass()
+ ruleName = ruleObject.name = ruleElement.attrib.get("name")
+ # read any stray conditions outside a condition set
+ externalConditions = self._readConditionElements(
+ ruleElement,
+ ruleName,
+ )
+ if externalConditions:
+ ruleObject.conditionSets.append(externalConditions)
+ self.log.info(
+ "Found stray rule conditions outside a conditionset. "
+ "Wrapped them in a new conditionset."
+ )
+ # read the conditionsets
+ for conditionSetElement in ruleElement.findall(".conditionset"):
+ conditionSet = self._readConditionElements(
+ conditionSetElement,
+ ruleName,
+ )
+ if conditionSet is not None:
+ ruleObject.conditionSets.append(conditionSet)
+ for subElement in ruleElement.findall(".sub"):
+ a = subElement.attrib["name"]
+ b = subElement.attrib["with"]
+ ruleObject.subs.append((a, b))
+ rules.append(ruleObject)
+ self.documentObject.rules = rules
+
+ def _readConditionElements(self, parentElement, ruleName=None):
+ cds = []
+ for conditionElement in parentElement.findall(".condition"):
+ cd = {}
+ cdMin = conditionElement.attrib.get("minimum")
+ if cdMin is not None:
+ cd["minimum"] = float(cdMin)
+ else:
+ # will allow these to be None, assume axis.minimum
+ cd["minimum"] = None
+ cdMax = conditionElement.attrib.get("maximum")
+ if cdMax is not None:
+ cd["maximum"] = float(cdMax)
+ else:
+ # will allow these to be None, assume axis.maximum
+ cd["maximum"] = None
+ cd["name"] = conditionElement.attrib.get("name")
+ # # test for things
+ if cd.get("minimum") is None and cd.get("maximum") is None:
+ raise DesignSpaceDocumentError(
+ "condition missing required minimum or maximum in rule"
+ + (" '%s'" % ruleName if ruleName is not None else "")
+ )
+ cds.append(cd)
+ return cds
+
+ def readAxes(self):
+ # read the axes elements, including the warp map.
+ axesElement = self.root.find(".axes")
+ if axesElement is not None and "elidedfallbackname" in axesElement.attrib:
+ self.documentObject.elidedFallbackName = axesElement.attrib[
+ "elidedfallbackname"
+ ]
+ axisElements = self.root.findall(".axes/axis")
+ if not axisElements:
+ return
+ for axisElement in axisElements:
+ if (
+ self.documentObject.formatTuple >= (5, 0)
+ and "values" in axisElement.attrib
+ ):
+ axisObject = self.discreteAxisDescriptorClass()
+ axisObject.values = [
+ float(s) for s in axisElement.attrib["values"].split(" ")
+ ]
+ else:
+ axisObject = self.axisDescriptorClass()
+ axisObject.minimum = float(axisElement.attrib.get("minimum"))
+ axisObject.maximum = float(axisElement.attrib.get("maximum"))
+ axisObject.default = float(axisElement.attrib.get("default"))
+ axisObject.name = axisElement.attrib.get("name")
+ if axisElement.attrib.get("hidden", False):
+ axisObject.hidden = True
+ axisObject.tag = axisElement.attrib.get("tag")
+ for mapElement in axisElement.findall("map"):
+ a = float(mapElement.attrib["input"])
+ b = float(mapElement.attrib["output"])
+ axisObject.map.append((a, b))
+ for labelNameElement in axisElement.findall("labelname"):
+ # Note: elementtree reads the "xml:lang" attribute name as
+ # '{http://www.w3.org/XML/1998/namespace}lang'
+ for key, lang in labelNameElement.items():
+ if key == XML_LANG:
+ axisObject.labelNames[lang] = tostr(labelNameElement.text)
+ labelElement = axisElement.find(".labels")
+ if labelElement is not None:
+ if "ordering" in labelElement.attrib:
+ axisObject.axisOrdering = int(labelElement.attrib["ordering"])
+ for label in labelElement.findall(".label"):
+ axisObject.axisLabels.append(self.readAxisLabel(label))
+ self.documentObject.axes.append(axisObject)
+ self.axisDefaults[axisObject.name] = axisObject.default
+
+ self.documentObject.axisMappings = []
+ for mappingsElement in self.root.findall(".axes/mappings"):
+ groupDescription = mappingsElement.attrib.get("description")
+ for mappingElement in mappingsElement.findall("mapping"):
+ description = mappingElement.attrib.get("description")
+ inputElement = mappingElement.find("input")
+ outputElement = mappingElement.find("output")
+ inputLoc = {}
+ outputLoc = {}
+ for dimElement in inputElement.findall(".dimension"):
+ name = dimElement.attrib["name"]
+ value = float(dimElement.attrib["xvalue"])
+ inputLoc[name] = value
+ for dimElement in outputElement.findall(".dimension"):
+ name = dimElement.attrib["name"]
+ value = float(dimElement.attrib["xvalue"])
+ outputLoc[name] = value
+ axisMappingObject = self.axisMappingDescriptorClass(
+ inputLocation=inputLoc,
+ outputLocation=outputLoc,
+ description=description,
+ groupDescription=groupDescription,
+ )
+ self.documentObject.axisMappings.append(axisMappingObject)
+
+ def readAxisLabel(self, element: ET.Element):
+ xml_attrs = {
+ "userminimum",
+ "uservalue",
+ "usermaximum",
+ "name",
+ "elidable",
+ "oldersibling",
+ "linkeduservalue",
+ }
+ unknown_attrs = set(element.attrib) - xml_attrs
+ if unknown_attrs:
+ raise DesignSpaceDocumentError(
+ f"label element contains unknown attributes: {', '.join(unknown_attrs)}"
+ )
+
+ name = element.get("name")
+ if name is None:
+ raise DesignSpaceDocumentError("label element must have a name attribute.")
+ valueStr = element.get("uservalue")
+ if valueStr is None:
+ raise DesignSpaceDocumentError(
+ "label element must have a uservalue attribute."
+ )
+ value = float(valueStr)
+ minimumStr = element.get("userminimum")
+ minimum = float(minimumStr) if minimumStr is not None else None
+ maximumStr = element.get("usermaximum")
+ maximum = float(maximumStr) if maximumStr is not None else None
+ linkedValueStr = element.get("linkeduservalue")
+ linkedValue = float(linkedValueStr) if linkedValueStr is not None else None
+ elidable = True if element.get("elidable") == "true" else False
+ olderSibling = True if element.get("oldersibling") == "true" else False
+ labelNames = {
+ lang: label_name.text or ""
+ for label_name in element.findall("labelname")
+ for attr, lang in label_name.items()
+ if attr == XML_LANG
+ # Note: elementtree reads the "xml:lang" attribute name as
+ # '{http://www.w3.org/XML/1998/namespace}lang'
+ }
+ return self.axisLabelDescriptorClass(
+ name=name,
+ userValue=value,
+ userMinimum=minimum,
+ userMaximum=maximum,
+ elidable=elidable,
+ olderSibling=olderSibling,
+ linkedUserValue=linkedValue,
+ labelNames=labelNames,
+ )
+
+ def readLabels(self):
+ if self.documentObject.formatTuple < (5, 0):
+ return
+
+ xml_attrs = {"name", "elidable", "oldersibling"}
+ for labelElement in self.root.findall(".labels/label"):
+ unknown_attrs = set(labelElement.attrib) - xml_attrs
+ if unknown_attrs:
+ raise DesignSpaceDocumentError(
+ f"Label element contains unknown attributes: {', '.join(unknown_attrs)}"
+ )
+
+ name = labelElement.get("name")
+ if name is None:
+ raise DesignSpaceDocumentError(
+ "label element must have a name attribute."
+ )
+ designLocation, userLocation = self.locationFromElement(labelElement)
+ if designLocation:
+ raise DesignSpaceDocumentError(
+ f'