diff --git a/lib/python3.12/site-packages/datasets/__init__.py b/lib/python3.12/site-packages/datasets/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..18b38d485e5412ea6bd294ea1bf090b3e1291967 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/__init__.py @@ -0,0 +1,47 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__version__ = "3.6.0" + +from .arrow_dataset import Dataset +from .arrow_reader import ReadInstruction +from .builder import ArrowBasedBuilder, BuilderConfig, DatasetBuilder, GeneratorBasedBuilder +from .combine import concatenate_datasets, interleave_datasets +from .dataset_dict import DatasetDict, IterableDatasetDict +from .download import * +from .features import * +from .fingerprint import disable_caching, enable_caching, is_caching_enabled +from .info import DatasetInfo +from .inspect import ( + get_dataset_config_info, + get_dataset_config_names, + get_dataset_default_config_name, + get_dataset_infos, + get_dataset_split_names, +) +from .iterable_dataset import IterableDataset +from .load import load_dataset, load_dataset_builder, load_from_disk +from .splits import ( + NamedSplit, + NamedSplitAll, + Split, + SplitBase, + SplitDict, + SplitGenerator, + SplitInfo, + SubSplitInfo, + percent, +) +from .utils import * +from .utils import logging diff --git a/lib/python3.12/site-packages/datasets/arrow_dataset.py b/lib/python3.12/site-packages/datasets/arrow_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..9b4ad016692013d28852563ed77c1a29ea47da8a --- /dev/null +++ b/lib/python3.12/site-packages/datasets/arrow_dataset.py @@ -0,0 +1,6508 @@ +# Copyright 2020 The HuggingFace Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +"""Simple Dataset wrapping an Arrow Table.""" + +import asyncio +import contextlib +import copy +import fnmatch +import inspect +import itertools +import json +import math +import os +import posixpath +import re +import shutil +import sys +import tempfile +import time +import warnings +import weakref +from collections import Counter +from collections.abc import Iterable, Iterator, Mapping +from collections.abc import Sequence as Sequence_ +from copy import deepcopy +from functools import partial, wraps +from io import BytesIO +from math import ceil, floor +from pathlib import Path +from random import sample +from typing import ( + TYPE_CHECKING, + Any, + BinaryIO, + Callable, + Optional, + Union, + overload, +) + +import fsspec +import numpy as np +import pandas as pd +import pyarrow as pa +import pyarrow.compute as pc +from fsspec.core import url_to_fs +from huggingface_hub import ( + CommitInfo, + CommitOperationAdd, + CommitOperationDelete, + DatasetCard, + DatasetCardData, + HfApi, +) +from huggingface_hub.hf_api import RepoFile +from multiprocess import Pool +from tqdm.contrib.concurrent import thread_map + +from . import config +from .arrow_reader import ArrowReader +from .arrow_writer import ArrowWriter, OptimizedTypedSequence +from .data_files import sanitize_patterns +from .download.streaming_download_manager import xgetsize +from .features import Audio, ClassLabel, Features, Image, Sequence, Value, Video +from .features.features import ( + FeatureType, + _align_features, + _check_if_features_can_be_aligned, + generate_from_arrow_type, + pandas_types_mapper, + require_decoding, +) +from .filesystems import is_remote_filesystem +from .fingerprint import ( + fingerprint_transform, + format_kwargs_for_fingerprint, + format_transform_for_fingerprint, + generate_fingerprint, + generate_random_fingerprint, + get_temporary_cache_files_directory, + is_caching_enabled, + maybe_register_dataset_for_temp_dir_deletion, + update_fingerprint, + validate_fingerprint, +) +from .formatting import format_table, get_format_type_from_alias, get_formatter, query_table +from .formatting.formatting import LazyDict, _is_range_contiguous +from .info import DatasetInfo, DatasetInfosDict +from .naming import _split_re +from .search import IndexableMixin +from .splits import NamedSplit, Split, SplitDict, SplitInfo +from .table import ( + InMemoryTable, + MemoryMappedTable, + Table, + _memory_mapped_record_batch_reader_from_file, + cast_array_to_feature, + concat_tables, + embed_table_storage, + list_table_cache_files, + table_cast, + table_iter, + table_visitor, +) +from .utils import logging +from .utils import tqdm as hf_tqdm +from .utils.file_utils import estimate_dataset_size +from .utils.info_utils import is_small_dataset +from .utils.metadata import MetadataConfigs +from .utils.py_utils import ( + Literal, + asdict, + convert_file_size_to_int, + glob_pattern_to_regex, + iflatmap_unordered, + string_to_dict, +) +from .utils.stratify import stratified_shuffle_split_generate_indices +from .utils.tf_utils import dataset_to_tf, minimal_tf_collate_fn, multiprocess_dataset_to_tf +from .utils.typing import ListLike, PathLike + + +if TYPE_CHECKING: + import sqlite3 + + import polars as pl + import pyspark + import sqlalchemy + + from .dataset_dict import DatasetDict + from .iterable_dataset import IterableDataset + +logger = logging.get_logger(__name__) + +PUSH_TO_HUB_WITHOUT_METADATA_CONFIGS_SPLIT_PATTERN_SHARDED = ( + "data/{split}-[0-9][0-9][0-9][0-9][0-9]-of-[0-9][0-9][0-9][0-9][0-9]*.parquet" +) + + +class DatasetInfoMixin: + """This base class exposes some attributes of DatasetInfo + at the base level of the Dataset for easy access. + """ + + def __init__(self, info: DatasetInfo, split: Optional[NamedSplit]): + self._info = info + self._split = split + + @property + def info(self): + """[`~datasets.DatasetInfo`] object containing all the metadata in the dataset.""" + return self._info + + @property + def split(self): + """[`~datasets.NamedSplit`] object corresponding to a named dataset split.""" + return self._split + + @property + def builder_name(self) -> str: + return self._info.builder_name + + @property + def citation(self) -> str: + return self._info.citation + + @property + def config_name(self) -> str: + return self._info.config_name + + @property + def dataset_size(self) -> Optional[int]: + return self._info.dataset_size + + @property + def description(self) -> str: + return self._info.description + + @property + def download_checksums(self) -> Optional[dict]: + return self._info.download_checksums + + @property + def download_size(self) -> Optional[int]: + return self._info.download_size + + @property + def features(self) -> Optional[Features]: + return self._info.features.copy() if self._info.features is not None else None + + @property + def homepage(self) -> Optional[str]: + return self._info.homepage + + @property + def license(self) -> Optional[str]: + return self._info.license + + @property + def size_in_bytes(self) -> Optional[int]: + return self._info.size_in_bytes + + @property + def supervised_keys(self): + return self._info.supervised_keys + + @property + def version(self): + return self._info.version + + +class TensorflowDatasetMixin: + _TF_DATASET_REFS = set() + + @staticmethod + def _get_output_signature( + dataset: "Dataset", + collate_fn: Callable, + collate_fn_args: dict, + cols_to_retain: Optional[list[str]] = None, + batch_size: Optional[int] = None, + num_test_batches: int = 20, + ): + """Private method used by `to_tf_dataset()` to find the shapes and dtypes of samples from this dataset + after being passed through the collate_fn. Tensorflow needs an exact signature for tf.numpy_function, so + the only way to do this is to run test batches - the collator may add or rename columns, so we can't figure + it out just by inspecting the dataset. + + Args: + dataset (`Dataset`): Dataset to load samples from. + collate_fn(`bool`): Shuffle the dataset order when loading. Recommended True for training, False for + validation/evaluation. + collate_fn(`Callable`): A function or callable object (such as a `DataCollator`) that will collate + lists of samples into a batch. + collate_fn_args (`Dict`): A `dict` of keyword arguments to be passed to the + `collate_fn`. + batch_size (`int`, optional): The size of batches loaded from the dataset. Used for shape inference. + Can be None, which indicates that batch sizes can be variable. + num_test_batches (`int`): The number of batches to load from the dataset for shape inference. + + Returns: + `dict`: Dict mapping column names to tf.Tensorspec objects + `dict`: Dict mapping column names to np.dtype objects + """ + if config.TF_AVAILABLE: + import tensorflow as tf + else: + raise ImportError("Called a Tensorflow-specific function but Tensorflow is not installed.") + + if len(dataset) == 0: + raise ValueError("Unable to get the output signature because the dataset is empty.") + if batch_size is not None: + batch_size = min(len(dataset), batch_size) + test_batch_size = 1 + + if cols_to_retain is not None: + cols_to_retain = list(set(cols_to_retain + ["label_ids", "label", "labels"])) + + test_batches = [] + for _ in range(num_test_batches): + indices = sample(range(len(dataset)), test_batch_size) + test_batch = dataset[indices] + if cols_to_retain is not None: + test_batch = {key: value for key, value in test_batch.items() if key in cols_to_retain} + test_batch = [{key: value[i] for key, value in test_batch.items()} for i in range(test_batch_size)] + test_batch = collate_fn(test_batch, **collate_fn_args) + test_batches.append(test_batch) + + tf_columns_to_signatures = {} + np_columns_to_dtypes = {} + for column in test_batches[0].keys(): + raw_arrays = [batch[column] for batch in test_batches] + # In case the collate_fn returns something strange + np_arrays = [] + for array in raw_arrays: + if isinstance(array, np.ndarray): + np_arrays.append(array) + elif isinstance(array, tf.Tensor): + np_arrays.append(array.numpy()) + else: + np_arrays.append(np.array(array)) + + if np.issubdtype(np_arrays[0].dtype, np.integer) or np_arrays[0].dtype == bool: + tf_dtype = tf.int64 + np_dtype = np.int64 + elif np.issubdtype(np_arrays[0].dtype, np.number): + tf_dtype = tf.float32 + np_dtype = np.float32 + elif np_arrays[0].dtype.kind == "U": # Unicode strings + np_dtype = np.str_ + tf_dtype = tf.string + else: + raise RuntimeError( + f"Unrecognized array dtype {np_arrays[0].dtype}. \n" + "Nested types and image/audio types are not supported yet." + ) + shapes = [array.shape for array in np_arrays] + static_shape = [] + for dim in range(len(shapes[0])): + sizes = {shape[dim] for shape in shapes} + if dim == 0: + static_shape.append(batch_size) + continue + if len(sizes) == 1: # This dimension looks constant + static_shape.append(sizes.pop()) + else: # Use None for variable dimensions + static_shape.append(None) + tf_columns_to_signatures[column] = tf.TensorSpec(shape=static_shape, dtype=tf_dtype) + np_columns_to_dtypes[column] = np_dtype + + return tf_columns_to_signatures, np_columns_to_dtypes + + def to_tf_dataset( + self, + batch_size: Optional[int] = None, + columns: Optional[Union[str, list[str]]] = None, + shuffle: bool = False, + collate_fn: Optional[Callable] = None, + drop_remainder: bool = False, + collate_fn_args: Optional[dict[str, Any]] = None, + label_cols: Optional[Union[str, list[str]]] = None, + prefetch: bool = True, + num_workers: int = 0, + num_test_batches: int = 20, + ): + """Create a `tf.data.Dataset` from the underlying Dataset. This `tf.data.Dataset` will load and collate batches from + the Dataset, and is suitable for passing to methods like `model.fit()` or `model.predict()`. The dataset will yield + `dicts` for both inputs and labels unless the `dict` would contain only a single key, in which case a raw + `tf.Tensor` is yielded instead. + + Args: + batch_size (`int`, *optional*): + Size of batches to load from the dataset. Defaults to `None`, which implies that the dataset won't be + batched, but the returned dataset can be batched later with `tf_dataset.batch(batch_size)`. + columns (`List[str]` or `str`, *optional*): + Dataset column(s) to load in the `tf.data.Dataset`. + Column names that are created by the `collate_fn` and that do not exist in the original dataset can be used. + shuffle(`bool`, defaults to `False`): + Shuffle the dataset order when loading. Recommended `True` for training, `False` for + validation/evaluation. + drop_remainder(`bool`, defaults to `False`): + Drop the last incomplete batch when loading. Ensures + that all batches yielded by the dataset will have the same length on the batch dimension. + collate_fn(`Callable`, *optional*): + A function or callable object (such as a `DataCollator`) that will collate + lists of samples into a batch. + collate_fn_args (`Dict`, *optional*): + An optional `dict` of keyword arguments to be passed to the + `collate_fn`. + label_cols (`List[str]` or `str`, defaults to `None`): + Dataset column(s) to load as labels. + Note that many models compute loss internally rather than letting Keras do it, in which case + passing the labels here is optional, as long as they're in the input `columns`. + prefetch (`bool`, defaults to `True`): + Whether to run the dataloader in a separate thread and maintain + a small buffer of batches for training. Improves performance by allowing data to be loaded in the + background while the model is training. + num_workers (`int`, defaults to `0`): + Number of workers to use for loading the dataset. + num_test_batches (`int`, defaults to `20`): + Number of batches to use to infer the output signature of the dataset. + The higher this number, the more accurate the signature will be, but the longer it will take to + create the dataset. + + Returns: + `tf.data.Dataset` + + Example: + + ```py + >>> ds_train = ds["train"].to_tf_dataset( + ... columns=['input_ids', 'token_type_ids', 'attention_mask', 'label'], + ... shuffle=True, + ... batch_size=16, + ... collate_fn=data_collator, + ... ) + ``` + """ + if config.TF_AVAILABLE: + import tensorflow as tf + else: + raise ImportError("Called a Tensorflow-specific function but Tensorflow is not installed.") + + if (isinstance(columns, list) and len(columns) == 1) or ( + isinstance(label_cols, list) and len(label_cols) == 1 + ): + warnings.warn( + "The output of `to_tf_dataset` will change when a passing single element list for `labels` or " + "`columns` in the next datasets version. To return a tuple structure rather than dict, pass a " + "single string.\n" + "Old behaviour: columns=['a'], labels=['labels'] -> (tf.Tensor, tf.Tensor) \n" + " : columns='a', labels='labels' -> (tf.Tensor, tf.Tensor) \n" + "New behaviour: columns=['a'],labels=['labels'] -> ({'a': tf.Tensor}, {'labels': tf.Tensor}) \n" + " : columns='a', labels='labels' -> (tf.Tensor, tf.Tensor) ", + FutureWarning, + ) + + if isinstance(tf.distribute.get_strategy(), tf.distribute.TPUStrategy): + logger.warning( + "Note that to_tf_dataset() loads the data with a generator rather than a full tf.data " + "pipeline and is not compatible with remote TPU connections. If you encounter errors, please " + "try using a TPU VM or, if your data can fit in memory, loading it into memory as a dict of " + "Tensors instead of streaming with to_tf_dataset()." + ) + + if collate_fn is None: + # Set a very simple default collator that just stacks things together + collate_fn = minimal_tf_collate_fn + if collate_fn_args is None: + collate_fn_args = {} + if label_cols and not columns: + raise ValueError("Cannot specify label_cols without specifying columns!") + if label_cols is None: + label_cols = [] + elif isinstance(label_cols, str): + label_cols = [label_cols] + if len(set(label_cols)) < len(label_cols): + raise ValueError("List of label_cols contains duplicates.") + if columns: + if isinstance(columns, str): + columns = [columns] + if len(set(columns)) < len(columns): + raise ValueError("List of columns contains duplicates.") + cols_to_retain = list(set(columns + label_cols)) + else: + cols_to_retain = None # Indicates keeping all valid columns + columns = [] + + if self.format["type"] not in ["custom", "numpy"]: + dataset = self.with_format("numpy") + else: + dataset = self + + # TODO(Matt, QL): deprecate the retention of label_ids and label + + output_signature, columns_to_np_types = dataset._get_output_signature( + dataset, + collate_fn=collate_fn, + collate_fn_args=collate_fn_args, + cols_to_retain=cols_to_retain, + batch_size=batch_size if drop_remainder else None, + num_test_batches=num_test_batches, + ) + + if "labels" in output_signature: + if ("label_ids" in columns or "label" in columns) and "labels" not in columns: + columns = [col for col in columns if col not in ["label_ids", "label"]] + ["labels"] + if ("label_ids" in label_cols or "label" in label_cols) and "labels" not in label_cols: + label_cols = [col for col in label_cols if col not in ["label_ids", "label"]] + ["labels"] + + for col in columns: + if col not in output_signature: + raise ValueError(f"Column {col} not found in dataset!") + + for col in label_cols: + if col not in output_signature: + raise ValueError(f"Label column {col} not found in dataset!") + + if num_workers == 0: + tf_dataset = dataset_to_tf( + dataset=dataset, + cols_to_retain=cols_to_retain, + collate_fn=collate_fn, + collate_fn_args=collate_fn_args, + columns_to_np_types=columns_to_np_types, + output_signature=output_signature, + shuffle=shuffle, + batch_size=batch_size, + drop_remainder=drop_remainder, + ) + elif num_workers > 0: + if batch_size is None: + raise NotImplementedError( + "`batch_size` must be specified when using multiple workers, as unbatched multiprocessing " + "is not supported yet. Please provide a `batch_size` if `num_workers` is greater than 0." + ) + tf_dataset = multiprocess_dataset_to_tf( + dataset=dataset, + cols_to_retain=cols_to_retain, + collate_fn=collate_fn, + collate_fn_args=collate_fn_args, + columns_to_np_types=columns_to_np_types, + output_signature=output_signature, + shuffle=shuffle, + batch_size=batch_size, + drop_remainder=drop_remainder, + num_workers=num_workers, + ) + else: + raise ValueError("num_workers must be >= 0") + + def split_features_and_labels(input_batch): + # TODO(Matt, QL): deprecate returning the dict content when there's only one key + features = {key: tensor for key, tensor in input_batch.items() if key in columns} + labels = {key: tensor for key, tensor in input_batch.items() if key in label_cols} + if len(features) == 1: + features = list(features.values())[0] + if len(labels) == 1: + labels = list(labels.values())[0] + if isinstance(labels, dict) and len(labels) == 0: + return features + else: + return features, labels + + if cols_to_retain is not None: + tf_dataset = tf_dataset.map(split_features_and_labels) + + if prefetch: + tf_dataset = tf_dataset.prefetch(tf.data.experimental.AUTOTUNE) + + # Remove a reference to the open Arrow file on delete + def cleanup_callback(ref): + dataset.__del__() + self._TF_DATASET_REFS.remove(ref) + + self._TF_DATASET_REFS.add(weakref.ref(tf_dataset, cleanup_callback)) + + return tf_dataset + + +class DatasetTransformationNotAllowedError(Exception): + pass + + +def transmit_format(func): + """Wrapper for dataset transforms that recreate a new Dataset to transmit the format of the original dataset to the new dataset""" + + @wraps(func) + def wrapper(*args, **kwargs): + if args: + self: "Dataset" = args[0] + args = args[1:] + else: + self: "Dataset" = kwargs.pop("self") + # don't use self.format since it returns a list of columns for 'columns' even if self_format_columns is None + unformatted_columns = set(self.column_names) - set(self._format_columns or []) + self_format = { + "type": self._format_type, + "format_kwargs": self._format_kwargs, + "columns": self._format_columns, + "output_all_columns": self._output_all_columns, + } + # apply actual function + out: Union["Dataset", "DatasetDict"] = func(self, *args, **kwargs) + datasets: list["Dataset"] = list(out.values()) if isinstance(out, dict) else [out] + # re-apply format to the output + for dataset in datasets: + new_format = self_format.copy() + if new_format["columns"] is not None: # new formatted columns = (columns - previously unformatted columns) + # sort the columns to have a deterministic list of columns that we can compare with `out_format` + new_format["columns"] = sorted(set(dataset.column_names) - unformatted_columns) + out_format = { + "type": dataset._format_type, + "format_kwargs": dataset._format_kwargs, + "columns": sorted(dataset._format_columns) if dataset._format_columns is not None else None, + "output_all_columns": dataset._output_all_columns, + } + if out_format != new_format: + fingerprint = dataset._fingerprint + dataset.set_format(**new_format) + dataset._fingerprint = fingerprint + return out + + wrapper._decorator_name_ = "transmit_format" + return wrapper + + +def update_metadata_with_features(table: Table, features: Features): + """To be used in dataset transforms that modify the features of the dataset, in order to update the features stored in the metadata of its schema.""" + features = Features({col_name: features[col_name] for col_name in table.column_names}) + if table.schema.metadata is None or b"huggingface" not in table.schema.metadata: + pa_metadata = ArrowWriter._build_metadata(DatasetInfo(features=features)) + else: + metadata = json.loads(table.schema.metadata[b"huggingface"].decode()) + if "info" not in metadata: + metadata["info"] = asdict(DatasetInfo(features=features)) + else: + metadata["info"]["features"] = asdict(DatasetInfo(features=features))["features"] + pa_metadata = {"huggingface": json.dumps(metadata)} + table = table.replace_schema_metadata(pa_metadata) + return table + + +def _check_table(table) -> Table: + """We check the table type to make sure it's an instance of :class:`datasets.table.Table`""" + if isinstance(table, pa.Table): + # for a pyarrow table, we can just consider it as a in-memory table + # this is here for backward compatibility + return InMemoryTable(table) + elif isinstance(table, Table): + return table + else: + raise TypeError(f"Expected a pyarrow.Table or a datasets.table.Table object, but got {table}.") + + +def _check_column_names(column_names: list[str]): + """Check the column names to make sure they don't contain duplicates.""" + counter = Counter(column_names) + if not all(count == 1 for count in counter.values()): + duplicated_columns = [col for col in counter if counter[col] > 1] + raise ValueError(f"The table can't have duplicated columns but columns {duplicated_columns} are duplicated.") + + +def _check_valid_indices_value(index, size): + if (index < 0 and index + size < 0) or (index >= size): + raise IndexError(f"Index {index} out of range for dataset of size {size}.") + + +class NonExistentDatasetError(Exception): + """Used when we expect the existence of a dataset""" + + pass + + +class Dataset(DatasetInfoMixin, IndexableMixin, TensorflowDatasetMixin): + """A Dataset backed by an Arrow table.""" + + def __init__( + self, + arrow_table: Table, + info: Optional[DatasetInfo] = None, + split: Optional[NamedSplit] = None, + indices_table: Optional[Table] = None, + fingerprint: Optional[str] = None, + ): + info = info.copy() if info is not None else DatasetInfo() + DatasetInfoMixin.__init__(self, info=info, split=split) + IndexableMixin.__init__(self) + + self._data: Table = _check_table(arrow_table) + self._indices: Optional[Table] = _check_table(indices_table) if indices_table is not None else None + maybe_register_dataset_for_temp_dir_deletion(self) + + self._format_type: Optional[str] = None + self._format_kwargs: dict = {} + self._format_columns: Optional[list] = None + self._output_all_columns: bool = False + self._fingerprint: str = fingerprint + + # Read metadata + + if self._data.schema.metadata is not None and b"huggingface" in self._data.schema.metadata: + metadata = json.loads(self._data.schema.metadata[b"huggingface"].decode()) + if ( + "fingerprint" in metadata and self._fingerprint is None + ): # try to load fingerprint from the arrow file metadata + self._fingerprint = metadata["fingerprint"] + + # Infer features if None + inferred_features = Features.from_arrow_schema(arrow_table.schema) + if self.info.features is None: + self.info.features = inferred_features + else: # make sure the nested columns are in the right order + try: + self.info.features = self.info.features.reorder_fields_as(inferred_features) + except ValueError as e: + raise ValueError( + f"{e}\nThe 'source' features come from dataset_info.json, and the 'target' ones are those of the dataset arrow file." + ) + + # In case there are types like pa.dictionary that we need to convert to the underlying type + + if self.data.schema != self.info.features.arrow_schema: + self._data = self.data.cast(self.info.features.arrow_schema) + + # Infer fingerprint if None + + if self._fingerprint is None: + self._fingerprint = generate_fingerprint(self) + + # Sanity checks + + if self._info.features is None: + raise ValueError("Features can't be None in a Dataset object") + if self._fingerprint is None: + raise ValueError("Fingerprint can't be None in a Dataset object") + if self.info.features.type != inferred_features.type: + raise ValueError( + f"External features info don't match the dataset:\nGot\n{self.info.features}\nwith type\n{self.info.features.type}\n\nbut expected something like\n{inferred_features}\nwith type\n{inferred_features.type}" + ) + + if self._indices is not None: + if not pa.types.is_unsigned_integer(self._indices.column(0).type): + raise ValueError( + f"indices must be an Arrow table of unsigned integers, current type is {self._indices.column(0).type}" + ) + _check_column_names(self._data.column_names) + + self._data = update_metadata_with_features(self._data, self._info.features) + + @property + def features(self) -> Features: + features = super().features + if features is None: # this is already checked in __init__ + raise ValueError("Features can't be None in a Dataset object") + return features + + @classmethod + def from_file( + cls, + filename: str, + info: Optional[DatasetInfo] = None, + split: Optional[NamedSplit] = None, + indices_filename: Optional[str] = None, + in_memory: bool = False, + ) -> "Dataset": + """Instantiate a Dataset backed by an Arrow table at filename. + + Args: + filename (`str`): + File name of the dataset. + info (`DatasetInfo`, *optional*): + Dataset information, like description, citation, etc. + split (`NamedSplit`, *optional*): + Name of the dataset split. + indices_filename (`str`, *optional*): + File names of the indices. + in_memory (`bool`, defaults to `False`): + Whether to copy the data in-memory. + + Returns: + [`Dataset`] + """ + table = ArrowReader.read_table(filename, in_memory=in_memory) + + if indices_filename is not None: + indices_pa_table = ArrowReader.read_table(indices_filename, in_memory=in_memory) + else: + indices_pa_table = None + + return cls( + arrow_table=table, + info=info, + split=split, + indices_table=indices_pa_table, + ) + + @classmethod + def from_buffer( + cls, + buffer: pa.Buffer, + info: Optional[DatasetInfo] = None, + split: Optional[NamedSplit] = None, + indices_buffer: Optional[pa.Buffer] = None, + ) -> "Dataset": + """Instantiate a Dataset backed by an Arrow buffer. + + Args: + buffer (`pyarrow.Buffer`): + Arrow buffer. + info (`DatasetInfo`, *optional*): + Dataset information, like description, citation, etc. + split (`NamedSplit`, *optional*): + Name of the dataset split. + indices_buffer (`pyarrow.Buffer`, *optional*): + Indices Arrow buffer. + + Returns: + [`Dataset`] + """ + table = InMemoryTable.from_buffer(buffer) + + if indices_buffer is not None: + indices_table = InMemoryTable.from_buffer(buffer) + else: + indices_table = None + + return cls(table, info=info, split=split, indices_table=indices_table) + + @classmethod + def from_pandas( + cls, + df: pd.DataFrame, + features: Optional[Features] = None, + info: Optional[DatasetInfo] = None, + split: Optional[NamedSplit] = None, + preserve_index: Optional[bool] = None, + ) -> "Dataset": + """ + Convert `pandas.DataFrame` to a `pyarrow.Table` to create a [`Dataset`]. + + The column types in the resulting Arrow Table are inferred from the dtypes of the `pandas.Series` in the + DataFrame. In the case of non-object Series, the NumPy dtype is translated to its Arrow equivalent. In the + case of `object`, we need to guess the datatype by looking at the Python objects in this Series. + + Be aware that Series of the `object` dtype don't carry enough information to always lead to a meaningful Arrow + type. In the case that we cannot infer a type, e.g. because the DataFrame is of length 0 or the Series only + contains `None/nan` objects, the type is set to `null`. This behavior can be avoided by constructing explicit + features and passing it to this function. + + Important: a dataset created with from_pandas() lives in memory + and therefore doesn't have an associated cache directory. + This may change in the future, but in the meantime if you + want to reduce memory usage you should write it back on disk + and reload using e.g. save_to_disk / load_from_disk. + + Args: + df (`pandas.DataFrame`): + Dataframe that contains the dataset. + features ([`Features`], *optional*): + Dataset features. + info (`DatasetInfo`, *optional*): + Dataset information, like description, citation, etc. + split (`NamedSplit`, *optional*): + Name of the dataset split. + preserve_index (`bool`, *optional*): + Whether to store the index as an additional column in the resulting Dataset. + The default of `None` will store the index as a column, except for `RangeIndex` which is stored as metadata only. + Use `preserve_index=True` to force it to be stored as a column. + + Returns: + [`Dataset`] + + Example: + + ```py + >>> ds = Dataset.from_pandas(df) + ``` + """ + if info is not None and features is not None and info.features != features: + raise ValueError( + f"Features specified in `features` and `info.features` can't be different:\n{features}\n{info.features}" + ) + features = features if features is not None else info.features if info is not None else None + if info is None: + info = DatasetInfo() + info.features = features + table = InMemoryTable.from_pandas( + df=df, + preserve_index=preserve_index, + ) + if features is not None: + # more expensive cast than InMemoryTable.from_pandas(..., schema=features.arrow_schema) + # needed to support the str to Audio conversion for instance + table = table.cast(features.arrow_schema) + return cls(table, info=info, split=split) + + @classmethod + def from_polars( + cls, + df: "pl.DataFrame", + features: Optional[Features] = None, + info: Optional[DatasetInfo] = None, + split: Optional[NamedSplit] = None, + ) -> "Dataset": + """ + Collect the underlying arrow arrays in an Arrow Table. + + This operation is mostly zero copy. + + Data types that do copy: + * CategoricalType + + Args: + df (`polars.DataFrame`): DataFrame to convert to Arrow Table + features (`Features`, optional): Dataset features. + info (`DatasetInfo`, optional): Dataset information, like description, citation, etc. + split (`NamedSplit`, optional): Name of the dataset split. + + Examples: + ```py + >>> ds = Dataset.from_polars(df) + ``` + """ + if info is not None and features is not None and info.features != features: + raise ValueError( + f"Features specified in `features` and `info.features` can't be different:\n{features}\n{info.features}" + ) + features = features if features is not None else info.features if info is not None else None + if info is None: + info = DatasetInfo() + info.features = features + table = InMemoryTable(df.to_arrow()) + if features is not None: + # more expensive cast than InMemoryTable.from_polars(..., schema=features.arrow_schema) + # needed to support the str to Audio conversion for instance + table = table.cast(features.arrow_schema) + return cls(table, info=info, split=split) + + @classmethod + def from_dict( + cls, + mapping: dict, + features: Optional[Features] = None, + info: Optional[DatasetInfo] = None, + split: Optional[NamedSplit] = None, + ) -> "Dataset": + """ + Convert `dict` to a `pyarrow.Table` to create a [`Dataset`]. + + Important: a dataset created with from_dict() lives in memory + and therefore doesn't have an associated cache directory. + This may change in the future, but in the meantime if you + want to reduce memory usage you should write it back on disk + and reload using e.g. save_to_disk / load_from_disk. + + Args: + mapping (`Mapping`): + Mapping of strings to Arrays or Python lists. + features ([`Features`], *optional*): + Dataset features. + info (`DatasetInfo`, *optional*): + Dataset information, like description, citation, etc. + split (`NamedSplit`, *optional*): + Name of the dataset split. + + Returns: + [`Dataset`] + """ + if info is not None and features is not None and info.features != features: + raise ValueError( + f"Features specified in `features` and `info.features` can't be different:\n{features}\n{info.features}" + ) + features = features if features is not None else info.features if info is not None else None + arrow_typed_mapping = {} + for col, data in mapping.items(): + if isinstance(data, (pa.Array, pa.ChunkedArray)): + data = cast_array_to_feature(data, features[col]) if features is not None else data + else: + data = OptimizedTypedSequence( + features.encode_column(data, col) if features is not None else data, + type=features[col] if features is not None else None, + col=col, + ) + arrow_typed_mapping[col] = data + mapping = arrow_typed_mapping + pa_table = InMemoryTable.from_pydict(mapping=mapping) + if info is None: + info = DatasetInfo() + info.features = features + if info.features is None: + info.features = Features( + { + col: generate_from_arrow_type(data.type) + if isinstance(data, (pa.Array, pa.ChunkedArray)) + else data.get_inferred_type() + for col, data in mapping.items() + } + ) + return cls(pa_table, info=info, split=split) + + @classmethod + def from_list( + cls, + mapping: list[dict], + features: Optional[Features] = None, + info: Optional[DatasetInfo] = None, + split: Optional[NamedSplit] = None, + ) -> "Dataset": + """ + Convert a list of dicts to a `pyarrow.Table` to create a [`Dataset`]`. + + Note that the keys of the first entry will be used to determine the dataset columns, + regardless of what is passed to features. + + Important: a dataset created with from_list() lives in memory + and therefore doesn't have an associated cache directory. + This may change in the future, but in the meantime if you + want to reduce memory usage you should write it back on disk + and reload using e.g. save_to_disk / load_from_disk. + + Args: + mapping (`List[dict]`): A list of mappings of strings to row values. + features (`Features`, optional): Dataset features. + info (`DatasetInfo`, optional): Dataset information, like description, citation, etc. + split (`NamedSplit`, optional): Name of the dataset split. + + Returns: + [`Dataset`] + """ + # for simplicity and consistency wrt OptimizedTypedSequence we do not use InMemoryTable.from_pylist here + mapping = {k: [r.get(k) for r in mapping] for k in mapping[0]} if mapping else {} + return cls.from_dict(mapping, features, info, split) + + @staticmethod + def from_csv( + path_or_paths: Union[PathLike, list[PathLike]], + split: Optional[NamedSplit] = None, + features: Optional[Features] = None, + cache_dir: str = None, + keep_in_memory: bool = False, + num_proc: Optional[int] = None, + **kwargs, + ): + """Create Dataset from CSV file(s). + + Args: + path_or_paths (`path-like` or list of `path-like`): + Path(s) of the CSV file(s). + split ([`NamedSplit`], *optional*): + Split name to be assigned to the dataset. + features ([`Features`], *optional*): + Dataset features. + cache_dir (`str`, *optional*, defaults to `"~/.cache/huggingface/datasets"`): + Directory to cache data. + keep_in_memory (`bool`, defaults to `False`): + Whether to copy the data in-memory. + num_proc (`int`, *optional*, defaults to `None`): + Number of processes when downloading and generating the dataset locally. + This is helpful if the dataset is made of multiple files. Multiprocessing is disabled by default. + + + **kwargs (additional keyword arguments): + Keyword arguments to be passed to [`pandas.read_csv`]. + + Returns: + [`Dataset`] + + Example: + + ```py + >>> ds = Dataset.from_csv('path/to/dataset.csv') + ``` + """ + # Dynamic import to avoid circular dependency + from .io.csv import CsvDatasetReader + + return CsvDatasetReader( + path_or_paths, + split=split, + features=features, + cache_dir=cache_dir, + keep_in_memory=keep_in_memory, + num_proc=num_proc, + **kwargs, + ).read() + + @staticmethod + def from_generator( + generator: Callable, + features: Optional[Features] = None, + cache_dir: str = None, + keep_in_memory: bool = False, + gen_kwargs: Optional[dict] = None, + num_proc: Optional[int] = None, + split: NamedSplit = Split.TRAIN, + **kwargs, + ): + """Create a Dataset from a generator. + + Args: + generator (:`Callable`): + A generator function that `yields` examples. + features ([`Features`], *optional*): + Dataset features. + cache_dir (`str`, *optional*, defaults to `"~/.cache/huggingface/datasets"`): + Directory to cache data. + keep_in_memory (`bool`, defaults to `False`): + Whether to copy the data in-memory. + gen_kwargs(`dict`, *optional*): + Keyword arguments to be passed to the `generator` callable. + You can define a sharded dataset by passing the list of shards in `gen_kwargs` and setting `num_proc` greater than 1. + num_proc (`int`, *optional*, defaults to `None`): + Number of processes when downloading and generating the dataset locally. + This is helpful if the dataset is made of multiple files. Multiprocessing is disabled by default. + If `num_proc` is greater than one, then all list values in `gen_kwargs` must be the same length. These values will be split between calls to the generator. The number of shards will be the minimum of the shortest list in `gen_kwargs` and `num_proc`. + + + split ([`NamedSplit`], defaults to `Split.TRAIN`): + Split name to be assigned to the dataset. + + + **kwargs (additional keyword arguments): + Keyword arguments to be passed to :[`GeneratorConfig`]. + + Returns: + [`Dataset`] + + Example: + + ```py + >>> def gen(): + ... yield {"text": "Good", "label": 0} + ... yield {"text": "Bad", "label": 1} + ... + >>> ds = Dataset.from_generator(gen) + ``` + + ```py + >>> def gen(shards): + ... for shard in shards: + ... with open(shard) as f: + ... for line in f: + ... yield {"line": line} + ... + >>> shards = [f"data{i}.txt" for i in range(32)] + >>> ds = Dataset.from_generator(gen, gen_kwargs={"shards": shards}) + ``` + """ + from .io.generator import GeneratorDatasetInputStream + + return GeneratorDatasetInputStream( + generator=generator, + features=features, + cache_dir=cache_dir, + keep_in_memory=keep_in_memory, + gen_kwargs=gen_kwargs, + num_proc=num_proc, + split=split, + **kwargs, + ).read() + + @staticmethod + def from_json( + path_or_paths: Union[PathLike, list[PathLike]], + split: Optional[NamedSplit] = None, + features: Optional[Features] = None, + cache_dir: str = None, + keep_in_memory: bool = False, + field: Optional[str] = None, + num_proc: Optional[int] = None, + **kwargs, + ): + """Create Dataset from JSON or JSON Lines file(s). + + Args: + path_or_paths (`path-like` or list of `path-like`): + Path(s) of the JSON or JSON Lines file(s). + split ([`NamedSplit`], *optional*): + Split name to be assigned to the dataset. + features ([`Features`], *optional*): + Dataset features. + cache_dir (`str`, *optional*, defaults to `"~/.cache/huggingface/datasets"`): + Directory to cache data. + keep_in_memory (`bool`, defaults to `False`): + Whether to copy the data in-memory. + field (`str`, *optional*): + Field name of the JSON file where the dataset is contained in. + num_proc (`int`, *optional* defaults to `None`): + Number of processes when downloading and generating the dataset locally. + This is helpful if the dataset is made of multiple files. Multiprocessing is disabled by default. + + + **kwargs (additional keyword arguments): + Keyword arguments to be passed to [`JsonConfig`]. + + Returns: + [`Dataset`] + + Example: + + ```py + >>> ds = Dataset.from_json('path/to/dataset.json') + ``` + """ + # Dynamic import to avoid circular dependency + from .io.json import JsonDatasetReader + + return JsonDatasetReader( + path_or_paths, + split=split, + features=features, + cache_dir=cache_dir, + keep_in_memory=keep_in_memory, + field=field, + num_proc=num_proc, + **kwargs, + ).read() + + @staticmethod + def from_parquet( + path_or_paths: Union[PathLike, list[PathLike]], + split: Optional[NamedSplit] = None, + features: Optional[Features] = None, + cache_dir: str = None, + keep_in_memory: bool = False, + columns: Optional[list[str]] = None, + num_proc: Optional[int] = None, + **kwargs, + ): + """Create Dataset from Parquet file(s). + + Args: + path_or_paths (`path-like` or list of `path-like`): + Path(s) of the Parquet file(s). + split (`NamedSplit`, *optional*): + Split name to be assigned to the dataset. + features (`Features`, *optional*): + Dataset features. + cache_dir (`str`, *optional*, defaults to `"~/.cache/huggingface/datasets"`): + Directory to cache data. + keep_in_memory (`bool`, defaults to `False`): + Whether to copy the data in-memory. + columns (`List[str]`, *optional*): + If not `None`, only these columns will be read from the file. + A column name may be a prefix of a nested field, e.g. 'a' will select + 'a.b', 'a.c', and 'a.d.e'. + num_proc (`int`, *optional*, defaults to `None`): + Number of processes when downloading and generating the dataset locally. + This is helpful if the dataset is made of multiple files. Multiprocessing is disabled by default. + + + **kwargs (additional keyword arguments): + Keyword arguments to be passed to [`ParquetConfig`]. + + Returns: + [`Dataset`] + + Example: + + ```py + >>> ds = Dataset.from_parquet('path/to/dataset.parquet') + ``` + """ + # Dynamic import to avoid circular dependency + from .io.parquet import ParquetDatasetReader + + return ParquetDatasetReader( + path_or_paths, + split=split, + features=features, + cache_dir=cache_dir, + keep_in_memory=keep_in_memory, + columns=columns, + num_proc=num_proc, + **kwargs, + ).read() + + @staticmethod + def from_text( + path_or_paths: Union[PathLike, list[PathLike]], + split: Optional[NamedSplit] = None, + features: Optional[Features] = None, + cache_dir: str = None, + keep_in_memory: bool = False, + num_proc: Optional[int] = None, + **kwargs, + ): + """Create Dataset from text file(s). + + Args: + path_or_paths (`path-like` or list of `path-like`): + Path(s) of the text file(s). + split (`NamedSplit`, *optional*): + Split name to be assigned to the dataset. + features (`Features`, *optional*): + Dataset features. + cache_dir (`str`, *optional*, defaults to `"~/.cache/huggingface/datasets"`): + Directory to cache data. + keep_in_memory (`bool`, defaults to `False`): + Whether to copy the data in-memory. + num_proc (`int`, *optional*, defaults to `None`): + Number of processes when downloading and generating the dataset locally. + This is helpful if the dataset is made of multiple files. Multiprocessing is disabled by default. + + + **kwargs (additional keyword arguments): + Keyword arguments to be passed to [`TextConfig`]. + + Returns: + [`Dataset`] + + Example: + + ```py + >>> ds = Dataset.from_text('path/to/dataset.txt') + ``` + """ + # Dynamic import to avoid circular dependency + from .io.text import TextDatasetReader + + return TextDatasetReader( + path_or_paths, + split=split, + features=features, + cache_dir=cache_dir, + keep_in_memory=keep_in_memory, + num_proc=num_proc, + **kwargs, + ).read() + + @staticmethod + def from_spark( + df: "pyspark.sql.DataFrame", + split: Optional[NamedSplit] = None, + features: Optional[Features] = None, + keep_in_memory: bool = False, + cache_dir: str = None, + working_dir: str = None, + load_from_cache_file: bool = True, + **kwargs, + ): + """Create a Dataset from Spark DataFrame. Dataset downloading is distributed over Spark workers. + + Args: + df (`pyspark.sql.DataFrame`): + The DataFrame containing the desired data. + split (`NamedSplit`, *optional*): + Split name to be assigned to the dataset. + features (`Features`, *optional*): + Dataset features. + cache_dir (`str`, *optional*, defaults to `"~/.cache/huggingface/datasets"`): + Directory to cache data. When using a multi-node Spark cluster, the cache_dir must be accessible to both + workers and the driver. + keep_in_memory (`bool`): + Whether to copy the data in-memory. + working_dir (`str`, *optional*) + Intermediate directory for each Spark worker to write data to before moving it to `cache_dir`. Setting + a non-NFS intermediate directory may improve performance. + load_from_cache_file (`bool`): + Whether to load the dataset from the cache if possible. + + Returns: + [`Dataset`] + + Example: + + ```py + >>> df = spark.createDataFrame( + >>> data=[[1, "Elia"], [2, "Teo"], [3, "Fang"]], + >>> columns=["id", "name"], + >>> ) + >>> ds = Dataset.from_spark(df) + ``` + """ + # Dynamic import to avoid circular dependency + from .io.spark import SparkDatasetReader + + if sys.platform == "win32": + raise OSError("Dataset.from_spark is not currently supported on Windows") + + return SparkDatasetReader( + df, + split=split, + features=features, + streaming=False, + cache_dir=cache_dir, + keep_in_memory=keep_in_memory, + working_dir=working_dir, + load_from_cache_file=load_from_cache_file, + **kwargs, + ).read() + + @staticmethod + def from_sql( + sql: Union[str, "sqlalchemy.sql.Selectable"], + con: Union[str, "sqlalchemy.engine.Connection", "sqlalchemy.engine.Engine", "sqlite3.Connection"], + features: Optional[Features] = None, + cache_dir: str = None, + keep_in_memory: bool = False, + **kwargs, + ): + """Create Dataset from SQL query or database table. + + Args: + sql (`str` or `sqlalchemy.sql.Selectable`): + SQL query to be executed or a table name. + con (`str` or `sqlite3.Connection` or `sqlalchemy.engine.Connection` or `sqlalchemy.engine.Connection`): + A [URI string](https://docs.sqlalchemy.org/en/13/core/engines.html#database-urls) used to instantiate a database connection or a SQLite3/SQLAlchemy connection object. + features ([`Features`], *optional*): + Dataset features. + cache_dir (`str`, *optional*, defaults to `"~/.cache/huggingface/datasets"`): + Directory to cache data. + keep_in_memory (`bool`, defaults to `False`): + Whether to copy the data in-memory. + **kwargs (additional keyword arguments): + Keyword arguments to be passed to [`SqlConfig`]. + + Returns: + [`Dataset`] + + Example: + + ```py + >>> # Fetch a database table + >>> ds = Dataset.from_sql("test_data", "postgres:///db_name") + >>> # Execute a SQL query on the table + >>> ds = Dataset.from_sql("SELECT sentence FROM test_data", "postgres:///db_name") + >>> # Use a Selectable object to specify the query + >>> from sqlalchemy import select, text + >>> stmt = select([text("sentence")]).select_from(text("test_data")) + >>> ds = Dataset.from_sql(stmt, "postgres:///db_name") + ``` + + + + The returned dataset can only be cached if `con` is specified as URI string. + + + """ + from .io.sql import SqlDatasetReader + + return SqlDatasetReader( + sql, + con, + features=features, + cache_dir=cache_dir, + keep_in_memory=keep_in_memory, + **kwargs, + ).read() + + def __setstate__(self, state): + self.__dict__.update(state) + maybe_register_dataset_for_temp_dir_deletion(self) + return self + + def __del__(self): + if hasattr(self, "_data"): + del self._data + if hasattr(self, "_indices"): + del self._indices + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + # Here `del` is used to del the pyarrow tables. This properly closes the files used for memory mapped tables + self.__del__() + + def save_to_disk( + self, + dataset_path: PathLike, + max_shard_size: Optional[Union[str, int]] = None, + num_shards: Optional[int] = None, + num_proc: Optional[int] = None, + storage_options: Optional[dict] = None, + ): + """ + Saves a dataset to a dataset directory, or in a filesystem using any implementation of `fsspec.spec.AbstractFileSystem`. + + For [`Image`], [`Audio`] and [`Video`] data: + + All the Image(), Audio() and Video() data are stored in the arrow files. + If you want to store paths or urls, please use the Value("string") type. + + Args: + dataset_path (`path-like`): + Path (e.g. `dataset/train`) or remote URI (e.g. `s3://my-bucket/dataset/train`) + of the dataset directory where the dataset will be saved to. + max_shard_size (`int` or `str`, *optional*, defaults to `"500MB"`): + The maximum size of the dataset shards to be uploaded to the hub. If expressed as a string, needs to be digits followed by a unit + (like `"50MB"`). + num_shards (`int`, *optional*): + Number of shards to write. By default the number of shards depends on `max_shard_size` and `num_proc`. + + + num_proc (`int`, *optional*): + Number of processes when downloading and generating the dataset locally. + Multiprocessing is disabled by default. + + + storage_options (`dict`, *optional*): + Key/value pairs to be passed on to the file-system backend, if any. + + + + Example: + + ```py + >>> ds.save_to_disk("path/to/dataset/directory") + >>> ds.save_to_disk("path/to/dataset/directory", max_shard_size="1GB") + >>> ds.save_to_disk("path/to/dataset/directory", num_shards=1024) + ``` + """ + if max_shard_size is not None and num_shards is not None: + raise ValueError( + "Failed to push_to_hub: please specify either max_shard_size or num_shards, but not both." + ) + if self.list_indexes(): + raise ValueError("please remove all the indexes using `dataset.drop_index` before saving a dataset") + + if num_shards is None: + dataset_nbytes = self._estimate_nbytes() + max_shard_size = convert_file_size_to_int(max_shard_size or config.MAX_SHARD_SIZE) + num_shards = int(dataset_nbytes / max_shard_size) + 1 + num_shards = max(num_shards, num_proc or 1) + + num_proc = num_proc if num_proc is not None else 1 + num_shards = num_shards if num_shards is not None else num_proc + + fs: fsspec.AbstractFileSystem + fs, _ = url_to_fs(dataset_path, **(storage_options or {})) + + if not is_remote_filesystem(fs): + parent_cache_files_paths = { + Path(cache_filename["filename"]).resolve().parent for cache_filename in self.cache_files + } + # Check that the dataset doesn't overwrite itself. It can cause a permission error on Windows and a segfault on linux. + if Path(dataset_path).expanduser().resolve() in parent_cache_files_paths: + raise PermissionError( + f"Tried to overwrite {Path(dataset_path).expanduser().resolve()} but a dataset can't overwrite itself." + ) + + fs.makedirs(dataset_path, exist_ok=True) + + # Get json serializable state + state = { + key: self.__dict__[key] + for key in [ + "_fingerprint", + "_format_columns", + "_format_kwargs", + "_format_type", + "_output_all_columns", + ] + } + state["_split"] = str(self.split) if self.split is not None else self.split + state["_data_files"] = [ + {"filename": f"data-{shard_idx:05d}-of-{num_shards:05d}.arrow"} for shard_idx in range(num_shards) + ] + for k in state["_format_kwargs"].keys(): + try: + json.dumps(state["_format_kwargs"][k]) + except TypeError as e: + raise TypeError( + str(e) + f"\nThe format kwargs must be JSON serializable, but key '{k}' isn't." + ) from None + # Get json serializable dataset info + dataset_info = asdict(self._info) + + shards_done = 0 + pbar = hf_tqdm( + unit=" examples", + total=len(self), + desc=f"Saving the dataset ({shards_done}/{num_shards} shards)", + ) + kwargs_per_job = ( + { + "job_id": shard_idx, + "shard": self.shard(num_shards=num_shards, index=shard_idx, contiguous=True), + "fpath": posixpath.join(dataset_path, f"data-{shard_idx:05d}-of-{num_shards:05d}.arrow"), + "storage_options": storage_options, + } + for shard_idx in range(num_shards) + ) + shard_lengths = [None] * num_shards + shard_sizes = [None] * num_shards + if num_proc > 1: + with Pool(num_proc) as pool: + with pbar: + for job_id, done, content in iflatmap_unordered( + pool, Dataset._save_to_disk_single, kwargs_iterable=kwargs_per_job + ): + if done: + shards_done += 1 + pbar.set_description(f"Saving the dataset ({shards_done}/{num_shards} shards)") + logger.debug(f"Finished writing shard number {job_id} of {num_shards}.") + shard_lengths[job_id], shard_sizes[job_id] = content + else: + pbar.update(content) + else: + with pbar: + for kwargs in kwargs_per_job: + for job_id, done, content in Dataset._save_to_disk_single(**kwargs): + if done: + shards_done += 1 + pbar.set_description(f"Saving the dataset ({shards_done}/{num_shards} shards)") + logger.debug(f"Finished writing shard number {job_id} of {num_shards}.") + shard_lengths[job_id], shard_sizes[job_id] = content + else: + pbar.update(content) + with fs.open( + posixpath.join(dataset_path, config.DATASET_STATE_JSON_FILENAME), "w", encoding="utf-8" + ) as state_file: + json.dump(state, state_file, indent=2, sort_keys=True) + with fs.open( + posixpath.join(dataset_path, config.DATASET_INFO_FILENAME), "w", encoding="utf-8" + ) as dataset_info_file: + # Sort only the first level of keys, or we might shuffle fields of nested features if we use sort_keys=True + sorted_keys_dataset_info = {key: dataset_info[key] for key in sorted(dataset_info)} + json.dump(sorted_keys_dataset_info, dataset_info_file, indent=2) + + @staticmethod + def _save_to_disk_single(job_id: int, shard: "Dataset", fpath: str, storage_options: Optional[dict]): + batch_size = config.DEFAULT_MAX_BATCH_SIZE + + num_examples_progress_update = 0 + writer = ArrowWriter( + features=shard.features, + path=fpath, + storage_options=storage_options, + embed_local_files=True, + ) + try: + _time = time.time() + for pa_table in shard.with_format("arrow").iter(batch_size): + writer.write_table(pa_table) + num_examples_progress_update += len(pa_table) + if time.time() > _time + config.PBAR_REFRESH_TIME_INTERVAL: + _time = time.time() + yield job_id, False, num_examples_progress_update + num_examples_progress_update = 0 + finally: + yield job_id, False, num_examples_progress_update + num_examples, num_bytes = writer.finalize() + writer.close() + + yield job_id, True, (num_examples, num_bytes) + + @staticmethod + def _build_local_temp_path(uri_or_path: str) -> Path: + """ + Builds and returns a Path concatenating a local temporary dir with the dir path (or absolute/relative + path extracted from the uri) passed. + + Args: + uri_or_path (`str`): Path (e.g. `"dataset/train"`) or remote URI (e.g. + `"s3://my-bucket/dataset/train"`) to concatenate. + + Returns: + :class:`Path`: the concatenated path (temp dir + path) + """ + src_dataset_path = Path(uri_or_path) + tmp_dir = get_temporary_cache_files_directory() + return Path(tmp_dir, src_dataset_path.relative_to(src_dataset_path.anchor)) + + @staticmethod + def load_from_disk( + dataset_path: PathLike, + keep_in_memory: Optional[bool] = None, + storage_options: Optional[dict] = None, + ) -> "Dataset": + """ + Loads a dataset that was previously saved using [`save_to_disk`] from a dataset directory, or from a + filesystem using any implementation of `fsspec.spec.AbstractFileSystem`. + + Args: + dataset_path (`path-like`): + Path (e.g. `"dataset/train"`) or remote URI (e.g. `"s3//my-bucket/dataset/train"`) + of the dataset directory where the dataset will be loaded from. + keep_in_memory (`bool`, defaults to `None`): + Whether to copy the dataset in-memory. If `None`, the + dataset will not be copied in-memory unless explicitly enabled by setting + `datasets.config.IN_MEMORY_MAX_SIZE` to nonzero. See more details in the + [improve performance](../cache#improve-performance) section. + storage_options (`dict`, *optional*): + Key/value pairs to be passed on to the file-system backend, if any. + + + + Returns: + [`Dataset`] or [`DatasetDict`]: + - If `dataset_path` is a path of a dataset directory, the dataset requested. + - If `dataset_path` is a path of a dataset dict directory, a `datasets.DatasetDict` with each split. + + Example: + + ```py + >>> ds = load_from_disk("path/to/dataset/directory") + ``` + """ + fs: fsspec.AbstractFileSystem + fs, dataset_path = url_to_fs(dataset_path, **(storage_options or {})) + + dest_dataset_path = dataset_path + dataset_dict_json_path = posixpath.join(dest_dataset_path, config.DATASETDICT_JSON_FILENAME) + dataset_state_json_path = posixpath.join(dest_dataset_path, config.DATASET_STATE_JSON_FILENAME) + dataset_info_path = posixpath.join(dest_dataset_path, config.DATASET_INFO_FILENAME) + + dataset_dict_is_file = fs.isfile(dataset_dict_json_path) + dataset_info_is_file = fs.isfile(dataset_info_path) + dataset_state_is_file = fs.isfile(dataset_state_json_path) + if not dataset_info_is_file and not dataset_state_is_file: + if dataset_dict_is_file: + raise FileNotFoundError( + f"No such files: '{dataset_info_path}', nor '{dataset_state_json_path}' found. Expected to load a `Dataset` object, but got a `DatasetDict`. Please use either `datasets.load_from_disk` or `DatasetDict.load_from_disk` instead." + ) + raise FileNotFoundError( + f"No such files: '{dataset_info_path}', nor '{dataset_state_json_path}' found. Expected to load a `Dataset` object but provided path is not a `Dataset`." + ) + if not dataset_info_is_file: + if dataset_dict_is_file: + raise FileNotFoundError( + f"No such file: '{dataset_info_path}' found. Expected to load a `Dataset` object, but got a `DatasetDict`. Please use either `datasets.load_from_disk` or `DatasetDict.load_from_disk` instead." + ) + raise FileNotFoundError( + f"No such file: '{dataset_info_path}'. Expected to load a `Dataset` object but provided path is not a `Dataset`." + ) + if not dataset_state_is_file: + if dataset_dict_is_file: + raise FileNotFoundError( + f"No such file: '{dataset_state_json_path}' found. Expected to load a `Dataset` object, but got a `DatasetDict`. Please use either `datasets.load_from_disk` or `DatasetDict.load_from_disk` instead." + ) + raise FileNotFoundError( + f"No such file: '{dataset_state_json_path}'. Expected to load a `Dataset` object but provided path is not a `Dataset`." + ) + + # copies file from filesystem if it is remote filesystem to local filesystem and modifies dataset_path to temp directory containing local copies + if is_remote_filesystem(fs): + src_dataset_path = dest_dataset_path + dest_dataset_path = Dataset._build_local_temp_path(src_dataset_path) + fs.download(src_dataset_path, dest_dataset_path.as_posix(), recursive=True) + dataset_state_json_path = posixpath.join(dest_dataset_path, config.DATASET_STATE_JSON_FILENAME) + dataset_info_path = posixpath.join(dest_dataset_path, config.DATASET_INFO_FILENAME) + + with open(dataset_state_json_path, encoding="utf-8") as state_file: + state = json.load(state_file) + with open(dataset_info_path, encoding="utf-8") as dataset_info_file: + dataset_info = DatasetInfo.from_dict(json.load(dataset_info_file)) + + dataset_size = estimate_dataset_size( + Path(dest_dataset_path, data_file["filename"]) for data_file in state["_data_files"] + ) + keep_in_memory = keep_in_memory if keep_in_memory is not None else is_small_dataset(dataset_size) + table_cls = InMemoryTable if keep_in_memory else MemoryMappedTable + + arrow_table = concat_tables( + thread_map( + table_cls.from_file, + [posixpath.join(dest_dataset_path, data_file["filename"]) for data_file in state["_data_files"]], + tqdm_class=hf_tqdm, + desc="Loading dataset from disk", + # set `disable=None` rather than `disable=False` by default to disable progress bar when no TTY attached + disable=len(state["_data_files"]) <= 16 or None, + ) + ) + + split = state["_split"] + split = Split(split) if split is not None else split + + dataset = Dataset( + arrow_table=arrow_table, + info=dataset_info, + split=split, + fingerprint=state["_fingerprint"], + ) + + format = { + "type": state["_format_type"], + "format_kwargs": state["_format_kwargs"], + "columns": state["_format_columns"], + "output_all_columns": state["_output_all_columns"], + } + dataset = dataset.with_format(**format) + + return dataset + + @property + def data(self) -> Table: + """The Apache Arrow table backing the dataset. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> ds.data + MemoryMappedTable + text: string + label: int64 + ---- + text: [["compassionately explores the seemingly irreconcilable situation between conservative christian parents and their estranged gay and lesbian children .","the soundtrack alone is worth the price of admission .","rodriguez does a splendid job of racial profiling hollywood style--casting excellent latin actors of all ages--a trend long overdue .","beneath the film's obvious determination to shock at any cost lies considerable skill and determination , backed by sheer nerve .","bielinsky is a filmmaker of impressive talent .","so beautifully acted and directed , it's clear that washington most certainly has a new career ahead of him if he so chooses .","a visual spectacle full of stunning images and effects .","a gentle and engrossing character study .","it's enough to watch huppert scheming , with her small , intelligent eyes as steady as any noir villain , and to enjoy the perfectly pitched web of tension that chabrol spins .","an engrossing portrait of uncompromising artists trying to create something original against the backdrop of a corporate music industry that only seems to care about the bottom line .",...,"ultimately , jane learns her place as a girl , softens up and loses some of the intensity that made her an interesting character to begin with .","ah-nuld's action hero days might be over .","it's clear why deuces wild , which was shot two years ago , has been gathering dust on mgm's shelf .","feels like nothing quite so much as a middle-aged moviemaker's attempt to surround himself with beautiful , half-naked women .","when the precise nature of matthew's predicament finally comes into sharp focus , the revelation fails to justify the build-up .","this picture is murder by numbers , and as easy to be bored by as your abc's , despite a few whopping shootouts .","hilarious musical comedy though stymied by accents thick as mud .","if you are into splatter movies , then you will probably have a reasonably good time with the salton sea .","a dull , simple-minded and stereotypical tale of drugs , death and mind-numbing indifference on the inner-city streets .","the feature-length stretch . . . strains the show's concept ."]] + label: [[1,1,1,1,1,1,1,1,1,1,...,0,0,0,0,0,0,0,0,0,0]] + ``` + """ + return self._data + + @property + def cache_files(self) -> list[dict]: + """The cache files containing the Apache Arrow table backing the dataset. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> ds.cache_files + [{'filename': '/root/.cache/huggingface/datasets/rotten_tomatoes_movie_review/default/1.0.0/40d411e45a6ce3484deed7cc15b82a53dad9a72aafd9f86f8f227134bec5ca46/rotten_tomatoes_movie_review-validation.arrow'}] + ``` + """ + cache_files = list_table_cache_files(self._data) + if self._indices is not None: + cache_files += list_table_cache_files(self._indices) + return [{"filename": cache_filename} for cache_filename in cache_files] + + @property + def num_columns(self) -> int: + """Number of columns in the dataset. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> ds.num_columns + 2 + ``` + """ + return self._data.num_columns + + @property + def num_rows(self) -> int: + """Number of rows in the dataset (same as [`Dataset.__len__`]). + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> ds.num_rows + 1066 + ``` + """ + if self._indices is not None: + return self._indices.num_rows + return self._data.num_rows + + @property + def column_names(self) -> list[str]: + """Names of the columns in the dataset. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> ds.column_names + ['text', 'label'] + ``` + """ + return self._data.column_names + + @property + def shape(self) -> tuple[int, int]: + """Shape of the dataset (number of columns, number of rows). + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> ds.shape + (1066, 2) + ``` + """ + if self._indices is not None: + return (self._indices.num_rows, self._data.num_columns) + return self._data.shape + + def unique(self, column: str) -> list: + """Return a list of the unique elements in a column. + + This is implemented in the low-level backend and as such, very fast. + + Args: + column (`str`): + Column name (list all the column names with [`~datasets.Dataset.column_names`]). + + Returns: + `list`: List of unique elements in the given column. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> ds.unique('label') + [1, 0] + ``` + """ + if column not in self._data.column_names: + raise ValueError(f"Column ({column}) not in table columns ({self._data.column_names}).") + + if self._indices is not None and self._indices.num_rows != self._data.num_rows: + dataset = self.flatten_indices() + else: + dataset = self + + return dataset._data.column(column).unique().to_pylist() + + def class_encode_column(self, column: str, include_nulls: bool = False) -> "Dataset": + """Casts the given column as [`~datasets.features.ClassLabel`] and updates the table. + + Args: + column (`str`): + The name of the column to cast (list all the column names with [`~datasets.Dataset.column_names`]) + include_nulls (`bool`, defaults to `False`): + Whether to include null values in the class labels. If `True`, the null values will be encoded as the `"None"` class label. + + + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("boolq", split="validation") + >>> ds.features + {'answer': Value(dtype='bool', id=None), + 'passage': Value(dtype='string', id=None), + 'question': Value(dtype='string', id=None)} + >>> ds = ds.class_encode_column('answer') + >>> ds.features + {'answer': ClassLabel(num_classes=2, names=['False', 'True'], id=None), + 'passage': Value(dtype='string', id=None), + 'question': Value(dtype='string', id=None)} + ``` + """ + # Sanity checks + if column not in self._data.column_names: + raise ValueError(f"Column ({column}) not in table columns ({self._data.column_names}).") + src_feat = self._info.features[column] + if not isinstance(src_feat, Value): + raise ValueError( + f"Class encoding is only supported for {Value.__name__} column, and column {column} is {type(src_feat).__name__}." + ) + + if src_feat.dtype != "string" or (include_nulls and None in self.unique(column)): + + def stringify_column(batch): + batch[column] = [ + str(sample) if include_nulls or sample is not None else None for sample in batch[column] + ] + return batch + + dset = self.map( + stringify_column, + batched=True, + desc="Stringifying the column", + ) + else: + dset = self + + # Create the new feature + class_names = sorted(str(sample) for sample in dset.unique(column) if include_nulls or sample is not None) + dst_feat = ClassLabel(names=class_names) + + def cast_to_class_labels(batch): + batch[column] = [ + dst_feat.str2int(str(sample)) if include_nulls or sample is not None else None + for sample in batch[column] + ] + return batch + + new_features = dset.features.copy() + new_features[column] = dst_feat + + dset = dset.map( + cast_to_class_labels, + batched=True, + features=new_features, + desc="Casting to class labels", + ) + + return dset + + @fingerprint_transform(inplace=False) + def flatten(self, new_fingerprint: Optional[str] = None, max_depth=16) -> "Dataset": + """Flatten the table. + Each column with a struct type is flattened into one column per struct field. + Other columns are left unchanged. + + Args: + new_fingerprint (`str`, *optional*): + The new fingerprint of the dataset after transform. + If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments. + + Returns: + [`Dataset`]: A copy of the dataset with flattened columns. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("rajpurkar/squad", split="train") + >>> ds.features + {'answers': Sequence(feature={'text': Value(dtype='string', id=None), 'answer_start': Value(dtype='int32', id=None)}, length=-1, id=None), + 'context': Value(dtype='string', id=None), + 'id': Value(dtype='string', id=None), + 'question': Value(dtype='string', id=None), + 'title': Value(dtype='string', id=None)} + >>> ds.flatten() + Dataset({ + features: ['id', 'title', 'context', 'question', 'answers.text', 'answers.answer_start'], + num_rows: 87599 + }) + ``` + """ + dataset = copy.deepcopy(self) + for depth in range(1, max_depth): + if any(isinstance(field.type, pa.StructType) for field in dataset._data.schema): + dataset._data = dataset._data.flatten() + else: + break + dataset.info.features = self._info.features.flatten(max_depth=max_depth) + dataset.info.features = Features({col: dataset.info.features[col] for col in dataset.data.column_names}) + dataset._data = update_metadata_with_features(dataset._data, dataset.features) + logger.info(f"Flattened dataset from depth {depth} to depth {1 if depth + 1 < max_depth else 'unknown'}.") + dataset._fingerprint = new_fingerprint + return dataset + + def cast( + self, + features: Features, + batch_size: Optional[int] = 1000, + keep_in_memory: bool = False, + load_from_cache_file: Optional[bool] = None, + cache_file_name: Optional[str] = None, + writer_batch_size: Optional[int] = 1000, + num_proc: Optional[int] = None, + ) -> "Dataset": + """ + Cast the dataset to a new set of features. + + Args: + features ([`Features`]): + New features to cast the dataset to. + The name of the fields in the features must match the current column names. + The type of the data must also be convertible from one type to the other. + For non-trivial conversion, e.g. `str` <-> `ClassLabel` you should use [`~datasets.Dataset.map`] to update the Dataset. + batch_size (`int`, defaults to `1000`): + Number of examples per batch provided to cast. + If `batch_size <= 0` or `batch_size == None` then provide the full dataset as a single batch to cast. + keep_in_memory (`bool`, defaults to `False`): + Whether to copy the data in-memory. + load_from_cache_file (`bool`, defaults to `True` if caching is enabled): + If a cache file storing the current computation from `function` + can be identified, use it instead of recomputing. + cache_file_name (`str`, *optional*, defaults to `None`): + Provide the name of a path for the cache file. It is used to store the + results of the computation instead of the automatically generated cache file name. + writer_batch_size (`int`, defaults to `1000`): + Number of rows per write operation for the cache file writer. + This value is a good trade-off between memory usage during the processing, and processing speed. + Higher value makes the processing do fewer lookups, lower value consume less temporary memory while running [`~datasets.Dataset.map`]. + num_proc (`int`, *optional*, defaults to `None`): + Number of processes for multiprocessing. By default it doesn't + use multiprocessing. + + Returns: + [`Dataset`]: A copy of the dataset with casted features. + + Example: + + ```py + >>> from datasets import load_dataset, ClassLabel, Value + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> ds.features + {'label': ClassLabel(names=['neg', 'pos'], id=None), + 'text': Value(dtype='string', id=None)} + >>> new_features = ds.features.copy() + >>> new_features['label'] = ClassLabel(names=['bad', 'good']) + >>> new_features['text'] = Value('large_string') + >>> ds = ds.cast(new_features) + >>> ds.features + {'label': ClassLabel(names=['bad', 'good'], id=None), + 'text': Value(dtype='large_string', id=None)} + ``` + """ + if sorted(features) != sorted(self._data.column_names): + raise ValueError( + f"The columns in features ({list(features)}) must be identical " + f"as the columns in the dataset: {self._data.column_names}" + ) + + schema = features.arrow_schema + format = self.format + dataset = self.with_format("arrow") + # capture the PyArrow version here to make the lambda serializable on Windows + dataset = dataset.map( + partial(table_cast, schema=schema), + batched=True, + batch_size=batch_size, + keep_in_memory=keep_in_memory, + load_from_cache_file=load_from_cache_file, + cache_file_name=cache_file_name, + writer_batch_size=writer_batch_size, + num_proc=num_proc, + features=features, + desc="Casting the dataset", + ) + dataset = dataset.with_format(**format) + return dataset + + @fingerprint_transform(inplace=False) + def cast_column(self, column: str, feature: FeatureType, new_fingerprint: Optional[str] = None) -> "Dataset": + """Cast column to feature for decoding. + + Args: + column (`str`): + Column name. + feature (`FeatureType`): + Target feature. + new_fingerprint (`str`, *optional*): + The new fingerprint of the dataset after transform. + If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments. + + Returns: + [`Dataset`] + + Example: + + ```py + >>> from datasets import load_dataset, ClassLabel + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> ds.features + {'label': ClassLabel(names=['neg', 'pos'], id=None), + 'text': Value(dtype='string', id=None)} + >>> ds = ds.cast_column('label', ClassLabel(names=['bad', 'good'])) + >>> ds.features + {'label': ClassLabel(names=['bad', 'good'], id=None), + 'text': Value(dtype='string', id=None)} + ``` + """ + if hasattr(feature, "decode_example"): + dataset = copy.deepcopy(self) + dataset._info.features[column] = feature + dataset._fingerprint = new_fingerprint + dataset._data = dataset._data.cast(dataset.features.arrow_schema) + dataset._data = update_metadata_with_features(dataset._data, dataset.features) + return dataset + else: + features = self.features + features[column] = feature + return self.cast(features) + + @transmit_format + @fingerprint_transform(inplace=False) + def remove_columns(self, column_names: Union[str, list[str]], new_fingerprint: Optional[str] = None) -> "Dataset": + """ + Remove one or several column(s) in the dataset and the features associated to them. + + You can also remove a column using [`~datasets.Dataset.map`] with `remove_columns` but the present method + doesn't copy the data of the remaining columns and is thus faster. + + Args: + column_names (`Union[str, List[str]]`): + Name of the column(s) to remove. + new_fingerprint (`str`, *optional*): + The new fingerprint of the dataset after transform. + If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments. + + Returns: + [`Dataset`]: A copy of the dataset object without the columns to remove. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> ds = ds.remove_columns('label') + Dataset({ + features: ['text'], + num_rows: 1066 + }) + >>> ds = ds.remove_columns(column_names=ds.column_names) # Removing all the columns returns an empty dataset with the `num_rows` property set to 0 + Dataset({ + features: [], + num_rows: 0 + }) + ``` + """ + dataset = copy.deepcopy(self) + if isinstance(column_names, str): + column_names = [column_names] + + missing_columns = set(column_names) - set(self._data.column_names) + if missing_columns: + raise ValueError( + f"Column name {list(missing_columns)} not in the dataset. " + f"Current columns in the dataset: {dataset._data.column_names}" + ) + + for column_name in column_names: + del dataset._info.features[column_name] + + dataset._data = dataset._data.drop(column_names) + dataset._data = update_metadata_with_features(dataset._data, dataset.features) + dataset._fingerprint = new_fingerprint + return dataset + + @fingerprint_transform(inplace=False) + def rename_column( + self, original_column_name: str, new_column_name: str, new_fingerprint: Optional[str] = None + ) -> "Dataset": + """ + Rename a column in the dataset, and move the features associated to the original column under the new column + name. + + Args: + original_column_name (`str`): + Name of the column to rename. + new_column_name (`str`): + New name for the column. + new_fingerprint (`str`, *optional*): + The new fingerprint of the dataset after transform. + If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments. + + Returns: + [`Dataset`]: A copy of the dataset with a renamed column. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> ds = ds.rename_column('label', 'label_new') + Dataset({ + features: ['text', 'label_new'], + num_rows: 1066 + }) + ``` + """ + dataset = copy.deepcopy(self) + if original_column_name not in dataset._data.column_names: + raise ValueError( + f"Original column name {original_column_name} not in the dataset. " + f"Current columns in the dataset: {dataset._data.column_names}" + ) + if new_column_name in dataset._data.column_names: + raise ValueError( + f"New column name {new_column_name} already in the dataset. " + f"Please choose a column name which is not already in the dataset. " + f"Current columns in the dataset: {dataset._data.column_names}" + ) + if not new_column_name: + raise ValueError("New column name is empty.") + + def rename(columns): + return [new_column_name if col == original_column_name else col for col in columns] + + new_column_names = rename(self._data.column_names) + if self._format_columns is not None: + dataset._format_columns = rename(self._format_columns) + + dataset._info.features = Features( + { + new_column_name if col == original_column_name else col: feature + for col, feature in self._info.features.items() + } + ) + + dataset._data = dataset._data.rename_columns(new_column_names) + dataset._data = update_metadata_with_features(dataset._data, dataset.features) + dataset._fingerprint = new_fingerprint + return dataset + + @fingerprint_transform(inplace=False) + def rename_columns(self, column_mapping: dict[str, str], new_fingerprint: Optional[str] = None) -> "Dataset": + """ + Rename several columns in the dataset, and move the features associated to the original columns under + the new column names. + + Args: + column_mapping (`Dict[str, str]`): + A mapping of columns to rename to their new names + new_fingerprint (`str`, *optional*): + The new fingerprint of the dataset after transform. + If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments. + + Returns: + [`Dataset`]: A copy of the dataset with renamed columns + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> ds = ds.rename_columns({'text': 'text_new', 'label': 'label_new'}) + Dataset({ + features: ['text_new', 'label_new'], + num_rows: 1066 + }) + ``` + """ + dataset = copy.deepcopy(self) + + extra_columns = set(column_mapping.keys()) - set(dataset.column_names) + if extra_columns: + raise ValueError( + f"Original column names {extra_columns} not in the dataset. " + f"Current columns in the dataset: {dataset._data.column_names}" + ) + + number_of_duplicates_in_new_columns = len(column_mapping.values()) - len(set(column_mapping.values())) + if number_of_duplicates_in_new_columns != 0: + raise ValueError( + "New column names must all be different, but this column mapping " + f"has {number_of_duplicates_in_new_columns} duplicates" + ) + + empty_new_columns = [new_col for new_col in column_mapping.values() if not new_col] + if empty_new_columns: + raise ValueError(f"New column names {empty_new_columns} are empty.") + + def rename(columns): + return [column_mapping[col] if col in column_mapping else col for col in columns] + + new_column_names = rename(self._data.column_names) + if self._format_columns is not None: + dataset._format_columns = rename(self._format_columns) + + dataset._info.features = Features( + { + column_mapping[col] if col in column_mapping else col: feature + for col, feature in (self._info.features or {}).items() + } + ) + + dataset._data = dataset._data.rename_columns(new_column_names) + dataset._data = update_metadata_with_features(dataset._data, dataset.features) + dataset._fingerprint = new_fingerprint + return dataset + + @transmit_format + @fingerprint_transform(inplace=False) + def select_columns(self, column_names: Union[str, list[str]], new_fingerprint: Optional[str] = None) -> "Dataset": + """Select one or several column(s) in the dataset and the features + associated to them. + + Args: + column_names (`Union[str, List[str]]`): + Name of the column(s) to keep. + new_fingerprint (`str`, *optional*): + The new fingerprint of the dataset after transform. If `None`, + the new fingerprint is computed using a hash of the previous + fingerprint, and the transform arguments. + + Returns: + [`Dataset`]: A copy of the dataset object which only consists of + selected columns. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> ds.select_columns(['text']) + Dataset({ + features: ['text'], + num_rows: 1066 + }) + ``` + """ + if isinstance(column_names, str): + column_names = [column_names] + + missing_columns = set(column_names) - set(self._data.column_names) + if missing_columns: + raise ValueError( + f"Column name {list(missing_columns)} not in the " + "dataset. Current columns in the dataset: " + f"{self._data.column_names}." + ) + + dataset = copy.deepcopy(self) + dataset._data = dataset._data.select(column_names) + dataset._info.features = Features({col: self._info.features[col] for col in dataset._data.column_names}) + dataset._data = update_metadata_with_features(dataset._data, dataset.features) + dataset._fingerprint = new_fingerprint + return dataset + + def __len__(self): + """Number of rows in the dataset. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> ds.__len__ + + ``` + """ + return self.num_rows + + def __iter__(self): + """Iterate through the examples. + + If a formatting is set with [`Dataset.set_format`] rows will be returned with the + selected format. + """ + if self._indices is None: + # Fast iteration + # Benchmark: https://gist.github.com/mariosasko/0248288a2e3a7556873969717c1fe52b (fast_iter_batch) + format_kwargs = self._format_kwargs if self._format_kwargs is not None else {} + formatter = get_formatter(self._format_type, features=self._info.features, **format_kwargs) + batch_size = config.ARROW_READER_BATCH_SIZE_IN_DATASET_ITER + for pa_subtable in table_iter(self.data, batch_size=batch_size): + for i in range(pa_subtable.num_rows): + pa_subtable_ex = pa_subtable.slice(i, 1) + formatted_output = format_table( + pa_subtable_ex, + 0, + formatter=formatter, + format_columns=self._format_columns, + output_all_columns=self._output_all_columns, + ) + yield formatted_output + else: + for i in range(self.num_rows): + yield self._getitem( + i, + ) + + def iter(self, batch_size: int, drop_last_batch: bool = False): + """Iterate through the batches of size `batch_size`. + + If a formatting is set with [`~datasets.Dataset.set_format`] rows will be returned with the + selected format. + + Args: + batch_size (:obj:`int`): size of each batch to yield. + drop_last_batch (:obj:`bool`, default `False`): Whether a last batch smaller than the batch_size should be + dropped + """ + if self._indices is None: + # Fast iteration + # Benchmark: https://gist.github.com/mariosasko/0248288a2e3a7556873969717c1fe52b (fast_iter_batch) + format_kwargs = self._format_kwargs if self._format_kwargs is not None else {} + formatter = get_formatter(self._format_type, features=self._info.features, **format_kwargs) + for pa_subtable in table_iter(self.data, batch_size=batch_size, drop_last_batch=drop_last_batch): + formatted_batch = format_table( + pa_subtable, + range(pa_subtable.num_rows), + formatter=formatter, + format_columns=self._format_columns, + output_all_columns=self._output_all_columns, + ) + yield formatted_batch + else: + num_rows = self.num_rows if not drop_last_batch else self.num_rows // batch_size * batch_size + for i in range(0, num_rows, batch_size): + yield self._getitem( + slice(i, i + batch_size), + ) + + def __repr__(self): + return f"Dataset({{\n features: {list(self._info.features.keys())},\n num_rows: {self.num_rows}\n}})" + + @property + def format(self): + return { + "type": self._format_type, + "format_kwargs": self._format_kwargs, + "columns": self.column_names if self._format_columns is None else self._format_columns, + "output_all_columns": self._output_all_columns, + } + + @contextlib.contextmanager + def formatted_as( + self, + type: Optional[str] = None, + columns: Optional[list] = None, + output_all_columns: bool = False, + **format_kwargs, + ): + """To be used in a `with` statement. Set `__getitem__` return format (type and columns). + + Args: + type (`str`, *optional*): + Either output type selected in `[None, 'numpy', 'torch', 'tensorflow', 'jax', 'arrow', 'pandas', 'polars']`. + `None` means `__getitem__`` returns python objects (default). + columns (`List[str]`, *optional*): + Columns to format in the output. + `None` means `__getitem__` returns all columns (default). + output_all_columns (`bool`, defaults to `False`): + Keep un-formatted columns as well in the output (as python objects). + **format_kwargs (additional keyword arguments): + Keywords arguments passed to the convert function like `np.array`, `torch.tensor` or `tensorflow.ragged.constant`. + """ + old_format_type = self._format_type + old_format_kwargs = self._format_kwargs + old_format_columns = self._format_columns + old_output_all_columns = self._output_all_columns + try: + self.set_format(type, columns, output_all_columns, **format_kwargs) + yield + finally: + self.set_format(old_format_type, old_format_columns, old_output_all_columns, **old_format_kwargs) + + @fingerprint_transform(inplace=True) + def set_format( + self, + type: Optional[str] = None, + columns: Optional[list] = None, + output_all_columns: bool = False, + **format_kwargs, + ): + """Set `__getitem__` return format (type and columns). The data formatting is applied on-the-fly. + The format `type` (for example "numpy") is used to format batches when using `__getitem__`. + It's also possible to use custom transforms for formatting using [`~datasets.Dataset.set_transform`]. + + Args: + type (`str`, *optional*): + Either output type selected in `[None, 'numpy', 'torch', 'tensorflow', 'jax', 'arrow', 'pandas', 'polars']`. + `None` means `__getitem__` returns python objects (default). + columns (`List[str]`, *optional*): + Columns to format in the output. + `None` means `__getitem__` returns all columns (default). + output_all_columns (`bool`, defaults to `False`): + Keep un-formatted columns as well in the output (as python objects). + **format_kwargs (additional keyword arguments): + Keywords arguments passed to the convert function like `np.array`, `torch.tensor` or `tensorflow.ragged.constant`. + + It is possible to call [`~datasets.Dataset.map`] after calling `set_format`. Since `map` may add new columns, then the list of formatted columns + gets updated. In this case, if you apply `map` on a dataset to add a new column, then this column will be formatted as: + + ``` + new formatted columns = (all columns - previously unformatted columns) + ``` + + Example: + + ```py + >>> from datasets import load_dataset + >>> from transformers import AutoTokenizer + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-cased") + >>> ds = ds.map(lambda x: tokenizer(x['text'], truncation=True, padding=True), batched=True) + >>> ds.set_format(type='numpy', columns=['text', 'label']) + >>> ds.format + {'type': 'numpy', + 'format_kwargs': {}, + 'columns': ['text', 'label'], + 'output_all_columns': False} + ``` + """ + format_kwargs.update(format_kwargs.pop("format_kwargs", {})) # allow to use self.set_format(**self.format) + + # Check that the format_type and format_kwargs are valid and make it possible to have a Formatter + type = get_format_type_from_alias(type) + get_formatter(type, features=self._info.features, **format_kwargs) + + # Check filter column + if isinstance(columns, str): + columns = [columns] + if isinstance(columns, tuple): + columns = list(columns) + if columns is not None: + missing_columns = set(columns) - set(self._data.column_names) + if missing_columns: + raise ValueError( + f"Columns {list(missing_columns)} not in the dataset. Current columns in the dataset: {self._data.column_names}" + ) + if columns is not None: + columns = columns.copy() # Ensures modifications made to the list after this call don't cause bugs + + self._format_type = type + self._format_kwargs = format_kwargs + self._format_columns = columns + self._output_all_columns = output_all_columns + logger.debug( + "Set __getitem__(key) output type to %s for %s columns " + " (when key is int or slice) and %s output other (un-formatted) columns.", + "python objects" if type is None else type, + "no" if columns is None else str(columns), + "do" if output_all_columns else "don't", + ) + + def reset_format(self): + """Reset `__getitem__` return format to python objects and all columns. + + Same as `self.set_format()` + + Example: + + ```py + >>> from datasets import load_dataset + >>> from transformers import AutoTokenizer + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-cased") + >>> ds = ds.map(lambda x: tokenizer(x['text'], truncation=True, padding=True), batched=True) + >>> ds.set_format(type='numpy', columns=['input_ids', 'token_type_ids', 'attention_mask', 'label']) + >>> ds.format + {'columns': ['input_ids', 'token_type_ids', 'attention_mask', 'label'], + 'format_kwargs': {}, + 'output_all_columns': False, + 'type': 'numpy'} + >>> ds.reset_format() + >>> ds.format + {'columns': ['text', 'label', 'input_ids', 'token_type_ids', 'attention_mask'], + 'format_kwargs': {}, + 'output_all_columns': False, + 'type': None} + ``` + """ + self.set_format() + + def set_transform( + self, + transform: Optional[Callable], + columns: Optional[list] = None, + output_all_columns: bool = False, + ): + """Set `__getitem__` return format using this transform. The transform is applied on-the-fly on batches when `__getitem__` is called. + As [`~datasets.Dataset.set_format`], this can be reset using [`~datasets.Dataset.reset_format`]. + + Args: + transform (`Callable`, *optional*): + User-defined formatting transform, replaces the format defined by [`~datasets.Dataset.set_format`]. + A formatting function is a callable that takes a batch (as a `dict`) as input and returns a batch. + This function is applied right before returning the objects in `__getitem__`. + columns (`List[str]`, *optional*): + Columns to format in the output. + If specified, then the input batch of the transform only contains those columns. + output_all_columns (`bool`, defaults to `False`): + Keep un-formatted columns as well in the output (as python objects). + If set to True, then the other un-formatted columns are kept with the output of the transform. + + Example: + + ```py + >>> from datasets import load_dataset + >>> from transformers import AutoTokenizer + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> tokenizer = AutoTokenizer.from_pretrained('bert-base-uncased') + >>> def encode(batch): + ... return tokenizer(batch['text'], padding=True, truncation=True, return_tensors='pt') + >>> ds.set_transform(encode) + >>> ds[0] + {'attention_mask': tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1]), + 'input_ids': tensor([ 101, 29353, 2135, 15102, 1996, 9428, 20868, 2890, 8663, 6895, + 20470, 2571, 3663, 2090, 4603, 3017, 3008, 1998, 2037, 24211, + 5637, 1998, 11690, 2336, 1012, 102]), + 'token_type_ids': tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0])} + ``` + """ + self.set_format("custom", columns=columns, output_all_columns=output_all_columns, transform=transform) + + def with_format( + self, + type: Optional[str] = None, + columns: Optional[list] = None, + output_all_columns: bool = False, + **format_kwargs, + ): + """Set `__getitem__` return format (type and columns). The data formatting is applied on-the-fly. + The format `type` (for example "numpy") is used to format batches when using `__getitem__`. + + It's also possible to use custom transforms for formatting using [`~datasets.Dataset.with_transform`]. + + Contrary to [`~datasets.Dataset.set_format`], `with_format` returns a new [`Dataset`] object. + + Args: + type (`str`, *optional*): + Either output type selected in `[None, 'numpy', 'torch', 'tensorflow', 'jax', 'arrow', 'pandas', 'polars']`. + `None` means `__getitem__` returns python objects (default). + columns (`List[str]`, *optional*): + Columns to format in the output. + `None` means `__getitem__` returns all columns (default). + output_all_columns (`bool`, defaults to `False`): + Keep un-formatted columns as well in the output (as python objects). + **format_kwargs (additional keyword arguments): + Keywords arguments passed to the convert function like `np.array`, `torch.tensor` or `tensorflow.ragged.constant`. + + Example: + + ```py + >>> from datasets import load_dataset + >>> from transformers import AutoTokenizer + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-cased") + >>> ds = ds.map(lambda x: tokenizer(x['text'], truncation=True, padding=True), batched=True) + >>> ds.format + {'columns': ['text', 'label', 'input_ids', 'token_type_ids', 'attention_mask'], + 'format_kwargs': {}, + 'output_all_columns': False, + 'type': None} + >>> ds = ds.with_format("torch") + >>> ds.format + {'columns': ['text', 'label', 'input_ids', 'token_type_ids', 'attention_mask'], + 'format_kwargs': {}, + 'output_all_columns': False, + 'type': 'torch'} + >>> ds[0] + {'text': 'compassionately explores the seemingly irreconcilable situation between conservative christian parents and their estranged gay and lesbian children .', + 'label': tensor(1), + 'input_ids': tensor([ 101, 18027, 16310, 16001, 1103, 9321, 178, 11604, 7235, 6617, + 1742, 2165, 2820, 1206, 6588, 22572, 12937, 1811, 2153, 1105, + 1147, 12890, 19587, 6463, 1105, 15026, 1482, 119, 102, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0]), + 'token_type_ids': tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]), + 'attention_mask': tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])} + ``` + """ + dataset = copy.deepcopy(self) + dataset.set_format(type=type, columns=columns, output_all_columns=output_all_columns, **format_kwargs) + return dataset + + def with_transform( + self, + transform: Optional[Callable], + columns: Optional[list] = None, + output_all_columns: bool = False, + ): + """Set `__getitem__` return format using this transform. The transform is applied on-the-fly on batches when `__getitem__` is called. + + As [`~datasets.Dataset.set_format`], this can be reset using [`~datasets.Dataset.reset_format`]. + + Contrary to [`~datasets.Dataset.set_transform`], `with_transform` returns a new [`Dataset`] object. + + Args: + transform (`Callable`, `optional`): + User-defined formatting transform, replaces the format defined by [`~datasets.Dataset.set_format`]. + A formatting function is a callable that takes a batch (as a `dict`) as input and returns a batch. + This function is applied right before returning the objects in `__getitem__`. + columns (`List[str]`, `optional`): + Columns to format in the output. + If specified, then the input batch of the transform only contains those columns. + output_all_columns (`bool`, defaults to `False`): + Keep un-formatted columns as well in the output (as python objects). + If set to `True`, then the other un-formatted columns are kept with the output of the transform. + + Example: + + ```py + >>> from datasets import load_dataset + >>> from transformers import AutoTokenizer + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-cased") + >>> def encode(example): + ... return tokenizer(example["text"], padding=True, truncation=True, return_tensors='pt') + >>> ds = ds.with_transform(encode) + >>> ds[0] + {'attention_mask': tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1]), + 'input_ids': tensor([ 101, 18027, 16310, 16001, 1103, 9321, 178, 11604, 7235, 6617, + 1742, 2165, 2820, 1206, 6588, 22572, 12937, 1811, 2153, 1105, + 1147, 12890, 19587, 6463, 1105, 15026, 1482, 119, 102]), + 'token_type_ids': tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0])} + ``` + """ + dataset = copy.deepcopy(self) + dataset.set_transform(transform=transform, columns=columns, output_all_columns=output_all_columns) + return dataset + + def _getitem(self, key: Union[int, slice, str, ListLike[int]], **kwargs) -> Union[dict, list]: + """ + Can be used to index columns (by string names) or rows (by integer, slice, or list-like of integer indices) + """ + if isinstance(key, bool): + raise TypeError("dataset index must be int, str, slice or collection of int, not bool") + format_type = kwargs["format_type"] if "format_type" in kwargs else self._format_type + format_columns = kwargs["format_columns"] if "format_columns" in kwargs else self._format_columns + output_all_columns = ( + kwargs["output_all_columns"] if "output_all_columns" in kwargs else self._output_all_columns + ) + format_kwargs = kwargs["format_kwargs"] if "format_kwargs" in kwargs else self._format_kwargs + format_kwargs = format_kwargs if format_kwargs is not None else {} + formatter = get_formatter(format_type, features=self._info.features, **format_kwargs) + pa_subtable = query_table(self._data, key, indices=self._indices) + formatted_output = format_table( + pa_subtable, key, formatter=formatter, format_columns=format_columns, output_all_columns=output_all_columns + ) + return formatted_output + + @overload + def __getitem__(self, key: Union[int, slice, Iterable[int]]) -> dict: # noqa: F811 + ... + + @overload + def __getitem__(self, key: str) -> list: # noqa: F811 + ... + + def __getitem__(self, key): # noqa: F811 + """Can be used to index columns (by string names) or rows (by integer index or iterable of indices or bools).""" + return self._getitem(key) + + def __getitems__(self, keys: list) -> list: + """Can be used to get a batch using a list of integers indices.""" + batch = self.__getitem__(keys) + n_examples = len(batch[next(iter(batch))]) + return [{col: array[i] for col, array in batch.items()} for i in range(n_examples)] + + def cleanup_cache_files(self) -> int: + """Clean up all cache files in the dataset cache directory, excepted the currently used cache file if there is + one. + + Be careful when running this command that no other process is currently using other cache files. + + Returns: + `int`: Number of removed files. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> ds.cleanup_cache_files() + 10 + ``` + """ + current_cache_files = [os.path.abspath(cache_file["filename"]) for cache_file in self.cache_files] + if not current_cache_files: + return 0 + cache_directory = os.path.dirname(current_cache_files[0]) + logger.info(f"Listing files in {cache_directory}") + files: list[str] = os.listdir(cache_directory) + files_to_remove = [] + for f_name in files: + full_name = os.path.abspath(os.path.join(cache_directory, f_name)) + if f_name.startswith("cache-") and f_name.endswith(".arrow"): + if full_name in current_cache_files: + logger.info(f"Keeping currently used cache file at {full_name}") + continue + files_to_remove.append(full_name) + for file_path in files_to_remove: + logger.info(f"Removing {file_path}") + os.remove(file_path) + return len(files_to_remove) + + def _get_cache_file_path(self, fingerprint): + if is_caching_enabled() and self.cache_files: + cache_file_name = "cache-" + fingerprint + ".arrow" + cache_directory = os.path.dirname(self.cache_files[0]["filename"]) + else: + cache_file_name = "cache-" + generate_random_fingerprint() + ".arrow" + cache_directory = get_temporary_cache_files_directory() + cache_file_path = os.path.join(cache_directory, cache_file_name) + return cache_file_path + + @transmit_format + def map( + self, + function: Optional[Callable] = None, + with_indices: bool = False, + with_rank: bool = False, + input_columns: Optional[Union[str, list[str]]] = None, + batched: bool = False, + batch_size: Optional[int] = 1000, + drop_last_batch: bool = False, + remove_columns: Optional[Union[str, list[str]]] = None, + keep_in_memory: bool = False, + load_from_cache_file: Optional[bool] = None, + cache_file_name: Optional[str] = None, + writer_batch_size: Optional[int] = 1000, + features: Optional[Features] = None, + disable_nullable: bool = False, + fn_kwargs: Optional[dict] = None, + num_proc: Optional[int] = None, + suffix_template: str = "_{rank:05d}_of_{num_proc:05d}", + new_fingerprint: Optional[str] = None, + desc: Optional[str] = None, + try_original_type: Optional[bool] = True, + ) -> "Dataset": + """ + Apply a function to all the examples in the table (individually or in batches) and update the table. + If your function returns a column that already exists, then it overwrites it. + + You can specify whether the function should be batched or not with the `batched` parameter: + + - If batched is `False`, then the function takes 1 example in and should return 1 example. + An example is a dictionary, e.g. `{"text": "Hello there !"}`. + - If batched is `True` and `batch_size` is 1, then the function takes a batch of 1 example as input and can return a batch with 1 or more examples. + A batch is a dictionary, e.g. a batch of 1 example is `{"text": ["Hello there !"]}`. + - If batched is `True` and `batch_size` is `n > 1`, then the function takes a batch of `n` examples as input and can return a batch with `n` examples, or with an arbitrary number of examples. + Note that the last batch may have less than `n` examples. + A batch is a dictionary, e.g. a batch of `n` examples is `{"text": ["Hello there !"] * n}`. + + If the function is asynchronous, then `map` will run your function in parallel, with up to one thousand simultaneous calls. + It is recommended to use a `asyncio.Semaphore` in your function if you want to set a maximum number of operations that can run at the same time. + + Args: + function (`Callable`): Function with one of the following signatures: + + - `function(example: Dict[str, Any]) -> Dict[str, Any]` if `batched=False` and `with_indices=False` and `with_rank=False` + - `function(example: Dict[str, Any], *extra_args) -> Dict[str, Any]` if `batched=False` and `with_indices=True` and/or `with_rank=True` (one extra arg for each) + - `function(batch: Dict[str, List]) -> Dict[str, List]` if `batched=True` and `with_indices=False` and `with_rank=False` + - `function(batch: Dict[str, List], *extra_args) -> Dict[str, List]` if `batched=True` and `with_indices=True` and/or `with_rank=True` (one extra arg for each) + + For advanced usage, the function can also return a `pyarrow.Table`. + If the function is asynchronous, then `map` will run your function in parallel. + Moreover if your function returns nothing (`None`), then `map` will run your function and return the dataset unchanged. + If no function is provided, default to identity function: `lambda x: x`. + with_indices (`bool`, defaults to `False`): + Provide example indices to `function`. Note that in this case the + signature of `function` should be `def function(example, idx[, rank]): ...`. + with_rank (`bool`, defaults to `False`): + Provide process rank to `function`. Note that in this case the + signature of `function` should be `def function(example[, idx], rank): ...`. + input_columns (`Optional[Union[str, List[str]]]`, defaults to `None`): + The columns to be passed into `function` + as positional arguments. If `None`, a `dict` mapping to all formatted columns is passed as one argument. + batched (`bool`, defaults to `False`): + Provide batch of examples to `function`. + batch_size (`int`, *optional*, defaults to `1000`): + Number of examples per batch provided to `function` if `batched=True`. + If `batch_size <= 0` or `batch_size == None`, provide the full dataset as a single batch to `function`. + drop_last_batch (`bool`, defaults to `False`): + Whether a last batch smaller than the batch_size should be + dropped instead of being processed by the function. + remove_columns (`Optional[Union[str, List[str]]]`, defaults to `None`): + Remove a selection of columns while doing the mapping. + Columns will be removed before updating the examples with the output of `function`, i.e. if `function` is adding + columns with names in `remove_columns`, these columns will be kept. + keep_in_memory (`bool`, defaults to `False`): + Keep the dataset in memory instead of writing it to a cache file. + load_from_cache_file (`Optional[bool]`, defaults to `True` if caching is enabled): + If a cache file storing the current computation from `function` + can be identified, use it instead of recomputing. + cache_file_name (`str`, *optional*, defaults to `None`): + Provide the name of a path for the cache file. It is used to store the + results of the computation instead of the automatically generated cache file name. + writer_batch_size (`int`, defaults to `1000`): + Number of rows per write operation for the cache file writer. + This value is a good trade-off between memory usage during the processing, and processing speed. + Higher value makes the processing do fewer lookups, lower value consume less temporary memory while running `map`. + features (`Optional[datasets.Features]`, defaults to `None`): + Use a specific Features to store the cache file + instead of the automatically generated one. + disable_nullable (`bool`, defaults to `False`): + Disallow null values in the table. + fn_kwargs (`Dict`, *optional*, defaults to `None`): + Keyword arguments to be passed to `function`. + num_proc (`int`, *optional*, defaults to `None`): + Max number of processes when generating cache. Already cached shards are loaded sequentially. + suffix_template (`str`): + If `cache_file_name` is specified, then this suffix + will be added at the end of the base name of each. Defaults to `"_{rank:05d}_of_{num_proc:05d}"`. For example, if `cache_file_name` is "processed.arrow", then for + `rank=1` and `num_proc=4`, the resulting file would be `"processed_00001_of_00004.arrow"` for the default suffix. + new_fingerprint (`str`, *optional*, defaults to `None`): + The new fingerprint of the dataset after transform. + If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments. + desc (`str`, *optional*, defaults to `None`): + Meaningful description to be displayed alongside with the progress bar while mapping examples. + try_original_type (`Optional[bool]`, defaults to `True`): + Try to keep the types of the original columns (e.g. int32 -> int32). + Set to False if you want to always infer new types. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> def add_prefix(example): + ... example["text"] = "Review: " + example["text"] + ... return example + >>> ds = ds.map(add_prefix) + >>> ds[0:3]["text"] + ['Review: compassionately explores the seemingly irreconcilable situation between conservative christian parents and their estranged gay and lesbian children .', + 'Review: the soundtrack alone is worth the price of admission .', + 'Review: rodriguez does a splendid job of racial profiling hollywood style--casting excellent latin actors of all ages--a trend long overdue .'] + + # process a batch of examples + >>> ds = ds.map(lambda example: tokenizer(example["text"]), batched=True) + # set number of processors + >>> ds = ds.map(add_prefix, num_proc=4) + ``` + """ + if keep_in_memory and cache_file_name is not None: + raise ValueError("Please use either `keep_in_memory` or `cache_file_name` but not both.") + + if num_proc is not None and num_proc <= 0: + raise ValueError("num_proc must be an integer > 0.") + + # If the array is empty we do nothing (but we make sure to handle an empty indices mapping and remove the requested columns anyway) + if len(self) == 0: + if self._indices is not None: # empty indices mapping + self = Dataset( + self.data.slice(0, 0), + info=self.info.copy(), + split=self.split, + fingerprint=new_fingerprint, + ) + if remove_columns: + return self.remove_columns(remove_columns) + else: + return self + + if function is None: + function = lambda x: x # noqa: E731 + + if isinstance(input_columns, str): + input_columns = [input_columns] + + if input_columns is not None: + missing_columns = set(input_columns) - set(self._data.column_names) + if missing_columns: + raise ValueError( + f"Input column {list(missing_columns)} not in the dataset. Current columns in the dataset: {self._data.column_names}" + ) + + if isinstance(remove_columns, str): + remove_columns = [remove_columns] + + if remove_columns is not None: + missing_columns = set(remove_columns) - set(self._data.column_names) + if missing_columns: + raise ValueError( + f"Column to remove {list(missing_columns)} not in the dataset. Current columns in the dataset: {self._data.column_names}" + ) + + load_from_cache_file = load_from_cache_file if load_from_cache_file is not None else is_caching_enabled() + + if fn_kwargs is None: + fn_kwargs = {} + + if num_proc is not None and num_proc > len(self): + num_proc = len(self) + logger.warning( + f"num_proc must be <= {len(self)}. Reducing num_proc to {num_proc} for dataset of size {len(self)}." + ) + + dataset_kwargs = { + "shard": self, + "function": function, + "with_indices": with_indices, + "with_rank": with_rank, + "input_columns": input_columns, + "batched": batched, + "batch_size": batch_size, + "drop_last_batch": drop_last_batch, + "remove_columns": remove_columns, + "keep_in_memory": keep_in_memory, + "writer_batch_size": writer_batch_size, + "features": features, + "disable_nullable": disable_nullable, + "fn_kwargs": fn_kwargs, + "try_original_type": try_original_type, + } + + if new_fingerprint is None: + # we create a unique hash from the function, + # current dataset file and the mapping args + transform = format_transform_for_fingerprint(Dataset._map_single) + kwargs_for_fingerprint = format_kwargs_for_fingerprint(Dataset._map_single, (), dataset_kwargs) + kwargs_for_fingerprint["fingerprint_name"] = "new_fingerprint" + new_fingerprint = update_fingerprint(self._fingerprint, transform, kwargs_for_fingerprint) + else: + validate_fingerprint(new_fingerprint) + dataset_kwargs["new_fingerprint"] = new_fingerprint + + if self.cache_files: + if cache_file_name is None: + cache_file_name = self._get_cache_file_path(new_fingerprint) + dataset_kwargs["cache_file_name"] = cache_file_name + + def load_processed_shard_from_cache(shard_kwargs): + """Load a processed shard from cache if it exists, otherwise throw an error.""" + shard = shard_kwargs["shard"] + # Check if we've already cached this computation (indexed by a hash) + if shard_kwargs["cache_file_name"] is not None: + if os.path.exists(shard_kwargs["cache_file_name"]) and load_from_cache_file: + info = shard.info.copy() + info.features = features + return Dataset.from_file(shard_kwargs["cache_file_name"], info=info, split=shard.split) + raise NonExistentDatasetError + + num_shards = num_proc if num_proc is not None else 1 + if batched and drop_last_batch: + pbar_total = len(self) // num_shards // batch_size * num_shards * batch_size + else: + pbar_total = len(self) + + shards_done = 0 + if num_proc is None or num_proc == 1: + transformed_dataset = None + try: + transformed_dataset = load_processed_shard_from_cache(dataset_kwargs) + logger.info(f"Loading cached processed dataset at {dataset_kwargs['cache_file_name']}") + except NonExistentDatasetError: + pass + if transformed_dataset is None: + with hf_tqdm( + unit=" examples", + total=pbar_total, + desc=desc or "Map", + ) as pbar: + for rank, done, content in Dataset._map_single(**dataset_kwargs): + if done: + shards_done += 1 + logger.debug(f"Finished processing shard number {rank} of {num_shards}.") + transformed_dataset = content + else: + pbar.update(content) + assert transformed_dataset is not None, "Failed to retrieve the result from map" + # update fingerprint if the dataset changed + if transformed_dataset._fingerprint != self._fingerprint: + transformed_dataset._fingerprint = new_fingerprint + return transformed_dataset + else: + + def format_cache_file_name( + cache_file_name: Optional[str], + rank: Union[int, Literal["*"]], # noqa: F722 + ) -> Optional[str]: + if not cache_file_name: + return cache_file_name + sep = cache_file_name.rindex(".") + base_name, extension = cache_file_name[:sep], cache_file_name[sep:] + if isinstance(rank, int): + cache_file_name = base_name + suffix_template.format(rank=rank, num_proc=num_proc) + extension + logger.info(f"Process #{rank} will write at {cache_file_name}") + else: + cache_file_name = ( + base_name + + suffix_template.replace("{rank:05d}", "{rank}").format(rank=rank, num_proc=num_proc) + + extension + ) + return cache_file_name + + def format_new_fingerprint(new_fingerprint: str, rank: int) -> str: + new_fingerprint = new_fingerprint + suffix_template.format(rank=rank, num_proc=num_proc) + validate_fingerprint(new_fingerprint) + return new_fingerprint + + prev_env = deepcopy(os.environ) + # check if parallelism if off + # from https://github.com/huggingface/tokenizers/blob/bb668bc439dc34389b71dbb8ce0c597f15707b53/tokenizers/src/utils/parallelism.rs#L22 + if prev_env.get("TOKENIZERS_PARALLELISM", "false").lower() not in ( + "", + "off", + "false", + "f", + "no", + "n", + "0", + ): + logger.warning("Setting TOKENIZERS_PARALLELISM=false for forked processes.") + os.environ["TOKENIZERS_PARALLELISM"] = "false" + shards = [ + self.shard(num_shards=num_proc, index=rank, contiguous=True, keep_in_memory=keep_in_memory) + for rank in range(num_proc) + ] + kwargs_per_job = [ + { + **dataset_kwargs, + "shard": shards[rank], + "cache_file_name": format_cache_file_name(cache_file_name, rank), + "rank": rank, + "offset": sum(len(s) for s in shards[:rank]), + "new_fingerprint": format_new_fingerprint(new_fingerprint, rank), + } + for rank in range(num_shards) + ] + + transformed_shards = [None] * num_shards + for rank in range(num_shards): + try: + transformed_shards[rank] = load_processed_shard_from_cache(kwargs_per_job[rank]) + kwargs_per_job[rank] = None + except NonExistentDatasetError: + pass + + kwargs_per_job = [kwargs for kwargs in kwargs_per_job if kwargs is not None] + + # We try to create a pool with as many workers as dataset not yet cached. + if kwargs_per_job: + if len(kwargs_per_job) < num_shards: + logger.info( + f"Reprocessing {len(kwargs_per_job)}/{num_shards} shards because some of them were missing from the cache." + ) + with Pool(len(kwargs_per_job)) as pool: + os.environ = prev_env + logger.info(f"Spawning {num_proc} processes") + with hf_tqdm( + unit=" examples", + total=pbar_total, + desc=(desc or "Map") + f" (num_proc={num_proc})", + ) as pbar: + for rank, done, content in iflatmap_unordered( + pool, Dataset._map_single, kwargs_iterable=kwargs_per_job + ): + if done: + shards_done += 1 + logger.debug(f"Finished processing shard number {rank} of {num_shards}.") + transformed_shards[rank] = content + else: + pbar.update(content) + pool.close() + pool.join() + # Avoids PermissionError on Windows (the error: https://github.com/huggingface/datasets/actions/runs/4026734820/jobs/6921621805) + for kwargs in kwargs_per_job: + del kwargs["shard"] + else: + logger.info(f"Loading cached processed dataset at {format_cache_file_name(cache_file_name, '*')}") + if None in transformed_shards: + raise ValueError( + f"Failed to retrieve results from map: result list {transformed_shards} still contains None - at " + "least one worker failed to return its results" + ) + logger.info(f"Concatenating {num_proc} shards") + result = _concatenate_map_style_datasets(transformed_shards) + # update fingerprint if the dataset changed + if any( + transformed_shard._fingerprint != shard._fingerprint + for transformed_shard, shard in zip(transformed_shards, shards) + ): + result._fingerprint = new_fingerprint + else: + result._fingerprint = self._fingerprint + return result + + @staticmethod + def _map_single( + shard: "Dataset", + function: Optional[Callable] = None, + with_indices: bool = False, + with_rank: bool = False, + input_columns: Optional[list[str]] = None, + batched: bool = False, + batch_size: Optional[int] = 1000, + drop_last_batch: bool = False, + remove_columns: Optional[list[str]] = None, + keep_in_memory: bool = False, + cache_file_name: Optional[str] = None, + writer_batch_size: Optional[int] = 1000, + features: Optional[Features] = None, + disable_nullable: bool = False, + fn_kwargs: Optional[dict] = None, + new_fingerprint: Optional[str] = None, + rank: Optional[int] = None, + offset: int = 0, + try_original_type: Optional[bool] = True, + ) -> Iterable[tuple[int, bool, Union[int, "Dataset"]]]: + """Apply a function to all the elements in the table (individually or in batches) + and update the table (if function does update examples). + + Args: + shard (`datasets.Dataset`): Dataset to map the transform on. + function (`Callable`): with one of the following signature: + - `function(example: Dict[str, Any]) -> Dict[str, Any]` if `batched=False` and `with_indices=False` and `with_rank=False` + - `function(example: Dict[str, Any], *extra_args) -> Dict[str, Any]` if `batched=False` and `with_indices=True` and/or `with_rank=True` (one extra arg for each) + - `function(batch: Dict[str, List]) -> Dict[str, List]` if `batched=True` and `with_indices=False` and `with_rank=False` + - `function(batch: Dict[str, List], *extra_args) -> Dict[str, List]` if `batched=True` and `with_indices=True` and/or `with_rank=True` (one extra arg for each) + + For advanced usage, the function can also return a `pyarrow.Table`. + Moreover if your function returns nothing (`None`), then `map` will run your function and return the dataset unchanged. + If no function is provided, default to identity function: lambda x: x + with_indices (`bool`, defaults to `False`): Provide example indices to `function`. Note that in this case the signature of `function` should be `def function(example, idx[, rank]): ...`. + with_rank (`bool`, default `False`): Provide process rank to `function`. Note that in this case the signature of `function` should be `def function(example[, idx], rank): ...`. + input_columns (`Optional[List[str]]`, defaults to `None`): The columns to be passed into `function` as + positional arguments. If `None`, a dict mapping to all formatted columns is passed as one argument. + batched (`bool`, defaults to `False`): Provide batch of examples to `function` + batch_size (`int`, optional, defaults to `1000`): Number of examples per batch provided to `function` if `batched=True` + `batch_size <= 0` or `batch_size == None`: Provide the full dataset as a single batch to `function` + drop_last_batch (`bool`, default: `False`): Whether a last batch smaller than the batch_size should be + dropped instead of being processed by the function. + remove_columns (`Optional[List[str]]`, defaults to `None`): Remove a selection of columns while doing the mapping. + Columns will be removed before updating the examples with the output of `function`, i.e. if `function` is adding + columns with names in `remove_columns`, these columns will be kept. + keep_in_memory (`bool`, defaults to `False`): Keep the dataset in memory instead of writing it to a cache file. + cache_file_name (`str`, optional, defaults to `None`): Provide the name of a path for the cache file. It is used to store the + results of the computation instead of the automatically generated cache file name. + writer_batch_size (`int`, default `1000`): Number of rows per write operation for the cache file writer. + This value is a good trade-off between memory usage during the processing, and processing speed. + Higher value makes the processing do fewer lookups, lower value consume less temporary memory while running `.map()`. + features (`Optional[datasets.Features]`, defaults to `None`): Use a specific Features to store the cache file + instead of the automatically generated one. + disable_nullable (`bool`, defaults to `False`): Disallow null values in the table. + fn_kwargs (`Dict`, optional, defaults to `None`): Keyword arguments to be passed to `function` + new_fingerprint (`str`, optional, defaults to `None`): the new fingerprint of the dataset after transform. + If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments + rank: (`int`, optional, defaults to `None`): If specified, this is the process rank when doing multiprocessing + offset: (`int`, defaults to 0): If specified, this is an offset applied to the indices passed to `function` if `with_indices=True`. + try_original_type: (`Optional[bool]`, defaults to `True`): + Try to keep the types of the original columns (e.g. int32 -> int32). + Set to False if you want to always infer new types. + """ + if fn_kwargs is None: + fn_kwargs = {} + + # If we do batch computation but no batch size is provided, default to the full dataset + if batched and (batch_size is None or batch_size <= 0): + batch_size = shard.num_rows + + # We set this variable to True after processing the first example/batch in + # `apply_function_on_filtered_inputs` if the map function returns a dict. + # If set to False, no new arrow table will be created + + update_data = None + + format_kwargs = shard._format_kwargs.copy() + # Lazy formatting is only available for the default format (None/python) + if not input_columns and shard._format_type is None: + format_kwargs["lazy"] = True + input_formatter = get_formatter( + shard._format_type, + features=shard.features, + **format_kwargs, + ) + + check_same_num_examples = batched and len(shard.list_indexes()) > 0 + + def validate_function_output(processed_inputs): + """Validate output of the map function.""" + allowed_processed_inputs_types = (Mapping, pa.Table, pd.DataFrame) + if config.POLARS_AVAILABLE and "polars" in sys.modules: + import polars as pl + + allowed_processed_inputs_types += (pl.DataFrame,) + if processed_inputs is not None and not isinstance(processed_inputs, allowed_processed_inputs_types): + raise TypeError( + f"Provided `function` which is applied to all elements of table returns a variable of type {type(processed_inputs)}. Make sure provided `function` returns a variable of type `dict` (or a pyarrow table) to update the dataset or `None` if you are only interested in side effects." + ) + if batched and isinstance(processed_inputs, Mapping): + allowed_batch_return_types = (list, np.ndarray, pd.Series) + if config.POLARS_AVAILABLE and "polars" in sys.modules: + import polars as pl + + allowed_batch_return_types += (pl.Series, pl.DataFrame) + if config.TF_AVAILABLE and "tensorflow" in sys.modules: + import tensorflow as tf + + allowed_batch_return_types += (tf.Tensor,) + if config.TORCH_AVAILABLE and "torch" in sys.modules: + import torch + + allowed_batch_return_types += (torch.Tensor,) + if config.JAX_AVAILABLE and "jax" in sys.modules: + import jax.numpy as jnp + + allowed_batch_return_types += (jnp.ndarray,) + all_dict_values_are_lists = all( + isinstance(value, allowed_batch_return_types) for value in processed_inputs.values() + ) + if all_dict_values_are_lists is False: + raise TypeError( + f"Provided `function` which is applied to all elements of table returns a `dict` of types {[type(x) for x in processed_inputs.values()]}. When using `batched=True`, make sure provided `function` returns a `dict` of types like `{allowed_batch_return_types}`." + ) + + def prepare_inputs(pa_inputs, indices, offset=0): + """Utility to apply the function on a selection of columns.""" + inputs = format_table( + pa_inputs, + 0 if not batched else range(pa_inputs.num_rows), + format_columns=input_columns, + formatter=input_formatter, + ) + fn_args = [inputs] if input_columns is None else [inputs[col] for col in input_columns] + if offset == 0: + effective_indices = indices + else: + effective_indices = [i + offset for i in indices] if isinstance(indices, list) else indices + offset + additional_args = () + if with_indices: + additional_args += (effective_indices,) + if with_rank: + additional_args += (rank,) + return inputs, fn_args, additional_args, fn_kwargs + + def prepare_outputs(pa_inputs, inputs, processed_inputs): + nonlocal update_data + if not (update_data := (processed_inputs is not None)): + return None + if isinstance(processed_inputs, LazyDict): + processed_inputs = { + k: v for k, v in processed_inputs.data.items() if k not in processed_inputs.keys_to_format + } + returned_lazy_dict = True + else: + returned_lazy_dict = False + validate_function_output(processed_inputs) + if shard._format_type or input_columns: + # TODO(QL, MS): ideally the behavior should be the same even if the dataset is formatted (may require major release) + inputs_to_merge = dict(zip(pa_inputs.column_names, pa_inputs.itercolumns())) + elif isinstance(inputs, LazyDict): + inputs_to_merge = { + k: (v if k not in inputs.keys_to_format else pa_inputs[k]) for k, v in inputs.data.items() + } + else: + inputs_to_merge = inputs + if remove_columns is not None: + for column in remove_columns: + # `function` can modify input in-place causing column to be already removed. + if column in inputs_to_merge: + inputs_to_merge.pop(column) + if returned_lazy_dict and column in processed_inputs: + processed_inputs.pop(column) + if check_same_num_examples: + input_num_examples = len(pa_inputs) + processed_inputs_num_examples = len(processed_inputs[next(iter(processed_inputs.keys()))]) + if input_num_examples != processed_inputs_num_examples: + raise DatasetTransformationNotAllowedError( + "Using `.map` in batched mode on a dataset with attached indexes is allowed only if it doesn't create or remove existing examples. You can first run `.drop_index() to remove your index and then re-add it." + ) from None + if isinstance(inputs, Mapping) and isinstance(processed_inputs, Mapping): + # The .map() transform *updates* the dataset: + # the output dictionary contains both the the input data and the output data. + # The output dictionary may contain Arrow values from `inputs_to_merge` so that we can re-write them efficiently. + return {**inputs_to_merge, **processed_inputs} + else: + return processed_inputs + + def apply_function(pa_inputs, indices, offset=0): + """Utility to apply the function on a selection of columns.""" + inputs, fn_args, additional_args, fn_kwargs = prepare_inputs(pa_inputs, indices, offset=offset) + processed_inputs = function(*fn_args, *additional_args, **fn_kwargs) + return prepare_outputs(pa_inputs, inputs, processed_inputs) + + async def async_apply_function(pa_inputs, indices, offset=0): + """Utility to apply the function on a selection of columns. Same code but async""" + inputs, fn_args, additional_args, fn_kwargs = prepare_inputs(pa_inputs, indices, offset=offset) + processed_inputs = await function(*fn_args, *additional_args, **fn_kwargs) + return prepare_outputs(pa_inputs, inputs, processed_inputs) + + def init_buffer_and_writer(): + # Prepare output buffer and batched writer in memory or on file if we update the table + writer_features = features + if writer_features is None: + writer_features = shard.features + update_features = True + else: + update_features = False + if keep_in_memory or cache_file_name is None: + buf_writer = pa.BufferOutputStream() + tmp_file = None + writer = ArrowWriter( + features=writer_features, + stream=buf_writer, + writer_batch_size=writer_batch_size, + update_features=update_features, + fingerprint=new_fingerprint, + disable_nullable=disable_nullable, + ) + else: + buf_writer = None + logger.info(f"Caching processed dataset at {cache_file_name}") + cache_dir = os.path.dirname(cache_file_name) + os.makedirs(cache_dir, exist_ok=True) + tmp_file = tempfile.NamedTemporaryFile("wb", dir=cache_dir, delete=False) + writer = ArrowWriter( + features=writer_features, + path=tmp_file.name, + writer_batch_size=writer_batch_size, + update_features=update_features, + fingerprint=new_fingerprint, + disable_nullable=disable_nullable, + ) + return buf_writer, writer, tmp_file + + tasks: list[asyncio.Task] = [] + if inspect.iscoroutinefunction(function): + try: + loop = asyncio.get_running_loop() + except RuntimeError: + loop = asyncio.new_event_loop() + else: + loop = None + + def iter_outputs(shard_iterable): + nonlocal tasks, loop + if inspect.iscoroutinefunction(function): + indices: Union[list[int], list[list[int]]] = [] + for i, example in shard_iterable: + indices.append(i) + tasks.append(loop.create_task(async_apply_function(example, i, offset=offset))) + # keep the total active tasks under a certain number + if len(tasks) >= config.MAX_NUM_RUNNING_ASYNC_MAP_FUNCTIONS_IN_PARALLEL: + done, pending = loop.run_until_complete( + asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED) + ) + while tasks and len(pending) >= config.MAX_NUM_RUNNING_ASYNC_MAP_FUNCTIONS_IN_PARALLEL: + done, pending = loop.run_until_complete( + asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED) + ) + # yield finished tasks + while tasks and tasks[0].done(): + yield indices.pop(0), tasks.pop(0).result() + while tasks: + yield indices[0], loop.run_until_complete(tasks[0]) + indices.pop(0), tasks.pop(0) + else: + for i, example in shard_iterable: + yield i, apply_function(example, i, offset=offset) + + num_examples_progress_update = 0 + # If `update_data` is True after processing the first example/batch, initialize these resources with `init_buffer_and_writer` + buf_writer, writer, tmp_file = None, None, None + + # Check if Polars is available and import it if so + if config.POLARS_AVAILABLE and "polars" in sys.modules: + import polars as pl + + # Optionally initialize the writer as a context manager + with contextlib.ExitStack() as stack: + try: + arrow_formatted_shard = shard.with_format("arrow") + + # Loop over single examples or batches and write to buffer/file if examples are to be updated + if not batched: + shard_iterable = enumerate(arrow_formatted_shard) + else: + num_rows = len(shard) if not drop_last_batch else len(shard) // batch_size * batch_size + shard_iterable = zip( + (list(range(i, min(i + batch_size, num_rows))) for i in range(0, num_rows, batch_size)), + arrow_formatted_shard.iter(batch_size, drop_last_batch=drop_last_batch), + ) + if not batched: + _time = time.time() + for i, example in iter_outputs(shard_iterable): + if update_data: + if i == 0: + buf_writer, writer, tmp_file = init_buffer_and_writer() + stack.enter_context(writer) + if isinstance(example, pa.Table): + writer.write_row(example) + elif isinstance(example, pd.DataFrame): + writer.write_row(pa.Table.from_pandas(example)) + elif ( + config.POLARS_AVAILABLE + and "polars" in sys.modules + and isinstance(example, pl.DataFrame) + ): + writer.write_row(example.to_arrow()) + else: + writer.write(example) + num_examples_progress_update += 1 + if time.time() > _time + config.PBAR_REFRESH_TIME_INTERVAL: + _time = time.time() + yield rank, False, num_examples_progress_update + num_examples_progress_update = 0 + else: + _time = time.time() + for i, batch in iter_outputs(shard_iterable): + num_examples_in_batch = len(i) + if update_data: + if i and i[0] == 0: + buf_writer, writer, tmp_file = init_buffer_and_writer() + stack.enter_context(writer) + if isinstance(batch, pa.Table): + writer.write_table(batch) + elif isinstance(batch, pd.DataFrame): + writer.write_table(pa.Table.from_pandas(batch)) + elif ( + config.POLARS_AVAILABLE and "polars" in sys.modules and isinstance(batch, pl.DataFrame) + ): + writer.write_table(batch.to_arrow()) + else: + writer.write_batch(batch, try_original_type=try_original_type) + num_examples_progress_update += num_examples_in_batch + if time.time() > _time + config.PBAR_REFRESH_TIME_INTERVAL: + _time = time.time() + yield rank, False, num_examples_progress_update + num_examples_progress_update = 0 + if update_data and writer is not None: + writer.finalize() # close_stream=bool(buf_writer is None)) # We only close if we are writing in a file + except (Exception, KeyboardInterrupt): + yield rank, False, num_examples_progress_update + if update_data: + if writer is not None: + writer.finalize() + if tmp_file is not None: + tmp_file.close() + if os.path.exists(tmp_file.name): + os.remove(tmp_file.name) + if loop: + logger.debug(f"Canceling {len(tasks)} async tasks.") + for task in tasks: + task.cancel(msg="KeyboardInterrupt") + try: + loop.run_until_complete(asyncio.gather(*tasks)) + except (asyncio.CancelledError, ValueError): + logger.debug("Tasks canceled.") + raise + + yield rank, False, num_examples_progress_update + if update_data and tmp_file is not None: + tmp_file.close() + shutil.move(tmp_file.name, cache_file_name) + umask = os.umask(0o666) + os.umask(umask) + os.chmod(cache_file_name, 0o666 & ~umask) + + if update_data: + # Create new Dataset from buffer or file + info = shard.info.copy() + info.features = writer._features + if buf_writer is None: + yield rank, True, Dataset.from_file(cache_file_name, info=info, split=shard.split) + else: + yield rank, True, Dataset.from_buffer(buf_writer.getvalue(), info=info, split=shard.split) + else: + yield rank, True, shard + + @transmit_format + @fingerprint_transform(inplace=False) + def batch( + self, + batch_size: int, + drop_last_batch: bool = False, + num_proc: Optional[int] = None, + new_fingerprint: Optional[str] = None, + ) -> "Dataset": + """ + Group samples from the dataset into batches. + + Args: + batch_size (`int`): + The number of samples in each batch. + drop_last_batch (`bool`, defaults to `False`): + Whether to drop the last incomplete batch. + num_proc (`int`, *optional*, defaults to `None`): + Max number of processes when generating cache. Already cached shards are loaded sequentially. + new_fingerprint (`str`, *optional*, defaults to `None`): + The new fingerprint of the dataset after transform. + If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments. + + Returns: + [`Dataset`]: A new Dataset where each item is a batch of multiple samples from the original dataset. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train") + >>> batched_ds = ds.batch(batch_size=4) + >>> batched_ds[0] + {'text': ['compassionately explores the seemingly irreconcilable situation...', ...], # 4 items + 'label': [1, 1, 1, 1]} + ``` + """ + + def batch_fn(example): + return {k: [v] for k, v in example.items()} + + return self.map( + batch_fn, + batched=True, + batch_size=batch_size, + drop_last_batch=drop_last_batch, + num_proc=num_proc, + new_fingerprint=new_fingerprint, + desc="Batching examples", + ) + + @transmit_format + @fingerprint_transform( + inplace=False, ignore_kwargs=["load_from_cache_file", "cache_file_name", "desc"], version="2.0.1" + ) + def filter( + self, + function: Optional[Callable] = None, + with_indices: bool = False, + with_rank: bool = False, + input_columns: Optional[Union[str, list[str]]] = None, + batched: bool = False, + batch_size: Optional[int] = 1000, + keep_in_memory: bool = False, + load_from_cache_file: Optional[bool] = None, + cache_file_name: Optional[str] = None, + writer_batch_size: Optional[int] = 1000, + fn_kwargs: Optional[dict] = None, + num_proc: Optional[int] = None, + suffix_template: str = "_{rank:05d}_of_{num_proc:05d}", + new_fingerprint: Optional[str] = None, + desc: Optional[str] = None, + ) -> "Dataset": + """Apply a filter function to all the elements in the table in batches + and update the table so that the dataset only includes examples according to the filter function. + + If the function is asynchronous, then `filter` will run your function in parallel, with up to one thousand simultaneous calls (configurable). + It is recommended to use a `asyncio.Semaphore` in your function if you want to set a maximum number of operations that can run at the same time. + + Args: + function (`Callable`): Callable with one of the following signatures: + + - `function(example: Dict[str, Any]) -> bool` if `batched=False` and `with_indices=False` and `with_rank=False` + - `function(example: Dict[str, Any], *extra_args) -> bool` if `batched=False` and `with_indices=True` and/or `with_rank=True` (one extra arg for each) + - `function(batch: Dict[str, List]) -> List[bool]` if `batched=True` and `with_indices=False` and `with_rank=False` + - `function(batch: Dict[str, List], *extra_args) -> List[bool]` if `batched=True` and `with_indices=True` and/or `with_rank=True` (one extra arg for each) + + If the function is asynchronous, then `filter` will run your function in parallel. + If no function is provided, defaults to an always `True` function: `lambda x: True`. + with_indices (`bool`, defaults to `False`): + Provide example indices to `function`. Note that in this case the + signature of `function` should be `def function(example, idx[, rank]): ...`. + with_rank (`bool`, defaults to `False`): + Provide process rank to `function`. Note that in this case the + signature of `function` should be `def function(example[, idx], rank): ...`. + input_columns (`str` or `List[str]`, *optional*): + The columns to be passed into `function` as + positional arguments. If `None`, a `dict` mapping to all formatted columns is passed as one argument. + batched (`bool`, defaults to `False`): + Provide batch of examples to `function`. + batch_size (`int`, *optional*, defaults to `1000`): + Number of examples per batch provided to `function` if + `batched = True`. If `batched = False`, one example per batch is passed to `function`. + If `batch_size <= 0` or `batch_size == None`, provide the full dataset as a single batch to `function`. + keep_in_memory (`bool`, defaults to `False`): + Keep the dataset in memory instead of writing it to a cache file. + load_from_cache_file (`Optional[bool]`, defaults to `True` if caching is enabled): + If a cache file storing the current computation from `function` + can be identified, use it instead of recomputing. + cache_file_name (`str`, *optional*): + Provide the name of a path for the cache file. It is used to store the + results of the computation instead of the automatically generated cache file name. + writer_batch_size (`int`, defaults to `1000`): + Number of rows per write operation for the cache file writer. + This value is a good trade-off between memory usage during the processing, and processing speed. + Higher value makes the processing do fewer lookups, lower value consume less temporary memory while running `map`. + fn_kwargs (`dict`, *optional*): + Keyword arguments to be passed to `function`. + num_proc (`int`, *optional*): + Number of processes for multiprocessing. By default it doesn't + use multiprocessing. + suffix_template (`str`): + If `cache_file_name` is specified, then this suffix will be added at the end of the base name of each. + For example, if `cache_file_name` is `"processed.arrow"`, then for `rank = 1` and `num_proc = 4`, + the resulting file would be `"processed_00001_of_00004.arrow"` for the default suffix (default + `_{rank:05d}_of_{num_proc:05d}`). + new_fingerprint (`str`, *optional*): + The new fingerprint of the dataset after transform. + If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments. + desc (`str`, *optional*, defaults to `None`): + Meaningful description to be displayed alongside with the progress bar while filtering examples. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> ds.filter(lambda x: x["label"] == 1) + Dataset({ + features: ['text', 'label'], + num_rows: 533 + }) + ``` + """ + if len(self.list_indexes()) > 0: + raise DatasetTransformationNotAllowedError( + "Using `.filter` on a dataset with attached indexes is not allowed. You can first run `.drop_index() to remove your index and then re-add it.`" + ) + + if function is None: + function = lambda x: True # noqa: E731 + + if len(self) == 0: + return self + + # We generally batch the underlying map() to get faster throughput, + # but in case of async we force batch_size=1 to enable parallelism + if inspect.iscoroutinefunction(function) and not batched: + batch_size = 1 + + indices = self.map( + function=partial( + async_get_indices_from_mask_function + if inspect.iscoroutinefunction(function) + else get_indices_from_mask_function, + function, + batched, + with_indices, + with_rank, + input_columns, + self._indices, + ), + with_indices=True, + with_rank=True, + features=Features({"indices": Value("uint64")}), + batched=True, + batch_size=batch_size, + remove_columns=self.column_names, + keep_in_memory=keep_in_memory, + load_from_cache_file=load_from_cache_file, + cache_file_name=cache_file_name, + writer_batch_size=writer_batch_size, + fn_kwargs=fn_kwargs, + num_proc=num_proc, + suffix_template=suffix_template, + new_fingerprint=new_fingerprint, + input_columns=input_columns, + desc=desc or "Filter", + ) + new_dataset = copy.deepcopy(self) + new_dataset._indices = indices.data + new_dataset._fingerprint = new_fingerprint + return new_dataset + + @transmit_format + @fingerprint_transform(inplace=False, ignore_kwargs=["cache_file_name"]) + def flatten_indices( + self, + keep_in_memory: bool = False, + cache_file_name: Optional[str] = None, + writer_batch_size: Optional[int] = 1000, + features: Optional[Features] = None, + disable_nullable: bool = False, + num_proc: Optional[int] = None, + new_fingerprint: Optional[str] = None, + ) -> "Dataset": + """Create and cache a new Dataset by flattening the indices mapping. + + Args: + keep_in_memory (`bool`, defaults to `False`): + Keep the dataset in memory instead of writing it to a cache file. + cache_file_name (`str`, *optional*, default `None`): + Provide the name of a path for the cache file. It is used to store the + results of the computation instead of the automatically generated cache file name. + writer_batch_size (`int`, defaults to `1000`): + Number of rows per write operation for the cache file writer. + This value is a good trade-off between memory usage during the processing, and processing speed. + Higher value makes the processing do fewer lookups, lower value consume less temporary memory while running `map`. + features (`Optional[datasets.Features]`, defaults to `None`): + Use a specific [`Features`] to store the cache file + instead of the automatically generated one. + disable_nullable (`bool`, defaults to `False`): + Allow null values in the table. + num_proc (`int`, optional, default `None`): + Max number of processes when generating cache. Already cached shards are loaded sequentially + new_fingerprint (`str`, *optional*, defaults to `None`): + The new fingerprint of the dataset after transform. + If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments + """ + + return self.map( + batched=True, # for speed + keep_in_memory=keep_in_memory, + cache_file_name=cache_file_name, + writer_batch_size=writer_batch_size, + features=features, + disable_nullable=disable_nullable, + new_fingerprint=new_fingerprint, + desc="Flattening the indices", + num_proc=num_proc, + ) + + def _new_dataset_with_indices( + self, + indices_cache_file_name: Optional[str] = None, + indices_buffer: Optional[pa.Buffer] = None, + fingerprint: Optional[str] = None, + ) -> "Dataset": + """Return a new Dataset obtained by adding indices (provided in indices_cache_file_name or in a buffer) to the + current Dataset. + """ + + if indices_cache_file_name is None and indices_buffer is None: + raise ValueError("At least one of indices_cache_file_name or indices_buffer must be provided.") + + if fingerprint is None: + raise ValueError("please specify a fingerprint for the dataset with indices") + + if indices_cache_file_name is not None: + indices_table = MemoryMappedTable.from_file(indices_cache_file_name) + else: + indices_table = InMemoryTable.from_buffer(indices_buffer) + + # Return new Dataset object + # don't forget to copy the objects + return Dataset( + self._data, + info=self.info.copy(), + split=self.split, + indices_table=indices_table, + fingerprint=fingerprint, + ) + + @transmit_format + @fingerprint_transform(inplace=False, ignore_kwargs=["indices_cache_file_name"]) + def select( + self, + indices: Iterable, + keep_in_memory: bool = False, + indices_cache_file_name: Optional[str] = None, + writer_batch_size: Optional[int] = 1000, + new_fingerprint: Optional[str] = None, + ) -> "Dataset": + """Create a new dataset with rows selected following the list/array of indices. + + Args: + indices (`range`, `list`, `iterable`, `ndarray` or `Series`): + Range, list or 1D-array of integer indices for indexing. + If the indices correspond to a contiguous range, the Arrow table is simply sliced. + However passing a list of indices that are not contiguous creates indices mapping, which is much less efficient, + but still faster than recreating an Arrow table made of the requested rows. + keep_in_memory (`bool`, defaults to `False`): + Keep the indices mapping in memory instead of writing it to a cache file. + indices_cache_file_name (`str`, *optional*, defaults to `None`): + Provide the name of a path for the cache file. It is used to store the + indices mapping instead of the automatically generated cache file name. + writer_batch_size (`int`, defaults to `1000`): + Number of rows per write operation for the cache file writer. + This value is a good trade-off between memory usage during the processing, and processing speed. + Higher value makes the processing do fewer lookups, lower value consume less temporary memory while running `map`. + new_fingerprint (`str`, *optional*, defaults to `None`): + The new fingerprint of the dataset after transform. + If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> ds.select(range(4)) + Dataset({ + features: ['text', 'label'], + num_rows: 4 + }) + ``` + """ + if keep_in_memory and indices_cache_file_name is not None: + raise ValueError("Please use either `keep_in_memory` or `indices_cache_file_name` but not both.") + + if len(self.list_indexes()) > 0: + raise DatasetTransformationNotAllowedError( + "Using `.select` on a dataset with attached indexes is not allowed. You can first run `.drop_index() to remove your index and then re-add it." + ) + + # If the array is empty we do nothing + if len(self) == 0: + return self + + # If indices is a PyArrow array, we convert to NumPy + if isinstance(indices, (pa.Array, pa.ChunkedArray)): + indices = indices.to_numpy().astype(np.int64) + + # Convert generator objects to lists + if isinstance(indices, Iterator): + indices = list(indices) + + # If the indices are contiguous, simply slice the arrow table + if isinstance(indices, range): + if _is_range_contiguous(indices) and indices.start >= 0: + start, length = indices.start, indices.stop - indices.start + return self._select_contiguous(start, length, new_fingerprint=new_fingerprint) + else: + try: + start = next(iter(indices)) + except StopIteration: + # if `indices` is an empty iterable, we return an empty dataset + return self._select_contiguous(0, 0, new_fingerprint=new_fingerprint) + if start >= 0: + counter_from_start = itertools.count(start=start) + if all(i == j for i, j in zip(indices, counter_from_start)): + length = next(counter_from_start) - start + return self._select_contiguous(start, length, new_fingerprint=new_fingerprint) + + # If not contiguous, we need to create a new indices mapping + return self._select_with_indices_mapping( + indices, + keep_in_memory=keep_in_memory, + indices_cache_file_name=indices_cache_file_name, + writer_batch_size=writer_batch_size, + new_fingerprint=new_fingerprint, + ) + + @transmit_format + @fingerprint_transform(inplace=False) + def _select_contiguous( + self, + start: int, + length: int, + new_fingerprint: Optional[str] = None, + ) -> "Dataset": + """Create a new dataset with rows from a contiguous slice of data. + The slice is defined by that start index and its length. + + Args: + start (`int`): start index. + length (`int`): length of the slice to select. + new_fingerprint (`str`, optional, default `None`): the new fingerprint of the dataset after transform. + If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> ds._select_contiguous(0, 4) + Dataset({ + features: ['text', 'label'], + num_rows: 4 + }) + ``` + """ + if len(self.list_indexes()) > 0: + raise DatasetTransformationNotAllowedError( + "Using `.select` on a dataset with attached indexes is not allowed. You can first run `.drop_index() to remove your index and then re-add it." + ) + + # If the array is empty we do nothing + if len(self) == 0: + return self + + _check_valid_indices_value(start, len(self)) + _check_valid_indices_value(start + length - 1, len(self)) + if self._indices is None or length == 0: + return Dataset( + self.data.slice(start, length), + info=self.info.copy(), + split=self.split, + fingerprint=new_fingerprint, + ) + else: + return Dataset( + self.data, + info=self.info.copy(), + split=self.split, + indices_table=self._indices.slice(start, length), + fingerprint=new_fingerprint, + ) + + @transmit_format + @fingerprint_transform(inplace=False, ignore_kwargs=["indices_cache_file_name"]) + def _select_with_indices_mapping( + self, + indices: Iterable, + keep_in_memory: bool = False, + indices_cache_file_name: Optional[str] = None, + writer_batch_size: Optional[int] = 1000, + new_fingerprint: Optional[str] = None, + ) -> "Dataset": + """Create a new dataset with rows selected following the list/array of indices. + The new dataset is made by creating a new indices mapping on top of the main arrow table. + + Args: + indices (sequence, iterable, range, ndarray or Series): List or 1D-array of integer indices for indexing. + keep_in_memory (`bool`, default `False`): Keep the indices mapping in memory instead of writing it to a cache file. + indices_cache_file_name (`str`, optional, default `None`): Provide the name of a path for the cache file. It is used to store the + indices mapping instead of the automatically generated cache file name. + writer_batch_size (`int`, default `1000`): Number of rows per write operation for the cache file writer. + This value is a good trade-off between memory usage during the processing, and processing speed. + Higher value makes the processing do fewer lookups, lower value consume less temporary memory while running `.map()`. + new_fingerprint (`str`, optional, default `None`): the new fingerprint of the dataset after transform. + If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> ds._select_with_indices_mapping(range(4)) + Dataset({ + features: ['text', 'label'], + num_rows: 4 + }) + ``` + """ + if keep_in_memory and indices_cache_file_name is not None: + raise ValueError("Please use either `keep_in_memory` or `indices_cache_file_name` but not both.") + + if len(self.list_indexes()) > 0: + raise DatasetTransformationNotAllowedError( + "Using `.select` on a dataset with attached indexes is not allowed. You can first run `.drop_index() to remove your index and then re-add it." + ) + + # If the array is empty we do nothing + if len(self) == 0: + return self + + # Prepare the writer for our indices arrow table + if keep_in_memory or indices_cache_file_name is None: + buf_writer = pa.BufferOutputStream() + tmp_file = None + writer = ArrowWriter( + stream=buf_writer, writer_batch_size=writer_batch_size, fingerprint=new_fingerprint, unit="indices" + ) + else: + buf_writer = None + logger.info(f"Caching indices mapping at {indices_cache_file_name}") + cache_dir = os.path.dirname(indices_cache_file_name) + os.makedirs(cache_dir, exist_ok=True) + tmp_file = tempfile.NamedTemporaryFile("wb", dir=cache_dir, delete=False) + writer = ArrowWriter( + path=tmp_file.name, writer_batch_size=writer_batch_size, fingerprint=new_fingerprint, unit="indices" + ) + + indices = indices if isinstance(indices, list) else list(indices) + + size = len(self) + if indices: + _check_valid_indices_value(int(max(indices)), size=size) + _check_valid_indices_value(int(min(indices)), size=size) + else: + return self._select_contiguous(0, 0, new_fingerprint=new_fingerprint) + + indices_array = pa.array(indices, type=pa.uint64()) + # Check if we need to convert indices + if self._indices is not None: + indices_array = self._indices.column(0).take(indices_array) + + indices_table = pa.Table.from_arrays([indices_array], names=["indices"]) + + with writer: + try: + writer.write_table(indices_table) + writer.finalize() # close_stream=bool(buf_writer is None)) We only close if we are writing in a file + except (Exception, KeyboardInterrupt): + if tmp_file is not None: + tmp_file.close() + if os.path.exists(tmp_file.name): + os.remove(tmp_file.name) + raise + + if tmp_file is not None: + tmp_file.close() + shutil.move(tmp_file.name, indices_cache_file_name) + umask = os.umask(0o666) + os.umask(umask) + os.chmod(indices_cache_file_name, 0o666 & ~umask) + + # Return new Dataset object + if buf_writer is None: + return self._new_dataset_with_indices( + indices_cache_file_name=indices_cache_file_name, fingerprint=new_fingerprint + ) + else: + return self._new_dataset_with_indices(indices_buffer=buf_writer.getvalue(), fingerprint=new_fingerprint) + + def skip(self, n: int) -> "Dataset": + """ + Create a new [`Dataset`] that skips the first `n` elements. + + Args: + n (`int`): + Number of elements to skip. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train") + >>> list(ds.take(3)) + [{'label': 1, + 'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'}, + {'label': 1, + 'text': 'the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson\'s expanded vision of j . r . r . tolkien\'s middle-earth .'}, + {'label': 1, 'text': 'effective but too-tepid biopic'}] + >>> ds = ds.skip(1) + >>> list(ds.take(3)) + [{'label': 1, + 'text': 'the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson\'s expanded vision of j . r . r . tolkien\'s middle-earth .'}, + {'label': 1, 'text': 'effective but too-tepid biopic'}, + {'label': 1, + 'text': 'if you sometimes like to go to the movies to have fun , wasabi is a good place to start .'}] + ``` + """ + return self.select(range(n, len(self))) + + def repeat(self, num_times: int) -> "Dataset": + """ + Create a new [`Dataset`] that repeats the underlying dataset `num_times` times. + + Like itertools.repeat, repeating once just returns the full dataset. + + Args: + num_times (`int`): + Number of times to repeat the dataset. + + Example: + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train") + >>> ds = ds.take(2).repeat(2) + >>> list(ds) + [{'label': 1, + 'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'}, + {'label': 1, + 'text': 'the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson\'s expanded vision of j . r . r . tolkien\'s middle-earth .'}, + {'label': 1, 'text': 'effective but too-tepid biopic'}, + {'label': 1, + 'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'}, + {'label': 1, + 'text': 'the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson\'s expanded vision of j . r . r . tolkien\'s middle-earth .'}, + {'label': 1, 'text': 'effective but too-tepid biopic'}] + ``` + """ + if num_times is None: + raise ValueError("Map style datasets do not support indefinite repetition.") + return _concatenate_map_style_datasets([self] * num_times) if num_times > 0 else self.select([]) + + def take(self, n: int) -> "Dataset": + """ + Create a new [`Dataset`] with only the first `n` elements. + + Args: + n (`int`): + Number of elements to take. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train") + >>> small_ds = ds.take(2) + >>> list(small_ds) + [{'label': 1, + 'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'}, + {'label': 1, + 'text': 'the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson\'s expanded vision of j . r . r . tolkien\'s middle-earth .'}] + ``` + """ + return self.select(range(n)) + + @transmit_format + @fingerprint_transform(inplace=False, ignore_kwargs=["load_from_cache_file", "indices_cache_file_name"]) + def sort( + self, + column_names: Union[str, Sequence_[str]], + reverse: Union[bool, Sequence_[bool]] = False, + null_placement: str = "at_end", + keep_in_memory: bool = False, + load_from_cache_file: Optional[bool] = None, + indices_cache_file_name: Optional[str] = None, + writer_batch_size: Optional[int] = 1000, + new_fingerprint: Optional[str] = None, + ) -> "Dataset": + """Create a new dataset sorted according to a single or multiple columns. + + Args: + column_names (`Union[str, Sequence[str]]`): + Column name(s) to sort by. + reverse (`Union[bool, Sequence[bool]]`, defaults to `False`): + If `True`, sort by descending order rather than ascending. If a single bool is provided, + the value is applied to the sorting of all column names. Otherwise a list of bools with the + same length and order as column_names must be provided. + null_placement (`str`, defaults to `at_end`): + Put `None` values at the beginning if `at_start` or `first` or at the end if `at_end` or `last` + + + keep_in_memory (`bool`, defaults to `False`): + Keep the sorted indices in memory instead of writing it to a cache file. + load_from_cache_file (`Optional[bool]`, defaults to `True` if caching is enabled): + If a cache file storing the sorted indices + can be identified, use it instead of recomputing. + indices_cache_file_name (`str`, *optional*, defaults to `None`): + Provide the name of a path for the cache file. It is used to store the + sorted indices instead of the automatically generated cache file name. + writer_batch_size (`int`, defaults to `1000`): + Number of rows per write operation for the cache file writer. + Higher value gives smaller cache files, lower value consume less temporary memory. + new_fingerprint (`str`, *optional*, defaults to `None`): + The new fingerprint of the dataset after transform. + If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset('cornell-movie-review-data/rotten_tomatoes', split='validation') + >>> ds['label'][:10] + [1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + >>> sorted_ds = ds.sort('label') + >>> sorted_ds['label'][:10] + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + >>> another_sorted_ds = ds.sort(['label', 'text'], reverse=[True, False]) + >>> another_sorted_ds['label'][:10] + [1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + ``` + """ + if len(self.list_indexes()) > 0: + raise DatasetTransformationNotAllowedError( + "Using `.sort` on a dataset with attached indexes is not allowed. You can first run `.drop_index() to remove your index and then re-add it." + ) + # If the array is empty we do nothing + if len(self) == 0: + return self + + # Check proper format of and for duplicates in column_names + if isinstance(column_names, str): + column_names = [column_names] + + # Check proper format and length of reverse + if not isinstance(reverse, bool): + if len(reverse) != len(column_names): + raise ValueError( + "Parameter 'reverse' should be either a boolean or a list of booleans with the same length as 'column_names'." + ) + else: + reverse = [reverse] * len(column_names) + + # Check whether column name(s) exist in dataset + for column in column_names: + if not isinstance(column, str) or column not in self._data.column_names: + raise ValueError( + f"Column '{column}' not found in the dataset. Please provide a column selected in: {self._data.column_names}" + ) + + # Change null_placement to conform to pyarrow's sort_indices() while ensuring backwards compatibility + if null_placement not in ["at_start", "at_end"]: + if null_placement == "first": + null_placement = "at_start" + elif null_placement == "last": + null_placement = "at_end" + else: + raise ValueError( + f"null_placement '{null_placement}' is an invalid parameter value. Must be either 'last', 'at_end', 'first' or 'at_start'." + ) + + load_from_cache_file = load_from_cache_file if load_from_cache_file is not None else is_caching_enabled() + + # Check if we've already cached this computation (indexed by a hash) + if self.cache_files: + if indices_cache_file_name is None: + # we create a unique hash from the function, current dataset file and the mapping args + indices_cache_file_name = self._get_cache_file_path(new_fingerprint) + if os.path.exists(indices_cache_file_name) and load_from_cache_file: + logger.info(f"Loading cached sorted indices for dataset at {indices_cache_file_name}") + return self._new_dataset_with_indices( + fingerprint=new_fingerprint, indices_cache_file_name=indices_cache_file_name + ) + + sort_table = query_table( + table=self._data, + key=slice(0, len(self)), + indices=self._indices, + ) + + sort_keys = [ + (col, "ascending" if not col_reverse else "descending") for col, col_reverse in zip(column_names, reverse) + ] + + indices = pc.sort_indices(sort_table, sort_keys=sort_keys, null_placement=null_placement) + + return self.select( + indices=indices, + keep_in_memory=keep_in_memory, + indices_cache_file_name=indices_cache_file_name, + writer_batch_size=writer_batch_size, + new_fingerprint=new_fingerprint, + ) + + @transmit_format + @fingerprint_transform( + inplace=False, randomized_function=True, ignore_kwargs=["load_from_cache_file", "indices_cache_file_name"] + ) + def shuffle( + self, + seed: Optional[int] = None, + generator: Optional[np.random.Generator] = None, + keep_in_memory: bool = False, + load_from_cache_file: Optional[bool] = None, + indices_cache_file_name: Optional[str] = None, + writer_batch_size: Optional[int] = 1000, + new_fingerprint: Optional[str] = None, + ) -> "Dataset": + """Create a new Dataset where the rows are shuffled. + + Currently shuffling uses numpy random generators. + You can either supply a NumPy BitGenerator to use, or a seed to initiate NumPy's default random generator (PCG64). + + Shuffling takes the list of indices `[0:len(my_dataset)]` and shuffles it to create an indices mapping. + However as soon as your [`Dataset`] has an indices mapping, the speed can become 10x slower. + This is because there is an extra step to get the row index to read using the indices mapping, and most importantly, you aren't reading contiguous chunks of data anymore. + To restore the speed, you'd need to rewrite the entire dataset on your disk again using [`Dataset.flatten_indices`], which removes the indices mapping. + This may take a lot of time depending of the size of your dataset though: + + ```python + my_dataset[0] # fast + my_dataset = my_dataset.shuffle(seed=42) + my_dataset[0] # up to 10x slower + my_dataset = my_dataset.flatten_indices() # rewrite the shuffled dataset on disk as contiguous chunks of data + my_dataset[0] # fast again + ``` + + In this case, we recommend switching to an [`IterableDataset`] and leveraging its fast approximate shuffling method [`IterableDataset.shuffle`]. + It only shuffles the shards order and adds a shuffle buffer to your dataset, which keeps the speed of your dataset optimal: + + ```python + my_iterable_dataset = my_dataset.to_iterable_dataset(num_shards=128) + for example in enumerate(my_iterable_dataset): # fast + pass + + shuffled_iterable_dataset = my_iterable_dataset.shuffle(seed=42, buffer_size=100) + + for example in enumerate(shuffled_iterable_dataset): # as fast as before + pass + ``` + + Args: + seed (`int`, *optional*): + A seed to initialize the default BitGenerator if `generator=None`. + If `None`, then fresh, unpredictable entropy will be pulled from the OS. + If an `int` or `array_like[ints]` is passed, then it will be passed to SeedSequence to derive the initial BitGenerator state. + generator (`numpy.random.Generator`, *optional*): + Numpy random Generator to use to compute the permutation of the dataset rows. + If `generator=None` (default), uses `np.random.default_rng` (the default BitGenerator (PCG64) of NumPy). + keep_in_memory (`bool`, default `False`): + Keep the shuffled indices in memory instead of writing it to a cache file. + load_from_cache_file (`Optional[bool]`, defaults to `True` if caching is enabled): + If a cache file storing the shuffled indices + can be identified, use it instead of recomputing. + indices_cache_file_name (`str`, *optional*): + Provide the name of a path for the cache file. It is used to store the + shuffled indices instead of the automatically generated cache file name. + writer_batch_size (`int`, defaults to `1000`): + Number of rows per write operation for the cache file writer. + This value is a good trade-off between memory usage during the processing, and processing speed. + Higher value makes the processing do fewer lookups, lower value consume less temporary memory while running `map`. + new_fingerprint (`str`, *optional*, defaults to `None`): + The new fingerprint of the dataset after transform. + If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> ds['label'][:10] + [1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + + # set a seed + >>> shuffled_ds = ds.shuffle(seed=42) + >>> shuffled_ds['label'][:10] + [1, 0, 1, 1, 0, 0, 0, 0, 0, 0] + ``` + """ + if len(self.list_indexes()) > 0: + raise DatasetTransformationNotAllowedError( + "Using `.shuffle` on a dataset with attached indexes is not allowed. You can first run `.drop_index() to remove your index and then re-add it." + ) + # If the array is empty we do nothing + if len(self) == 0: + return self + + if keep_in_memory and indices_cache_file_name is not None: + raise ValueError("Please use either `keep_in_memory` or `indices_cache_file_name` but not both.") + + if seed is not None and generator is not None: + raise ValueError("Both `seed` and `generator` were provided. Please specify just one of them.") + + if generator is not None and not isinstance(generator, np.random.Generator): + raise ValueError("The provided generator must be an instance of numpy.random.Generator") + + load_from_cache_file = load_from_cache_file if load_from_cache_file is not None else is_caching_enabled() + + if generator is None: + if seed is None: + _, seed, pos, *_ = np.random.get_state() + seed = seed[pos] if pos < 624 else seed[0] + _ = np.random.random() # do 1 step of rng + generator = np.random.default_rng(seed) + + # Check if we've already cached this computation (indexed by a hash) + if self.cache_files: + if indices_cache_file_name is None: + # we create a unique hash from the function, current dataset file and the mapping args + indices_cache_file_name = self._get_cache_file_path(new_fingerprint) + if os.path.exists(indices_cache_file_name) and load_from_cache_file: + logger.info(f"Loading cached shuffled indices for dataset at {indices_cache_file_name}") + return self._new_dataset_with_indices( + fingerprint=new_fingerprint, indices_cache_file_name=indices_cache_file_name + ) + + permutation = generator.permutation(len(self)) + + return self.select( + indices=permutation, + keep_in_memory=keep_in_memory, + indices_cache_file_name=indices_cache_file_name if not keep_in_memory else None, + writer_batch_size=writer_batch_size, + new_fingerprint=new_fingerprint, + ) + + @transmit_format + @fingerprint_transform( + inplace=False, + randomized_function=True, + fingerprint_names=["train_new_fingerprint", "test_new_fingerprint"], + ignore_kwargs=["load_from_cache_file", "train_indices_cache_file_name", "test_indices_cache_file_name"], + ) + def train_test_split( + self, + test_size: Union[float, int, None] = None, + train_size: Union[float, int, None] = None, + shuffle: bool = True, + stratify_by_column: Optional[str] = None, + seed: Optional[int] = None, + generator: Optional[np.random.Generator] = None, + keep_in_memory: bool = False, + load_from_cache_file: Optional[bool] = None, + train_indices_cache_file_name: Optional[str] = None, + test_indices_cache_file_name: Optional[str] = None, + writer_batch_size: Optional[int] = 1000, + train_new_fingerprint: Optional[str] = None, + test_new_fingerprint: Optional[str] = None, + ) -> "DatasetDict": + """Return a dictionary ([`datasets.DatasetDict`]) with two random train and test subsets (`train` and `test` `Dataset` splits). + Splits are created from the dataset according to `test_size`, `train_size` and `shuffle`. + + This method is similar to scikit-learn `train_test_split`. + + Args: + test_size (`numpy.random.Generator`, *optional*): + Size of the test split + If `float`, should be between `0.0` and `1.0` and represent the proportion of the dataset to include in the test split. + If `int`, represents the absolute number of test samples. + If `None`, the value is set to the complement of the train size. + If `train_size` is also `None`, it will be set to `0.25`. + train_size (`numpy.random.Generator`, *optional*): + Size of the train split + If `float`, should be between `0.0` and `1.0` and represent the proportion of the dataset to include in the train split. + If `int`, represents the absolute number of train samples. + If `None`, the value is automatically set to the complement of the test size. + shuffle (`bool`, *optional*, defaults to `True`): + Whether or not to shuffle the data before splitting. + stratify_by_column (`str`, *optional*, defaults to `None`): + The column name of labels to be used to perform stratified split of data. + seed (`int`, *optional*): + A seed to initialize the default BitGenerator if `generator=None`. + If `None`, then fresh, unpredictable entropy will be pulled from the OS. + If an `int` or `array_like[ints]` is passed, then it will be passed to SeedSequence to derive the initial BitGenerator state. + generator (`numpy.random.Generator`, *optional*): + Numpy random Generator to use to compute the permutation of the dataset rows. + If `generator=None` (default), uses `np.random.default_rng` (the default BitGenerator (PCG64) of NumPy). + keep_in_memory (`bool`, defaults to `False`): + Keep the splits indices in memory instead of writing it to a cache file. + load_from_cache_file (`Optional[bool]`, defaults to `True` if caching is enabled): + If a cache file storing the splits indices + can be identified, use it instead of recomputing. + train_cache_file_name (`str`, *optional*): + Provide the name of a path for the cache file. It is used to store the + train split indices instead of the automatically generated cache file name. + test_cache_file_name (`str`, *optional*): + Provide the name of a path for the cache file. It is used to store the + test split indices instead of the automatically generated cache file name. + writer_batch_size (`int`, defaults to `1000`): + Number of rows per write operation for the cache file writer. + This value is a good trade-off between memory usage during the processing, and processing speed. + Higher value makes the processing do fewer lookups, lower value consume less temporary memory while running `map`. + train_new_fingerprint (`str`, *optional*, defaults to `None`): + The new fingerprint of the train set after transform. + If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments + test_new_fingerprint (`str`, *optional*, defaults to `None`): + The new fingerprint of the test set after transform. + If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> ds = ds.train_test_split(test_size=0.2, shuffle=True) + DatasetDict({ + train: Dataset({ + features: ['text', 'label'], + num_rows: 852 + }) + test: Dataset({ + features: ['text', 'label'], + num_rows: 214 + }) + }) + + # set a seed + >>> ds = ds.train_test_split(test_size=0.2, seed=42) + + # stratified split + >>> ds = load_dataset("imdb",split="train") + Dataset({ + features: ['text', 'label'], + num_rows: 25000 + }) + >>> ds = ds.train_test_split(test_size=0.2, stratify_by_column="label") + DatasetDict({ + train: Dataset({ + features: ['text', 'label'], + num_rows: 20000 + }) + test: Dataset({ + features: ['text', 'label'], + num_rows: 5000 + }) + }) + ``` + """ + from .dataset_dict import DatasetDict # import here because of circular dependency + + if len(self.list_indexes()) > 0: + raise DatasetTransformationNotAllowedError( + "Using `.train_test_split` on a dataset with attached indexes is not allowed. You can first run `.drop_index() to remove your index and then re-add it." + ) + # If the array is empty we do nothing + if len(self) == 0: + return DatasetDict({"train": self, "test": self}) + + if test_size is None and train_size is None: + test_size = 0.25 + + # Safety checks similar to scikit-learn's ones. + # (adapted from https://github.com/scikit-learn/scikit-learn/blob/fd237278e895b42abe8d8d09105cbb82dc2cbba7/sklearn/model_selection/_split.py#L1750) + n_samples = len(self) + if ( + isinstance(test_size, int) + and (test_size >= n_samples or test_size <= 0) + or isinstance(test_size, float) + and (test_size <= 0 or test_size >= 1) + ): + raise ValueError( + f"test_size={test_size} should be either positive and smaller " + f"than the number of samples {n_samples} or a float in the (0, 1) range" + ) + + if ( + isinstance(train_size, int) + and (train_size >= n_samples or train_size <= 0) + or isinstance(train_size, float) + and (train_size <= 0 or train_size >= 1) + ): + raise ValueError( + f"train_size={train_size} should be either positive and smaller " + f"than the number of samples {n_samples} or a float in the (0, 1) range" + ) + + if train_size is not None and not isinstance(train_size, (int, float)): + raise ValueError(f"Invalid value for train_size: {train_size} of type {type(train_size)}") + if test_size is not None and not isinstance(test_size, (int, float)): + raise ValueError(f"Invalid value for test_size: {test_size} of type {type(test_size)}") + + if isinstance(train_size, float) and isinstance(test_size, float) and train_size + test_size > 1: + raise ValueError( + f"The sum of test_size and train_size = {train_size + test_size}, should be in the (0, 1)" + " range. Reduce test_size and/or train_size." + ) + + if isinstance(test_size, float): + n_test = ceil(test_size * n_samples) + elif isinstance(test_size, int): + n_test = float(test_size) + + if isinstance(train_size, float): + n_train = floor(train_size * n_samples) + elif isinstance(train_size, int): + n_train = float(train_size) + + if train_size is None: + n_train = n_samples - n_test + elif test_size is None: + n_test = n_samples - n_train + + if n_train + n_test > n_samples: + raise ValueError( + f"The sum of train_size and test_size = {n_train + n_test}, " + "should be smaller than the number of " + f"samples {n_samples}. Reduce test_size and/or " + "train_size." + ) + + n_train, n_test = int(n_train), int(n_test) + + if n_train == 0: + raise ValueError( + f"With n_samples={n_samples}, test_size={test_size} and train_size={train_size}, the " + "resulting train set will be empty. Adjust any of the " + "aforementioned parameters." + ) + + load_from_cache_file = load_from_cache_file if load_from_cache_file is not None else is_caching_enabled() + + if generator is None and shuffle is True: + if seed is None: + _, seed, pos, *_ = np.random.get_state() + seed = seed[pos] if pos < 624 else seed[0] + _ = np.random.random() # do 1 step of rng + generator = np.random.default_rng(seed) + + # Check if we've already cached this computation (indexed by a hash) + if self.cache_files: + if train_indices_cache_file_name is None or test_indices_cache_file_name is None: + # we create a unique hash from the function, current dataset file and the mapping args + + if train_indices_cache_file_name is None: + train_indices_cache_file_name = self._get_cache_file_path(train_new_fingerprint) + if test_indices_cache_file_name is None: + test_indices_cache_file_name = self._get_cache_file_path(test_new_fingerprint) + if ( + os.path.exists(train_indices_cache_file_name) + and os.path.exists(test_indices_cache_file_name) + and load_from_cache_file + ): + logger.info( + f"Loading cached split indices for dataset at {train_indices_cache_file_name} and {test_indices_cache_file_name}" + ) + return DatasetDict( + { + "train": self._new_dataset_with_indices( + fingerprint=train_new_fingerprint, indices_cache_file_name=train_indices_cache_file_name + ), + "test": self._new_dataset_with_indices( + fingerprint=test_new_fingerprint, indices_cache_file_name=test_indices_cache_file_name + ), + } + ) + if not shuffle: + if stratify_by_column is not None: + raise ValueError("Stratified train/test split is not implemented for `shuffle=False`") + train_indices = np.arange(n_train) + test_indices = np.arange(n_train, n_train + n_test) + else: + # stratified partition + if stratify_by_column is not None: + if stratify_by_column not in self._info.features.keys(): + raise ValueError(f"Key {stratify_by_column} not found in {self._info.features.keys()}") + if not isinstance(self._info.features[stratify_by_column], ClassLabel): + raise ValueError( + f"Stratifying by column is only supported for {ClassLabel.__name__} column, and column {stratify_by_column} is {type(self._info.features[stratify_by_column]).__name__}." + ) + try: + train_indices, test_indices = next( + stratified_shuffle_split_generate_indices( + self.with_format("numpy")[stratify_by_column], n_train, n_test, rng=generator + ) + ) + except Exception as error: + if str(error) == "Minimum class count error": + raise ValueError( + f"The least populated class in {stratify_by_column} column has only 1" + " member, which is too few. The minimum" + " number of groups for any class cannot" + " be less than 2." + ) + else: + raise error + + # random partition + else: + permutation = generator.permutation(len(self)) + test_indices = permutation[:n_test] + train_indices = permutation[n_test : (n_test + n_train)] + + train_split = self.select( + indices=train_indices, + keep_in_memory=keep_in_memory, + indices_cache_file_name=train_indices_cache_file_name, + writer_batch_size=writer_batch_size, + new_fingerprint=train_new_fingerprint, + ) + test_split = self.select( + indices=test_indices, + keep_in_memory=keep_in_memory, + indices_cache_file_name=test_indices_cache_file_name, + writer_batch_size=writer_batch_size, + new_fingerprint=test_new_fingerprint, + ) + + return DatasetDict({"train": train_split, "test": test_split}) + + def shard( + self, + num_shards: int, + index: int, + contiguous: bool = True, + keep_in_memory: bool = False, + indices_cache_file_name: Optional[str] = None, + writer_batch_size: Optional[int] = 1000, + ) -> "Dataset": + """Return the `index`-nth shard from dataset split into `num_shards` pieces. + + This shards deterministically. `dataset.shard(n, i)` splits the dataset into contiguous chunks, + so it can be easily concatenated back together after processing. If `len(dataset) % n == l`, then the + first `l` dataset each have length `(len(dataset) // n) + 1`, and the remaining dataset have length `(len(dataset) // n)`. + `datasets.concatenate_datasets([dset.shard(n, i) for i in range(n)])` returns a dataset with the same order as the original. + + Note: n should be less or equal to the number of elements in the dataset `len(dataset)`. + + On the other hand, `dataset.shard(n, i, contiguous=False)` contains all elements of the dataset whose index mod `n = i`. + + Be sure to shard before using any randomizing operator (such as `shuffle`). + It is best if the shard operator is used early in the dataset pipeline. + + Args: + num_shards (`int`): + How many shards to split the dataset into. + index (`int`): + Which shard to select and return. + contiguous: (`bool`, defaults to `True`): + Whether to select contiguous blocks of indices for shards. + keep_in_memory (`bool`, defaults to `False`): + Keep the dataset in memory instead of writing it to a cache file. + indices_cache_file_name (`str`, *optional*): + Provide the name of a path for the cache file. It is used to store the + indices of each shard instead of the automatically generated cache file name. + writer_batch_size (`int`, defaults to `1000`): + This only concerns the indices mapping. + Number of indices per write operation for the cache file writer. + This value is a good trade-off between memory usage during the processing, and processing speed. + Higher value makes the processing do fewer lookups, lower value consume less temporary memory while running `map`. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> ds + Dataset({ + features: ['text', 'label'], + num_rows: 1066 + }) + >>> ds.shard(num_shards=2, index=0) + Dataset({ + features: ['text', 'label'], + num_rows: 533 + }) + ``` + """ + if not 0 <= index < num_shards: + raise ValueError("index should be in [0, num_shards-1]") + if contiguous: + div = len(self) // num_shards + mod = len(self) % num_shards + start = div * index + min(index, mod) + end = start + div + (1 if index < mod else 0) + indices = range(start, end) + else: + indices = np.arange(index, len(self), num_shards) + + return self.select( + indices=indices, + keep_in_memory=keep_in_memory, + indices_cache_file_name=indices_cache_file_name, + writer_batch_size=writer_batch_size, + ) + + def to_csv( + self, + path_or_buf: Union[PathLike, BinaryIO], + batch_size: Optional[int] = None, + num_proc: Optional[int] = None, + storage_options: Optional[dict] = None, + **to_csv_kwargs, + ) -> int: + """Exports the dataset to csv + + Args: + path_or_buf (`PathLike` or `FileOrBuffer`): + Either a path to a file (e.g. `file.csv`), a remote URI (e.g. `hf://datasets/username/my_dataset_name/data.csv`), + or a BinaryIO, where the dataset will be saved to in the specified format. + batch_size (`int`, *optional*): + Size of the batch to load in memory and write at once. + Defaults to `datasets.config.DEFAULT_MAX_BATCH_SIZE`. + num_proc (`int`, *optional*): + Number of processes for multiprocessing. By default it doesn't + use multiprocessing. `batch_size` in this case defaults to + `datasets.config.DEFAULT_MAX_BATCH_SIZE` but feel free to make it 5x or 10x of the default + value if you have sufficient compute power. + storage_options (`dict`, *optional*): + Key/value pairs to be passed on to the file-system backend, if any. + + + **to_csv_kwargs (additional keyword arguments): + Parameters to pass to pandas's [`pandas.DataFrame.to_csv`](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.to_csv.html). + + + + Now, `index` defaults to `False` if not specified. + + If you would like to write the index, pass `index=True` and also set a name for the index column by + passing `index_label`. + + + + Returns: + `int`: The number of characters or bytes written. + + Example: + + ```py + >>> ds.to_csv("path/to/dataset/directory") + ``` + """ + # Dynamic import to avoid circular dependency + from .io.csv import CsvDatasetWriter + + return CsvDatasetWriter( + self, + path_or_buf, + batch_size=batch_size, + num_proc=num_proc, + storage_options=storage_options, + **to_csv_kwargs, + ).write() + + def to_dict(self, batch_size: Optional[int] = None) -> Union[dict, Iterator[dict]]: + """Returns the dataset as a Python dict. Can also return a generator for large datasets. + + Args: + batch_size (`int`, *optional*): The size (number of rows) of the batches if `batched` is `True`. + Defaults to `datasets.config.DEFAULT_MAX_BATCH_SIZE`. + + Returns: + `dict` or `Iterator[dict]` + + Example: + + ```py + >>> ds.to_dict() + ``` + """ + return query_table( + table=self._data, + key=slice(0, len(self)), + indices=self._indices, + ).to_pydict() + + def to_list(self) -> list: + """Returns the dataset as a Python list. + + Returns: + `list` + + Example: + + ```py + >>> ds.to_list() + ``` + """ + return query_table( + table=self._data, + key=slice(0, len(self)), + indices=self._indices, + ).to_pylist() + + def to_json( + self, + path_or_buf: Union[PathLike, BinaryIO], + batch_size: Optional[int] = None, + num_proc: Optional[int] = None, + storage_options: Optional[dict] = None, + **to_json_kwargs, + ) -> int: + """Export the dataset to JSON Lines or JSON. + + The default output format is [JSON Lines](https://jsonlines.org/). + To export to [JSON](https://www.json.org), pass `lines=False` argument and the desired `orient`. + + Args: + path_or_buf (`PathLike` or `FileOrBuffer`): + Either a path to a file (e.g. `file.json`), a remote URI (e.g. `hf://datasets/username/my_dataset_name/data.json`), + or a BinaryIO, where the dataset will be saved to in the specified format. + batch_size (`int`, *optional*): + Size of the batch to load in memory and write at once. + Defaults to `datasets.config.DEFAULT_MAX_BATCH_SIZE`. + num_proc (`int`, *optional*): + Number of processes for multiprocessing. By default, it doesn't + use multiprocessing. `batch_size` in this case defaults to + `datasets.config.DEFAULT_MAX_BATCH_SIZE` but feel free to make it 5x or 10x of the default + value if you have sufficient compute power. + storage_options (`dict`, *optional*): + Key/value pairs to be passed on to the file-system backend, if any. + + + **to_json_kwargs (additional keyword arguments): + Parameters to pass to pandas's [`pandas.DataFrame.to_json`](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.to_json.html). + Default arguments are `lines=True` and `orient="records". + + + + The parameter `index` defaults to `False` if `orient` is `"split"` or `"table"`. + + If you would like to write the index, pass `index=True`. + + + + Returns: + `int`: The number of characters or bytes written. + + Example: + + ```py + >>> ds.to_json("path/to/dataset/directory/filename.jsonl") + ``` + """ + # Dynamic import to avoid circular dependency + from .io.json import JsonDatasetWriter + + return JsonDatasetWriter( + self, + path_or_buf, + batch_size=batch_size, + num_proc=num_proc, + storage_options=storage_options, + **to_json_kwargs, + ).write() + + def to_pandas( + self, batch_size: Optional[int] = None, batched: bool = False + ) -> Union[pd.DataFrame, Iterator[pd.DataFrame]]: + """Returns the dataset as a `pandas.DataFrame`. Can also return a generator for large datasets. + + Args: + batched (`bool`): + Set to `True` to return a generator that yields the dataset as batches + of `batch_size` rows. Defaults to `False` (returns the whole datasets once). + batch_size (`int`, *optional*): + The size (number of rows) of the batches if `batched` is `True`. + Defaults to `datasets.config.DEFAULT_MAX_BATCH_SIZE`. + + Returns: + `pandas.DataFrame` or `Iterator[pandas.DataFrame]` + + Example: + + ```py + >>> ds.to_pandas() + ``` + """ + if not batched: + return query_table( + table=self._data, + key=slice(0, len(self)), + indices=self._indices, + ).to_pandas(types_mapper=pandas_types_mapper) + else: + batch_size = batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE + return ( + query_table( + table=self._data, + key=slice(offset, offset + batch_size), + indices=self._indices, + ).to_pandas(types_mapper=pandas_types_mapper) + for offset in range(0, len(self), batch_size) + ) + + def to_polars( + self, + batch_size: Optional[int] = None, + batched: bool = False, + schema_overrides: Optional[dict] = None, + rechunk: bool = True, + ) -> Union["pl.DataFrame", Iterator["pl.DataFrame"]]: + """Returns the dataset as a `polars.DataFrame`. Can also return a generator for large datasets. + + Args: + batched (`bool`): + Set to `True` to return a generator that yields the dataset as batches + of `batch_size` rows. Defaults to `False` (returns the whole datasets once). + batch_size (`int`, *optional*): + The size (number of rows) of the batches if `batched` is `True`. + Defaults to `genomicsml.datasets.config.DEFAULT_MAX_BATCH_SIZE`. + schema_overrides (`dict`, *optional*): + Support type specification or override of one or more columns; note that + any dtypes inferred from the schema param will be overridden. + rechunk (`bool`): + Make sure that all data is in contiguous memory. Defaults to `True`. + Returns: + `polars.DataFrame` or `Iterator[polars.DataFrame]` + + Example: + + ```py + >>> ds.to_polars() + ``` + """ + if config.POLARS_AVAILABLE: + import polars as pl + + if not batched: + return pl.from_arrow( + query_table( + table=self._data, + key=slice(0, len(self)), + indices=self._indices if self._indices is not None else None, + ), + schema_overrides=schema_overrides, + rechunk=rechunk, + ) + else: + batch_size = batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE + return ( + pl.from_arrow( + query_table( + table=self._data, + key=slice(offset, offset + batch_size), + indices=self._indices if self._indices is not None else None, + ), + schema_overrides=schema_overrides, + rechunk=rechunk, + ) + for offset in range(0, len(self), batch_size) + ) + else: + raise ValueError("Polars needs to be installed to be able to return Polars dataframes.") + + def to_parquet( + self, + path_or_buf: Union[PathLike, BinaryIO], + batch_size: Optional[int] = None, + storage_options: Optional[dict] = None, + **parquet_writer_kwargs, + ) -> int: + """Exports the dataset to parquet + + Args: + path_or_buf (`PathLike` or `FileOrBuffer`): + Either a path to a file (e.g. `file.parquet`), a remote URI (e.g. `hf://datasets/username/my_dataset_name/data.parquet`), + or a BinaryIO, where the dataset will be saved to in the specified format. + batch_size (`int`, *optional*): + Size of the batch to load in memory and write at once. + Defaults to `datasets.config.DEFAULT_MAX_BATCH_SIZE`. + storage_options (`dict`, *optional*): + Key/value pairs to be passed on to the file-system backend, if any. + + + **parquet_writer_kwargs (additional keyword arguments): + Parameters to pass to PyArrow's `pyarrow.parquet.ParquetWriter`. + + Returns: + `int`: The number of characters or bytes written. + + Example: + + ```py + >>> ds.to_parquet("path/to/dataset/directory") + ``` + """ + # Dynamic import to avoid circular dependency + from .io.parquet import ParquetDatasetWriter + + return ParquetDatasetWriter( + self, path_or_buf, batch_size=batch_size, storage_options=storage_options, **parquet_writer_kwargs + ).write() + + def to_sql( + self, + name: str, + con: Union[str, "sqlalchemy.engine.Connection", "sqlalchemy.engine.Engine", "sqlite3.Connection"], + batch_size: Optional[int] = None, + **sql_writer_kwargs, + ) -> int: + """Exports the dataset to a SQL database. + + Args: + name (`str`): + Name of SQL table. + con (`str` or `sqlite3.Connection` or `sqlalchemy.engine.Connection` or `sqlalchemy.engine.Connection`): + A [URI string](https://docs.sqlalchemy.org/en/13/core/engines.html#database-urls) or a SQLite3/SQLAlchemy connection object used to write to a database. + batch_size (`int`, *optional*): + Size of the batch to load in memory and write at once. + Defaults to `datasets.config.DEFAULT_MAX_BATCH_SIZE`. + **sql_writer_kwargs (additional keyword arguments): + Parameters to pass to pandas's [`pandas.DataFrame.to_sql`](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.to_sql.html). + + + + Now, `index` defaults to `False` if not specified. + + If you would like to write the index, pass `index=True` and also set a name for the index column by + passing `index_label`. + + + + Returns: + `int`: The number of records written. + + Example: + + ```py + >>> # con provided as a connection URI string + >>> ds.to_sql("data", "sqlite:///my_own_db.sql") + >>> # con provided as a sqlite3 connection object + >>> import sqlite3 + >>> con = sqlite3.connect("my_own_db.sql") + >>> with con: + ... ds.to_sql("data", con) + ``` + """ + # Dynamic import to avoid circular dependency + from .io.sql import SqlDatasetWriter + + return SqlDatasetWriter(self, name, con, batch_size=batch_size, **sql_writer_kwargs).write() + + def _estimate_nbytes(self) -> int: + dataset_nbytes = self.data.nbytes + + # Find decodable columns, because if there are any, we need to + # adjust the dataset size computation (needed for sharding) to account for possible external files + decodable_columns = [ + k for k, v in self._info.features.items() if require_decoding(v, ignore_decode_attribute=True) + ] + + if decodable_columns: + # Approximate the space needed to store the bytes from the external files by analyzing the first 1000 examples + extra_nbytes = 0 + + def extra_nbytes_visitor(array, feature): + nonlocal extra_nbytes + if isinstance(feature, (Audio, Image, Video)): + for x in array.to_pylist(): + if x is not None and x["bytes"] is None and x["path"] is not None: + size = xgetsize(x["path"]) + extra_nbytes += size + extra_nbytes -= array.field("path").nbytes + + table = self.with_format("arrow")[:1000] + table_visitor(table, extra_nbytes_visitor) + + extra_nbytes = extra_nbytes * len(self.data) / len(table) + dataset_nbytes = dataset_nbytes + extra_nbytes + + if self._indices is not None: + dataset_nbytes = dataset_nbytes * len(self._indices) / len(self.data) + return dataset_nbytes + + @staticmethod + def _generate_tables_from_shards(shards: list["Dataset"], batch_size: int): + for shard_idx, shard in enumerate(shards): + for pa_table in shard.with_format("arrow").iter(batch_size): + yield shard_idx, pa_table + + @staticmethod + def _generate_tables_from_cache_file(filename: str): + for batch_idx, batch in enumerate(_memory_mapped_record_batch_reader_from_file(filename)): + yield batch_idx, pa.Table.from_batches([batch]) + + def to_iterable_dataset(self, num_shards: Optional[int] = 1) -> "IterableDataset": + """Get an [`datasets.IterableDataset`] from a map-style [`datasets.Dataset`]. + This is equivalent to loading a dataset in streaming mode with [`datasets.load_dataset`], but much faster since the data is streamed from local files. + + Contrary to map-style datasets, iterable datasets are lazy and can only be iterated over (e.g. using a for loop). + Since they are read sequentially in training loops, iterable datasets are much faster than map-style datasets. + All the transformations applied to iterable datasets like filtering or processing are done on-the-fly when you start iterating over the dataset. + + Still, it is possible to shuffle an iterable dataset using [`datasets.IterableDataset.shuffle`]. + This is a fast approximate shuffling that works best if you have multiple shards and if you specify a buffer size that is big enough. + + To get the best speed performance, make sure your dataset doesn't have an indices mapping. + If this is the case, the data are not read contiguously, which can be slow sometimes. + You can use `ds = ds.flatten_indices()` to write your dataset in contiguous chunks of data and have optimal speed before switching to an iterable dataset. + + Args: + num_shards (`int`, default to `1`): + Number of shards to define when instantiating the iterable dataset. This is especially useful for big datasets to be able to shuffle properly, + and also to enable fast parallel loading using a PyTorch DataLoader or in distributed setups for example. + Shards are defined using [`datasets.Dataset.shard`]: it simply slices the data without writing anything on disk. + + Returns: + [`datasets.IterableDataset`] + + Example: + + Basic usage: + ```python + >>> ids = ds.to_iterable_dataset() + >>> for example in ids: + ... pass + ``` + + With lazy filtering and processing: + ```python + >>> ids = ds.to_iterable_dataset() + >>> ids = ids.filter(filter_fn).map(process_fn) # will filter and process on-the-fly when you start iterating over the iterable dataset + >>> for example in ids: + ... pass + ``` + + With sharding to enable efficient shuffling: + ```python + >>> ids = ds.to_iterable_dataset(num_shards=64) # the dataset is split into 64 shards to be iterated over + >>> ids = ids.shuffle(buffer_size=10_000) # will shuffle the shards order and use a shuffle buffer for fast approximate shuffling when you start iterating + >>> for example in ids: + ... pass + ``` + + With a PyTorch DataLoader: + ```python + >>> import torch + >>> ids = ds.to_iterable_dataset(num_shards=64) + >>> ids = ids.filter(filter_fn).map(process_fn) + >>> dataloader = torch.utils.data.DataLoader(ids, num_workers=4) # will assign 64 / 4 = 16 shards to each worker to load, filter and process when you start iterating + >>> for example in ids: + ... pass + ``` + + With a PyTorch DataLoader and shuffling: + ```python + >>> import torch + >>> ids = ds.to_iterable_dataset(num_shards=64) + >>> ids = ids.shuffle(buffer_size=10_000) # will shuffle the shards order and use a shuffle buffer when you start iterating + >>> dataloader = torch.utils.data.DataLoader(ids, num_workers=4) # will assign 64 / 4 = 16 shards from the shuffled list of shards to each worker when you start iterating + >>> for example in ids: + ... pass + ``` + + In a distributed setup like PyTorch DDP with a PyTorch DataLoader and shuffling + ```python + >>> from datasets.distributed import split_dataset_by_node + >>> ids = ds.to_iterable_dataset(num_shards=512) + >>> ids = ids.shuffle(buffer_size=10_000, seed=42) # will shuffle the shards order and use a shuffle buffer when you start iterating + >>> ids = split_dataset_by_node(ds, world_size=8, rank=0) # will keep only 512 / 8 = 64 shards from the shuffled lists of shards when you start iterating + >>> dataloader = torch.utils.data.DataLoader(ids, num_workers=4) # will assign 64 / 4 = 16 shards from this node's list of shards to each worker when you start iterating + >>> for example in ids: + ... pass + ``` + + With shuffling and multiple epochs: + ```python + >>> ids = ds.to_iterable_dataset(num_shards=64) + >>> ids = ids.shuffle(buffer_size=10_000, seed=42) # will shuffle the shards order and use a shuffle buffer when you start iterating + >>> for epoch in range(n_epochs): + ... ids.set_epoch(epoch) # will use effective_seed = seed + epoch to shuffle the shards and for the shuffle buffer when you start iterating + ... for example in ids: + ... pass + ``` + Feel free to also use [`IterableDataset.set_epoch`] when using a PyTorch DataLoader or in distributed setups. + """ + from .iterable_dataset import ArrowExamplesIterable, IterableDataset + + if self._format_type is not None: + if self._format_kwargs or ( + self._format_columns is not None and set(self._format_columns) != set(self.column_names) + ): + raise NotImplementedError( + "Converting a formatted dataset with kwargs or selected columns to a formatted iterable dataset is not implemented yet. Please run `my_dataset = my_dataset.with_format(None)` before calling to_iterable_dataset" + ) + if num_shards > len(self): + raise ValueError( + f"Unable to shard a dataset of size {len(self)} into {num_shards} shards (the number of shards exceeds the number of samples)." + ) + if self._indices is not None: + logger.info( + "Converting an Arrow dataset to iterable but it has an indices mapping that can make it slower. " + "You can use `ds = ds.flatten_indices()` to write your dataset in contiguous chunks of data and have optimal speed." + ) + shards = ( + [copy.deepcopy(self)] + if num_shards == 1 + else [ + self.shard(num_shards=num_shards, index=shard_idx, contiguous=True) for shard_idx in range(num_shards) + ] + ) + ex_iterable = ArrowExamplesIterable( + Dataset._generate_tables_from_shards, + kwargs={"shards": shards, "batch_size": config.DEFAULT_MAX_BATCH_SIZE}, + ) + ds = IterableDataset(ex_iterable, info=DatasetInfo(features=self.features)) + if self._format_type: + ds = ds.with_format(self._format_type) + return ds + + def _push_parquet_shards_to_hub( + self, + repo_id: str, + data_dir: str = "data", + split: Optional[str] = None, + token: Optional[str] = None, + revision: Optional[str] = None, + create_pr: Optional[bool] = False, + max_shard_size: Optional[Union[int, str]] = None, + num_shards: Optional[int] = None, + embed_external_files: bool = True, + ) -> tuple[list[CommitOperationAdd], int, int]: + """Pushes the dataset shards as Parquet files to the hub. + + Returns: + additions (`List[CommitOperation]`): list of the `CommitOperationAdd` of the uploaded shards + uploaded_size (`int`): number of uploaded bytes to the repository + dataset_nbytes (`int`): approximate size in bytes of the uploaded dataset after uncompression + """ + # Find decodable columns, because if there are any, we need to: + # embed the bytes from the files in the shards + decodable_columns = ( + [k for k, v in self._info.features.items() if require_decoding(v, ignore_decode_attribute=True)] + if embed_external_files + else [] + ) + + dataset_nbytes = self._estimate_nbytes() + + if num_shards is None: + max_shard_size = convert_file_size_to_int(max_shard_size or config.MAX_SHARD_SIZE) + num_shards = int(dataset_nbytes / max_shard_size) + 1 + num_shards = max(num_shards, 1) + + shards = (self.shard(num_shards=num_shards, index=i, contiguous=True) for i in range(num_shards)) + + if decodable_columns: + from .io.parquet import get_writer_batch_size + + def shards_with_embedded_external_files(shards: Iterator[Dataset]) -> Iterator[Dataset]: + for shard in shards: + format = shard.format + shard = shard.with_format("arrow") + shard = shard.map( + embed_table_storage, + batched=True, + batch_size=get_writer_batch_size(shard.features), + keep_in_memory=True, + ) + shard = shard.with_format(**format) + yield shard + + shards = shards_with_embedded_external_files(shards) + + api = HfApi(endpoint=config.HF_ENDPOINT, token=token) + + uploaded_size = 0 + additions: list[CommitOperationAdd] = [] + for index, shard in hf_tqdm( + enumerate(shards), + desc="Uploading the dataset shards", + total=num_shards, + ): + shard_path_in_repo = f"{data_dir}/{split}-{index:05d}-of-{num_shards:05d}.parquet" + buffer = BytesIO() + shard.to_parquet(buffer) + parquet_content = buffer.getvalue() + uploaded_size += len(parquet_content) + del buffer + shard_addition = CommitOperationAdd(path_in_repo=shard_path_in_repo, path_or_fileobj=parquet_content) + api.preupload_lfs_files( + repo_id=repo_id, + additions=[shard_addition], + repo_type="dataset", + revision=revision, + create_pr=create_pr, + ) + additions.append(shard_addition) + + return additions, uploaded_size, dataset_nbytes + + def push_to_hub( + self, + repo_id: str, + config_name: str = "default", + set_default: Optional[bool] = None, + split: Optional[str] = None, + data_dir: Optional[str] = None, + commit_message: Optional[str] = None, + commit_description: Optional[str] = None, + private: Optional[bool] = None, + token: Optional[str] = None, + revision: Optional[str] = None, + create_pr: Optional[bool] = False, + max_shard_size: Optional[Union[int, str]] = None, + num_shards: Optional[int] = None, + embed_external_files: bool = True, + ) -> CommitInfo: + """Pushes the dataset to the hub as a Parquet dataset. + The dataset is pushed using HTTP requests and does not need to have neither git or git-lfs installed. + + The resulting Parquet files are self-contained by default. If your dataset contains [`Image`], [`Audio`] or [`Video`] + data, the Parquet files will store the bytes of your images or audio files. + You can disable this by setting `embed_external_files` to `False`. + + Args: + repo_id (`str`): + The ID of the repository to push to in the following format: `/` or + `/`. Also accepts ``, which will default to the namespace + of the logged-in user. + config_name (`str`, defaults to "default"): + The configuration name (or subset) of a dataset. Defaults to "default". + set_default (`bool`, *optional*): + Whether to set this configuration as the default one. Otherwise, the default configuration is the one + named "default". + split (`str`, *optional*): + The name of the split that will be given to that dataset. Defaults to `self.split`. + data_dir (`str`, *optional*): + Directory name that will contain the uploaded data files. Defaults to the `config_name` if different + from "default", else "data". + + + commit_message (`str`, *optional*): + Message to commit while pushing. Will default to `"Upload dataset"`. + commit_description (`str`, *optional*): + Description of the commit that will be created. + Additionally, description of the PR if a PR is created (`create_pr` is True). + + + private (`bool`, *optional*): + Whether to make the repo private. If `None` (default), the repo will be public unless the + organization's default is private. This value is ignored if the repo already exists. + token (`str`, *optional*): + An optional authentication token for the Hugging Face Hub. If no token is passed, will default + to the token saved locally when logging in with `huggingface-cli login`. Will raise an error + if no token is passed and the user is not logged-in. + revision (`str`, *optional*): + Branch to push the uploaded files to. Defaults to the `"main"` branch. + + + create_pr (`bool`, *optional*, defaults to `False`): + Whether to create a PR with the uploaded files or directly commit. + + + max_shard_size (`int` or `str`, *optional*, defaults to `"500MB"`): + The maximum size of the dataset shards to be uploaded to the hub. If expressed as a string, needs to be digits followed by + a unit (like `"5MB"`). + num_shards (`int`, *optional*): + Number of shards to write. By default, the number of shards depends on `max_shard_size`. + + + embed_external_files (`bool`, defaults to `True`): + Whether to embed file bytes in the shards. + In particular, this will do the following before the push for the fields of type: + + - [`Audio`] and [`Image`]: remove local path information and embed file content in the Parquet files. + + Return: + huggingface_hub.CommitInfo + + Example: + + ```python + >>> dataset.push_to_hub("/") + >>> dataset_dict.push_to_hub("/", private=True) + >>> dataset.push_to_hub("/", max_shard_size="1GB") + >>> dataset.push_to_hub("/", num_shards=1024) + ``` + + If your dataset has multiple splits (e.g. train/validation/test): + + ```python + >>> train_dataset.push_to_hub("/", split="train") + >>> val_dataset.push_to_hub("/", split="validation") + >>> # later + >>> dataset = load_dataset("/") + >>> train_dataset = dataset["train"] + >>> val_dataset = dataset["validation"] + ``` + + If you want to add a new configuration (or subset) to a dataset (e.g. if the dataset has multiple tasks/versions/languages): + + ```python + >>> english_dataset.push_to_hub("/", "en") + >>> french_dataset.push_to_hub("/", "fr") + >>> # later + >>> english_dataset = load_dataset("/", "en") + >>> french_dataset = load_dataset("/", "fr") + ``` + """ + if "Video(" in str(self.features): + raise NotImplementedError( + "push_to_hub is not implemented for video datasets, instead you should upload the video files " + "using e.g. the huggingface_hub library and optionally upload a metadata.csv or metadata.jsonl " + "file containing other information like video captions, features or labels. More information " + "at https://huggingface.co/docs/datasets/main/en/video_load#videofolder" + ) + if config_name == "data": + raise ValueError("`config_name` cannot be 'data'. Please, choose another name for configuration.") + + if max_shard_size is not None and num_shards is not None: + raise ValueError( + "Failed to push_to_hub: please specify either max_shard_size or num_shards, but not both." + ) + + if split is None: + split = str(self.split) if self.split is not None else "train" + + if not re.match(_split_re, split): + raise ValueError(f"Split name should match '{_split_re}' but got '{split}'.") + + api = HfApi(endpoint=config.HF_ENDPOINT, token=token) + + repo_url = api.create_repo( + repo_id, + token=token, + repo_type="dataset", + private=private, + exist_ok=True, + ) + repo_id = repo_url.repo_id + + if revision is not None and not revision.startswith("refs/pr/"): + # We do not call create_branch for a PR reference: 400 Bad Request + api.create_branch(repo_id, branch=revision, token=token, repo_type="dataset", exist_ok=True) + + if not data_dir: + data_dir = config_name if config_name != "default" else "data" # for backward compatibility + + additions, uploaded_size, dataset_nbytes = self._push_parquet_shards_to_hub( + repo_id=repo_id, + data_dir=data_dir, + split=split, + token=token, + revision=revision, + max_shard_size=max_shard_size, + num_shards=num_shards, + create_pr=create_pr, + embed_external_files=embed_external_files, + ) + + # Check if the repo already has a README.md and/or a dataset_infos.json to update them with the new split info (size and pattern) + # and delete old split shards (if they exist) + repo_with_dataset_card, repo_with_dataset_infos = False, False + deletions: list[CommitOperationDelete] = [] + deleted_size = 0 + repo_splits: list[str] = [] # use a list to keep the order of the splits + repo_files_to_add = [addition.path_in_repo for addition in additions] + for repo_file in api.list_repo_tree( + repo_id=repo_id, revision=revision, repo_type="dataset", token=token, recursive=True + ): + if not isinstance(repo_file, RepoFile): + continue + if repo_file.rfilename == config.REPOCARD_FILENAME: + repo_with_dataset_card = True + elif repo_file.rfilename == config.DATASETDICT_INFOS_FILENAME: + repo_with_dataset_infos = True + elif ( + repo_file.rfilename.startswith(f"{data_dir}/{split}-") and repo_file.rfilename not in repo_files_to_add + ): + deletions.append(CommitOperationDelete(path_in_repo=repo_file.rfilename)) + deleted_size += repo_file.size + elif fnmatch.fnmatch( + repo_file.rfilename, PUSH_TO_HUB_WITHOUT_METADATA_CONFIGS_SPLIT_PATTERN_SHARDED.replace("{split}", "*") + ): + pattern = glob_pattern_to_regex(PUSH_TO_HUB_WITHOUT_METADATA_CONFIGS_SPLIT_PATTERN_SHARDED) + split_pattern_fields = string_to_dict(repo_file.rfilename, pattern) + assert split_pattern_fields is not None + repo_split = split_pattern_fields["split"] + if repo_split not in repo_splits: + repo_splits.append(repo_split) + + organization, dataset_name = repo_id.split("/") if "/" in repo_id else (None, repo_id) + info_to_dump = self.info.copy() + info_to_dump.download_checksums = None + info_to_dump.download_size = uploaded_size + info_to_dump.dataset_size = dataset_nbytes + info_to_dump.size_in_bytes = uploaded_size + dataset_nbytes + info_to_dump.config_name = config_name + info_to_dump.splits = SplitDict( + {split: SplitInfo(split, num_bytes=dataset_nbytes, num_examples=len(self), dataset_name=dataset_name)} + ) + # get the info from the README to update them + if repo_with_dataset_card: + dataset_card_path = api.hf_hub_download( + repo_id, config.REPOCARD_FILENAME, repo_type="dataset", revision=revision + ) + dataset_card = DatasetCard.load(Path(dataset_card_path)) + dataset_card_data = dataset_card.data + metadata_configs = MetadataConfigs.from_dataset_card_data(dataset_card_data) + dataset_infos: DatasetInfosDict = DatasetInfosDict.from_dataset_card_data(dataset_card_data) + if dataset_infos and config_name in dataset_infos: + repo_info = dataset_infos[config_name] + else: + repo_info = None + # get the deprecated dataset_infos.json to update them + elif repo_with_dataset_infos: + dataset_card = None + dataset_card_data = DatasetCardData() + metadata_configs = MetadataConfigs() + dataset_infos_path = api.hf_hub_download( + repo_id, config.DATASETDICT_INFOS_FILENAME, repo_type="dataset", revision=revision + ) + with open(dataset_infos_path, encoding="utf-8") as f: + dataset_infos: dict = json.load(f) + dataset_info = dataset_infos.get(config_name, None) if dataset_infos else None + repo_info = DatasetInfo.from_dict(dataset_info) if dataset_info else None + else: + dataset_card = None + dataset_card_data = DatasetCardData() + metadata_configs = MetadataConfigs() + repo_info = None + # update the total info to dump from existing info + if repo_info is not None: + logger.info("Updating downloaded metadata with the new split.") + if repo_info.splits and list(repo_info.splits) != [split]: + if self._info.features != repo_info.features: + raise ValueError( + f"Features of the new split don't match the features of the existing splits on the hub: {self._info.features} != {repo_info.features}" + ) + + if split in repo_info.splits: + repo_info.download_size -= deleted_size + repo_info.dataset_size -= repo_info.splits.get(split, SplitInfo()).num_bytes or 0 + + repo_info.download_checksums = None + repo_info.download_size = (repo_info.download_size or 0) + uploaded_size + repo_info.dataset_size = (repo_info.dataset_size or 0) + dataset_nbytes + repo_info.size_in_bytes = repo_info.download_size + repo_info.dataset_size + repo_info.splits.pop(split, None) + repo_info.splits[split] = SplitInfo( + split, num_bytes=dataset_nbytes, num_examples=len(self), dataset_name=dataset_name + ) + info_to_dump = repo_info + # create the metadata configs if it was uploaded with push_to_hub before metadata configs existed + if not metadata_configs and repo_splits: + default_metadata_configs_to_dump = { + "data_files": [{"split": split, "path": f"data/{split}-*"} for split in repo_splits] + } + MetadataConfigs({"default": default_metadata_configs_to_dump}).to_dataset_card_data(dataset_card_data) + # update the metadata configs + if config_name in metadata_configs: + metadata_config = metadata_configs[config_name] + if "data_files" in metadata_config: + data_files_to_dump = sanitize_patterns(metadata_config["data_files"]) + else: + data_files_to_dump = {} + # add the new split + data_files_to_dump[split] = [f"{data_dir}/{split}-*"] + metadata_config_to_dump = { + "data_files": [ + { + "split": _split, + "path": _pattern[0] if len(_pattern) == 1 else _pattern, + } + for _split, _pattern in data_files_to_dump.items() + ] + } + else: + metadata_config_to_dump = {"data_files": [{"split": split, "path": f"{data_dir}/{split}-*"}]} + if set_default and config_name != "default": + if metadata_configs: + default_config_name = metadata_configs.get_default_config_name() + if default_config_name == "default": + raise ValueError( + "There exists a configuration named 'default'. To set a different configuration as default, " + "rename the 'default' one first." + ) + else: + _ = metadata_configs[default_config_name].pop("default") + metadata_config_to_dump["default"] = True + # push to the deprecated dataset_infos.json + if repo_with_dataset_infos: + dataset_infos_path = api.hf_hub_download( + repo_id, config.DATASETDICT_INFOS_FILENAME, repo_type="dataset", revision=revision + ) + with open(dataset_infos_path, encoding="utf-8") as f: + dataset_infos: dict = json.load(f) + dataset_infos[config_name] = asdict(info_to_dump) + additions.append( + CommitOperationAdd( + path_in_repo=config.DATASETDICT_INFOS_FILENAME, + path_or_fileobj=json.dumps(dataset_infos, indent=4).encode("utf-8"), + ) + ) + # push to README + DatasetInfosDict({config_name: info_to_dump}).to_dataset_card_data(dataset_card_data) + MetadataConfigs({config_name: metadata_config_to_dump}).to_dataset_card_data(dataset_card_data) + dataset_card = DatasetCard(f"---\n{dataset_card_data}\n---\n") if dataset_card is None else dataset_card + additions.append( + CommitOperationAdd(path_in_repo=config.REPOCARD_FILENAME, path_or_fileobj=str(dataset_card).encode()) + ) + + commit_message = commit_message if commit_message is not None else "Upload dataset" + if len(additions) <= config.UPLOADS_MAX_NUMBER_PER_COMMIT: + commit_info = api.create_commit( + repo_id, + operations=additions + deletions, + commit_message=commit_message, + commit_description=commit_description, + token=token, + repo_type="dataset", + revision=revision, + create_pr=create_pr, + ) + else: + logger.info( + f"Number of files to upload is larger than {config.UPLOADS_MAX_NUMBER_PER_COMMIT}. Splitting the push into multiple commits." + ) + num_commits = math.ceil(len(additions) / config.UPLOADS_MAX_NUMBER_PER_COMMIT) + for i in range(0, num_commits): + operations = additions[ + i * config.UPLOADS_MAX_NUMBER_PER_COMMIT : (i + 1) * config.UPLOADS_MAX_NUMBER_PER_COMMIT + ] + (deletions if i == 0 else []) + commit_info = api.create_commit( + repo_id, + operations=operations, + commit_message=commit_message + f" (part {i:05d}-of-{num_commits:05d})", + commit_description=commit_description, + token=token, + repo_type="dataset", + revision=revision, + create_pr=create_pr, + ) + logger.info( + f"Commit #{i + 1} completed" + + (f" (still {num_commits - i - 1} to go)" if num_commits - i - 1 else "") + + "." + ) + return commit_info + + @transmit_format + @fingerprint_transform(inplace=False) + def add_column( + self, name: str, column: Union[list, np.array], new_fingerprint: str, feature: Optional[FeatureType] = None + ): + """Add column to Dataset. + + + + Args: + name (`str`): + Column name. + column (`list` or `np.array`): + Column data to be added. + feature (`FeatureType` or `None`, defaults to `None`): + Column datatype. + + Returns: + [`Dataset`] + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> more_text = ds["text"] + >>> ds.add_column(name="text_2", column=more_text) + Dataset({ + features: ['text', 'label', 'text_2'], + num_rows: 1066 + }) + ``` + """ + + if feature: + pyarrow_schema = Features({name: feature}).arrow_schema + else: + pyarrow_schema = None + + column_table = InMemoryTable.from_pydict({name: column}, schema=pyarrow_schema) + _check_column_names(self._data.column_names + column_table.column_names) + dataset = self.flatten_indices() if self._indices is not None else self + # Concatenate tables horizontally + table = concat_tables([dataset._data, column_table], axis=1) + # Update features + info = dataset.info.copy() + info.features.update(Features.from_arrow_schema(column_table.schema)) + table = update_metadata_with_features(table, info.features) + return Dataset(table, info=info, split=self.split, indices_table=None, fingerprint=new_fingerprint) + + def add_faiss_index( + self, + column: str, + index_name: Optional[str] = None, + device: Optional[int] = None, + string_factory: Optional[str] = None, + metric_type: Optional[int] = None, + custom_index: Optional["faiss.Index"] = None, # noqa: F821 + batch_size: int = 1000, + train_size: Optional[int] = None, + faiss_verbose: bool = False, + dtype=np.float32, + ): + """Add a dense index using Faiss for fast retrieval. + By default the index is done over the vectors of the specified column. + You can specify `device` if you want to run it on GPU (`device` must be the GPU index). + You can find more information about Faiss here: + + - For [string factory](https://github.com/facebookresearch/faiss/wiki/The-index-factory) + + Args: + column (`str`): + The column of the vectors to add to the index. + index_name (`str`, *optional*): + The `index_name`/identifier of the index. + This is the `index_name` that is used to call [`~datasets.Dataset.get_nearest_examples`] or [`~datasets.Dataset.search`]. + By default it corresponds to `column`. + device (`Union[int, List[int]]`, *optional*): + If positive integer, this is the index of the GPU to use. If negative integer, use all GPUs. + If a list of positive integers is passed in, run only on those GPUs. By default it uses the CPU. + string_factory (`str`, *optional*): + This is passed to the index factory of Faiss to create the index. + Default index class is `IndexFlat`. + metric_type (`int`, *optional*): + Type of metric. Ex: `faiss.METRIC_INNER_PRODUCT` or `faiss.METRIC_L2`. + custom_index (`faiss.Index`, *optional*): + Custom Faiss index that you already have instantiated and configured for your needs. + batch_size (`int`): + Size of the batch to use while adding vectors to the `FaissIndex`. Default value is `1000`. + + train_size (`int`, *optional*): + If the index needs a training step, specifies how many vectors will be used to train the index. + faiss_verbose (`bool`, defaults to `False`): + Enable the verbosity of the Faiss index. + dtype (`data-type`): + The dtype of the numpy arrays that are indexed. + Default is `np.float32`. + + Example: + + ```python + >>> ds = datasets.load_dataset('crime_and_punish', split='train') + >>> ds_with_embeddings = ds.map(lambda example: {'embeddings': embed(example['line']})) + >>> ds_with_embeddings.add_faiss_index(column='embeddings') + >>> # query + >>> scores, retrieved_examples = ds_with_embeddings.get_nearest_examples('embeddings', embed('my new query'), k=10) + >>> # save index + >>> ds_with_embeddings.save_faiss_index('embeddings', 'my_index.faiss') + + >>> ds = datasets.load_dataset('crime_and_punish', split='train') + >>> # load index + >>> ds.load_faiss_index('embeddings', 'my_index.faiss') + >>> # query + >>> scores, retrieved_examples = ds.get_nearest_examples('embeddings', embed('my new query'), k=10) + ``` + """ + with self.formatted_as(type="numpy", columns=[column], dtype=dtype): + super().add_faiss_index( + column=column, + index_name=index_name, + device=device, + string_factory=string_factory, + metric_type=metric_type, + custom_index=custom_index, + batch_size=batch_size, + train_size=train_size, + faiss_verbose=faiss_verbose, + ) + return self + + def add_faiss_index_from_external_arrays( + self, + external_arrays: np.array, + index_name: str, + device: Optional[int] = None, + string_factory: Optional[str] = None, + metric_type: Optional[int] = None, + custom_index: Optional["faiss.Index"] = None, # noqa: F821 + batch_size: int = 1000, + train_size: Optional[int] = None, + faiss_verbose: bool = False, + dtype=np.float32, + ): + """Add a dense index using Faiss for fast retrieval. + The index is created using the vectors of `external_arrays`. + You can specify `device` if you want to run it on GPU (`device` must be the GPU index). + You can find more information about Faiss here: + + - For [string factory](https://github.com/facebookresearch/faiss/wiki/The-index-factory) + + Args: + external_arrays (`np.array`): + If you want to use arrays from outside the lib for the index, you can set `external_arrays`. + It will use `external_arrays` to create the Faiss index instead of the arrays in the given `column`. + index_name (`str`): + The `index_name`/identifier of the index. + This is the `index_name` that is used to call [`~datasets.Dataset.get_nearest_examples`] or [`~datasets.Dataset.search`]. + device (Optional `Union[int, List[int]]`, *optional*): + If positive integer, this is the index of the GPU to use. If negative integer, use all GPUs. + If a list of positive integers is passed in, run only on those GPUs. By default it uses the CPU. + string_factory (`str`, *optional*): + This is passed to the index factory of Faiss to create the index. + Default index class is `IndexFlat`. + metric_type (`int`, *optional*): + Type of metric. Ex: `faiss.faiss.METRIC_INNER_PRODUCT` or `faiss.METRIC_L2`. + custom_index (`faiss.Index`, *optional*): + Custom Faiss index that you already have instantiated and configured for your needs. + batch_size (`int`, *optional*): + Size of the batch to use while adding vectors to the FaissIndex. Default value is 1000. + + train_size (`int`, *optional*): + If the index needs a training step, specifies how many vectors will be used to train the index. + faiss_verbose (`bool`, defaults to False): + Enable the verbosity of the Faiss index. + dtype (`numpy.dtype`): + The dtype of the numpy arrays that are indexed. Default is np.float32. + """ + super().add_faiss_index_from_external_arrays( + external_arrays=external_arrays.astype(dtype), + index_name=index_name, + device=device, + string_factory=string_factory, + metric_type=metric_type, + custom_index=custom_index, + batch_size=batch_size, + train_size=train_size, + faiss_verbose=faiss_verbose, + ) + + def add_elasticsearch_index( + self, + column: str, + index_name: Optional[str] = None, + host: Optional[str] = None, + port: Optional[int] = None, + es_client: Optional["elasticsearch.Elasticsearch"] = None, # noqa: F821 + es_index_name: Optional[str] = None, + es_index_config: Optional[dict] = None, + ): + """Add a text index using ElasticSearch for fast retrieval. This is done in-place. + + Args: + column (`str`): + The column of the documents to add to the index. + index_name (`str`, *optional*): + The `index_name`/identifier of the index. + This is the index name that is used to call [`~Dataset.get_nearest_examples`] or [`~Dataset.search`]. + By default it corresponds to `column`. + host (`str`, *optional*, defaults to `localhost`): + Host of where ElasticSearch is running. + port (`str`, *optional*, defaults to `9200`): + Port of where ElasticSearch is running. + es_client (`elasticsearch.Elasticsearch`, *optional*): + The elasticsearch client used to create the index if host and port are `None`. + es_index_name (`str`, *optional*): + The elasticsearch index name used to create the index. + es_index_config (`dict`, *optional*): + The configuration of the elasticsearch index. + Default config is: + ``` + { + "settings": { + "number_of_shards": 1, + "analysis": {"analyzer": {"stop_standard": {"type": "standard", " stopwords": "_english_"}}}, + }, + "mappings": { + "properties": { + "text": { + "type": "text", + "analyzer": "standard", + "similarity": "BM25" + }, + } + }, + } + ``` + Example: + + ```python + >>> es_client = elasticsearch.Elasticsearch() + >>> ds = datasets.load_dataset('crime_and_punish', split='train') + >>> ds.add_elasticsearch_index(column='line', es_client=es_client, es_index_name="my_es_index") + >>> scores, retrieved_examples = ds.get_nearest_examples('line', 'my new query', k=10) + ``` + """ + with self.formatted_as(type=None, columns=[column]): + super().add_elasticsearch_index( + column=column, + index_name=index_name, + host=host, + port=port, + es_client=es_client, + es_index_name=es_index_name, + es_index_config=es_index_config, + ) + return self + + @transmit_format + @fingerprint_transform(inplace=False) + def add_item(self, item: dict, new_fingerprint: str): + """Add item to Dataset. + + + + Args: + item (`dict`): + Item data to be added. + + Returns: + [`Dataset`] + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> new_review = {'label': 0, 'text': 'this movie is the absolute worst thing I have ever seen'} + >>> ds = ds.add_item(new_review) + >>> ds[-1] + {'label': 0, 'text': 'this movie is the absolute worst thing I have ever seen'} + ``` + """ + item_table = InMemoryTable.from_pydict({k: [v] for k, v in item.items()}) + # We don't call _check_if_features_can_be_aligned here so this cast is "unsafe" + dset_features, item_features = _align_features( + [self._info.features, Features.from_arrow_schema(item_table.schema)] + ) + # Cast to align the schemas of the tables and concatenate the tables + table = concat_tables( + [ + self._data.cast(dset_features.arrow_schema) if self._info.features != dset_features else self._data, + item_table.cast(item_features.arrow_schema), + ] + ) + if self._indices is None: + indices_table = None + else: + item_indices_array = pa.array([len(self._data)], type=pa.uint64()) + item_indices_table = InMemoryTable.from_arrays([item_indices_array], names=["indices"]) + indices_table = concat_tables([self._indices, item_indices_table]) + info = self.info.copy() + info.features.update(item_features) + table = update_metadata_with_features(table, info.features) + return Dataset( + table, + info=info, + split=self.split, + indices_table=indices_table, + fingerprint=new_fingerprint, + ) + + def align_labels_with_mapping(self, label2id: dict, label_column: str) -> "Dataset": + """Align the dataset's label ID and label name mapping to match an input `label2id` mapping. + This is useful when you want to ensure that a model's predicted labels are aligned with the dataset. + The alignment in done using the lowercase label names. + + Args: + label2id (`dict`): + The label name to ID mapping to align the dataset with. + label_column (`str`): + The column name of labels to align on. + + Example: + + ```python + >>> # dataset with mapping {'entailment': 0, 'neutral': 1, 'contradiction': 2} + >>> ds = load_dataset("nyu-mll/glue", "mnli", split="train") + >>> # mapping to align with + >>> label2id = {'CONTRADICTION': 0, 'NEUTRAL': 1, 'ENTAILMENT': 2} + >>> ds_aligned = ds.align_labels_with_mapping(label2id, "label") + ``` + + """ + # Sanity checks + if label_column not in self._data.column_names: + raise ValueError(f"Column ({label_column}) not in table columns ({self._data.column_names}).") + + label_feature = self._info.features[label_column] + if not ( + isinstance(label_feature, ClassLabel) + or (isinstance(label_feature, Sequence) and isinstance(label_feature.feature, ClassLabel)) + ): + raise ValueError( + f"Aligning labels with a mapping is only supported for {ClassLabel.__name__} column or {Sequence.__name__} column with the inner type {ClassLabel.__name__}, and column {label_feature} is of type {type(label_feature).__name__}." + ) + + # Sort input mapping by ID value to ensure the label names are aligned + label2id = dict(sorted(label2id.items(), key=lambda item: item[1])) + label_names = list(label2id.keys()) + # Some label mappings use uppercase label names so we lowercase them during alignment + label2id = {k.lower(): v for k, v in label2id.items()} + int2str_function = ( + label_feature.int2str if isinstance(label_feature, ClassLabel) else label_feature.feature.int2str + ) + + if isinstance(label_feature, ClassLabel): + + def process_label_ids(batch): + dset_label_names = [ + int2str_function(label_id).lower() if label_id is not None else None + for label_id in batch[label_column] + ] + batch[label_column] = [ + label2id[label_name] if label_name is not None else None for label_name in dset_label_names + ] + return batch + + else: + + def process_label_ids(batch): + dset_label_names = [ + [int2str_function(label_id).lower() if label_id is not None else None for label_id in seq] + for seq in batch[label_column] + ] + batch[label_column] = [ + [label2id[label_name] if label_name is not None else None for label_name in seq] + for seq in dset_label_names + ] + return batch + + features = self.features + features[label_column] = ( + ClassLabel(num_classes=len(label_names), names=label_names) + if isinstance(label_feature, ClassLabel) + else Sequence(ClassLabel(num_classes=len(label_names), names=label_names)) + ) + return self.map(process_label_ids, features=features, batched=True, desc="Aligning the labels") + + +def _concatenate_map_style_datasets( + dsets: list[Dataset], + info: Optional[DatasetInfo] = None, + split: Optional[NamedSplit] = None, + axis: int = 0, +): + """ + Converts a list of :class:`Dataset` with the same schema into a single :class:`Dataset`. + When you concatenate on axis 0, missing data are filled with None values. + + Args: + dsets (`List[datasets.Dataset]`): List of Datasets to concatenate. + info (:class:`DatasetInfo`, optional): Dataset information, like description, citation, etc. + split (:class:`NamedSplit`, optional): Name of the dataset split. + axis (``{0, 1}``, default ``0``, meaning over rows): + Axis to concatenate over, where ``0`` means over rows (vertically) and ``1`` means over columns + (horizontally). + + *New in version 1.6.0* + + Example: + + ```py + >>> ds3 = _concatenate_map_style_datasets([ds1, ds2]) + ``` + """ + # Ignore datasets with no rows + if any(dset.num_rows > 0 for dset in dsets): + dsets = [dset for dset in dsets if dset.num_rows > 0] + else: + # Return first dataset if all datasets are empty + return dsets[0] + + # Perform checks (and a potential cast if axis=0) + if axis == 0: + _check_if_features_can_be_aligned([dset.features for dset in dsets]) + else: + if not all(dset.num_rows == dsets[0].num_rows for dset in dsets): + raise ValueError("Number of rows must match for all datasets") + _check_column_names([col_name for dset in dsets for col_name in dset._data.column_names]) + + # Find common format or reset format + format = dsets[0].format + if any(dset.format != format for dset in dsets): + format = {} + logger.info("Some of the datasets have disparate format. Resetting the format of the concatenated dataset.") + + def apply_offset_to_indices_table(table, offset): + if offset == 0: + return table + else: + array = table["indices"] + new_array = pc.add(array, pa.scalar(offset, type=pa.uint64())) + return InMemoryTable.from_arrays([new_array], names=["indices"]) + + # Concatenate indices if they exist + if any(dset._indices is not None for dset in dsets): + if axis == 0: + # Datasets with no indices tables are replaced with a dataset with an indices table in memory. + # Applying an offset to an indices table also brings the table in memory. + indices_tables = [] + for i in range(len(dsets)): + if dsets[i]._indices is None: + dsets[i] = dsets[i]._select_with_indices_mapping(range(len(dsets[i]))) + indices_tables.append(dsets[i]._indices) + + # An offset needs to be applied to the indices before concatenating + offset = 0 + for i in range(len(dsets)): + indices_tables[i] = apply_offset_to_indices_table(indices_tables[i], offset) + offset += len(dsets[i]._data) + + # Concatenate indices + indices_tables = [t for t in indices_tables if len(t) > 0] + if indices_tables: + indices_table = concat_tables(indices_tables) + else: + indices_table = InMemoryTable.from_batches([], schema=pa.schema({"indices": pa.int64()})) + else: + if len(dsets) == 1: + indices_table = dsets[0]._indices + else: + for i in range(len(dsets)): + dsets[i] = dsets[i].flatten_indices() + indices_table = None + else: + indices_table = None + + table = concat_tables([dset._data for dset in dsets], axis=axis) + if axis == 0: + features_list = _align_features([dset.features for dset in dsets]) + else: + features_list = [dset.features for dset in dsets] + table = update_metadata_with_features(table, {k: v for features in features_list for k, v in features.items()}) + + # Concatenate infos + if info is None: + info = DatasetInfo.from_merge([dset.info for dset in dsets]) + fingerprint = update_fingerprint( + "".join(dset._fingerprint for dset in dsets), _concatenate_map_style_datasets, {"info": info, "split": split} + ) + + # Make final concatenated dataset + concatenated_dataset = Dataset( + table, + info=info, + split=split, + indices_table=indices_table, + fingerprint=fingerprint, + ) + concatenated_dataset.set_format(**format) + return concatenated_dataset + + +def _interleave_map_style_datasets( + datasets: list["Dataset"], + probabilities: Optional[list[float]] = None, + seed: Optional[int] = None, + info: Optional[DatasetInfo] = None, + split: Optional[NamedSplit] = None, + stopping_strategy: Literal["first_exhausted", "all_exhausted"] = "first_exhausted", + **kwargs, +) -> "Dataset": + """ + Interleave several map-style datasets (sources) into a single map-style dataset. + The new dataset is constructed by alternating between the sources to get the examples. + If `probabilities = None` (default) the new dataset is constructed by cycling between each source to get the examples. + If `probabilities` is not `None, the new dataset is constructed by getting examples from a random source at a time according to the provided probabilities. + + Args: + datasets (`List[Dataset]`): list of datasets to interleave + probabilities (`List[float]`, optional, default None): If specified, the new dataset is constructed by sampling + examples from one source at a time according to these probabilities. + seed (`int`, optional, default None): The random seed used to choose a source for each example. + info (:class:`DatasetInfo`, optional): Dataset information, like description, citation, etc. + split (:class:`NamedSplit`, optional): Name of the dataset split. + stopping_strategy (`str`, defaults to `first_exhausted`): + Two strategies are proposed right now. + By default, `first_exhausted` is an undersampling strategy, i.e the dataset construction is stopped as soon as one dataset has ran out of samples. + If the strategy is `all_exhausted`, we use an oversampling strategy, i.e the dataset construction is stopped as soon as every samples of every dataset has been added at least once. + Note that if the strategy is `all_exhausted`, the interleaved dataset size can get enormous: + - with no probabilities, the resulting dataset will have max_length_datasets*nb_dataset samples. + - with given probabilities, the resulting dataset will have more samples if some datasets have really low probability of visiting. + **kwargs (additional keyword arguments): Keyword arguments to be passed to :meth:`datasets.Datasets.select` when selecting the indices used to interleave the datasets. + + Output: + :class:`datasets.Dataset` + """ + if stopping_strategy not in ["first_exhausted", "all_exhausted"]: + raise ValueError( + f"{stopping_strategy} stopping strategy in `interleave_datasets` is not implemented yet with a list of {type(datasets[0])}" + ) + + # To interleave the datasets, we concatenate them and then we re-order the indices + concatenated_datasets = _concatenate_map_style_datasets(datasets, info=info, split=split) + + # Let's now build the indices to pass to .select() + lengths = [len(dset) for dset in datasets] + offsets = np.cumsum([0] + lengths[:-1]) + + # if stopping_strategy is "first_exhausted", it is an undersampling situation whereas it is an oversampling situation if it is "all_exhausted" + oversampling = stopping_strategy == "all_exhausted" + + if probabilities is None and not oversampling: + # Undersampling situation with cycling between each sources + # Example:: If lengths of the datasets are [3, 4, 5] + # Then the resulting indices should be [0, 3, 7, 1, 4, 8, 2, 6, 9] + # Note that we only have 3 examples per dataset since the first dataset ran out of examples + + # Reasoning behind the following operation: keeping the min_length first indices of each dataset + # while offsetting in order to correspond to the right indices of the concatenated dataset + # and flattening to effectively interleave the datasets + indices = (offsets.reshape(1, -1) + np.arange(min(lengths)).reshape(-1, 1)).flatten().tolist() + elif probabilities is None: + # Oversampling situation with cycling between each sources + # Then the resulting indices should be [0, 3, 7, 1, 4, 8, 2, 5, 9, 0, 6, 10, 1, 3, 11] + # Note that we have 5 examples per dataset with a rolling window since the longest dataset has 5 samples + + # Reasoning behind the following operation: for each dataset indices (i.e column) repeat the indices to have max_length indices per dataset + # For example, if the max_length is 5 and the i-th dataset has 3 samples, the i-th column will be [0,1,2,0,1] + indices = np.mod(np.arange(max(lengths)).reshape(-1, 1), np.array(lengths).reshape(1, -1)) + + # We have to keep the indices to their respective dataset offsets and to flatten to effectively interleave the datasets + indices = (indices + offsets).flatten().tolist() + + else: + # boolean array indicating if at index i if the dataset_i has been fully exhausted + is_exhausted = np.full(len(lengths), False) + + # if undersampling ("first_exhausted"), we stop as soon as one dataset is exhausted + # if oversampling ("all_exhausted"), we stop as soons as every dataset is exhausted, i.e as soon as every samples of every dataset has been visited at least once + bool_strategy_func = np.all if oversampling else np.any + + def iter_random_indices(): + """Get an infinite iterator that randomly samples the index of the source to pick examples from.""" + rng = np.random.default_rng(seed) + while True: + yield from (int(i) for i in rng.choice(len(datasets), size=1000, p=probabilities)) + + current_index = [0] * len(datasets) + indices = [] + for source_idx in iter_random_indices(): + # If no oversampling, we stop as soon as a dataset has ran out of examples (np.any) + # Otherwise, we stop as soon as every dataset has ran out of examples (np.all) + if bool_strategy_func(is_exhausted): + # the stopping condition was reached, let's stop + break + + # let's add the example at the current index of the `source_idx`-th dataset + indices.append(current_index[source_idx] + offsets[source_idx]) + current_index[source_idx] += 1 + + # we've ran out of examples for the current dataset, let's update our boolean array and bring the current_index back to 0 + if current_index[source_idx] >= lengths[source_idx]: + is_exhausted[source_idx] = True + current_index[source_idx] = 0 + + return concatenated_datasets.select(indices, **kwargs) + + +def _split_by_node_map_style_dataset(dataset: Dataset, rank: int, world_size: int) -> Dataset: + """ + Split a dataset for the node at rank `rank` in a pool of nodes of size `world_size`. + Each node is assigned a chunk of data, e.g. rank 0 is given the first chunk of the dataset. + To maximize data loading throughput, chunks are made of contiguous data on disk if possible. + + Args: + dataset ([`Dataset`]): + The dataset to split by node. + rank (`int`): + Rank of the current node. + world_size (`int`): + Total number of nodes. + + Returns: + [`Dataset`]: The dataset to be used on the node at rank `rank`. + """ + return dataset.shard(num_shards=world_size, index=rank, contiguous=True) + + +# This is outside Dataset.filter as it needs to be picklable for multiprocessing + + +def get_indices_from_mask_function( + function: Callable, + batched: bool, + with_indices: bool, + with_rank: bool, + input_columns: Optional[Union[str, list[str]]], + indices_mapping: Optional[Table] = None, + *args, + **fn_kwargs, +): + if batched: + # we extract indices and rank from args + *inputs, indices, rank = args + additional_args = () + if with_indices: + additional_args += (indices,) + if with_rank: + additional_args += (rank,) + mask = function(*inputs, *additional_args, **fn_kwargs) + if isinstance(mask, (pa.Array, pa.ChunkedArray)): + mask = mask.to_pylist() + else: + # we get batched data (to return less data than input) but `function` only accepts one example + # therefore we need to call `function` on each example of the batch to get the mask + *inputs, indices, rank = args + mask = [] + if input_columns is None: + # inputs only contains a batch of examples + batch: dict = inputs[0] + num_examples = len(batch[next(iter(batch.keys()))]) + for i in range(num_examples): + example = {key: batch[key][i] for key in batch} + additional_args = () + if with_indices: + additional_args += (indices[i],) + if with_rank: + additional_args += (rank,) + mask.append(function(example, *additional_args, **fn_kwargs)) + else: + # inputs is a list of columns + columns: list[list] = inputs + num_examples = len(columns[0]) + for i in range(num_examples): + input = [column[i] for column in columns] + additional_args = () + if with_indices: + additional_args += (indices[i],) + if with_rank: + additional_args += (rank,) + mask.append(function(*input, *additional_args, **fn_kwargs)) + indices_array = [i for i, to_keep in zip(indices, mask) if to_keep] + if indices_mapping is not None: + indices_array = pa.array(indices_array, type=pa.uint64()) + indices_array = indices_mapping.column(0).take(indices_array) + indices_array = indices_array.to_pylist() + return {"indices": indices_array} + + +async def async_get_indices_from_mask_function( + function: Callable, + batched: bool, + with_indices: bool, + with_rank: bool, + input_columns: Optional[Union[str, list[str]]], + indices_mapping: Optional[Table] = None, + *args, + **fn_kwargs, +): + """same function but async""" + if batched: + # we extract indices and rank from args + *inputs, indices, rank = args + additional_args = () + if with_indices: + additional_args += (indices,) + if with_rank: + additional_args += (rank,) + mask = await function(*inputs, *additional_args, **fn_kwargs) + if isinstance(mask, (pa.Array, pa.ChunkedArray)): + mask = mask.to_pylist() + else: + # we get batched data (to return less data than input) but `function` only accepts one example + # therefore we need to call `function` on each example of the batch to get the mask + *inputs, indices, rank = args + mask = [] + if input_columns is None: + # inputs only contains a batch of examples + batch: dict = inputs[0] + num_examples = len(batch[next(iter(batch.keys()))]) + for i in range(num_examples): + example = {key: batch[key][i] for key in batch} + additional_args = () + if with_indices: + additional_args += (indices[i],) + if with_rank: + additional_args += (rank,) + mask.append(await function(example, *additional_args, **fn_kwargs)) + else: + # inputs is a list of columns + columns: list[list] = inputs + num_examples = len(columns[0]) + for i in range(num_examples): + input = [column[i] for column in columns] + additional_args = () + if with_indices: + additional_args += (indices[i],) + if with_rank: + additional_args += (rank,) + mask.append(await function(*input, *additional_args, **fn_kwargs)) + indices_array = [i for i, to_keep in zip(indices, mask) if to_keep] + if indices_mapping is not None: + indices_array = pa.array(indices_array, type=pa.uint64()) + indices_array = indices_mapping.column(0).take(indices_array) + indices_array = indices_array.to_pylist() + return {"indices": indices_array} diff --git a/lib/python3.12/site-packages/datasets/arrow_reader.py b/lib/python3.12/site-packages/datasets/arrow_reader.py new file mode 100644 index 0000000000000000000000000000000000000000..3bbb58a59c3966d2f039ebb5d9ef86997d4c3cbd --- /dev/null +++ b/lib/python3.12/site-packages/datasets/arrow_reader.py @@ -0,0 +1,620 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +"""Arrow ArrowReader.""" + +import copy +import math +import os +import re +from dataclasses import dataclass +from functools import partial +from typing import TYPE_CHECKING, Optional, Union + +import pyarrow as pa +import pyarrow.parquet as pq +from tqdm.contrib.concurrent import thread_map + +from .download.download_config import DownloadConfig # noqa: F401 +from .naming import _split_re, filenames_for_dataset_split +from .table import InMemoryTable, MemoryMappedTable, Table, concat_tables +from .utils import logging +from .utils import tqdm as hf_tqdm + + +if TYPE_CHECKING: + from .info import DatasetInfo # noqa: F401 + from .splits import Split, SplitInfo # noqa: F401 + + +logger = logging.get_logger(__name__) + +HF_GCP_BASE_URL = "https://storage.googleapis.com/huggingface-nlp/cache/datasets" + +_SUB_SPEC_RE = re.compile( + rf""" +^ + (?P{_split_re[1:-1]}) + (\[ + ((?P-?[\d_]+) + (?P%)?)? + : + ((?P-?[\d_]+) + (?P%)?)? + \])?(\((?P[^\)]*)\))? +$ +""", # remove ^ and $ + re.X, +) + +_ADDITION_SEP_RE = re.compile(r"\s*\+\s*") + + +class DatasetNotOnHfGcsError(ConnectionError): + """When you can't get the dataset from the Hf google cloud storage""" + + pass + + +class MissingFilesOnHfGcsError(ConnectionError): + """When some files are missing on the Hf oogle cloud storage""" + + pass + + +@dataclass(frozen=True) +class FileInstructions: + """The file instructions associated with a split ReadInstruction. + + Attributes: + num_examples: `int`, The total number of examples + file_instructions: List[dict(filename, skip, take)], the files information. + The filenames contains the relative path, not absolute. + skip/take indicates which example read in the file: `ds.slice(skip, take)` + """ + + num_examples: int + file_instructions: list[dict] + + +def make_file_instructions( + name: str, + split_infos: list["SplitInfo"], + instruction: Union[str, "ReadInstruction"], + filetype_suffix: Optional[str] = None, + prefix_path: Optional[str] = None, +) -> FileInstructions: + """Returns instructions of the split dict. + + Args: + name (`str`): Name of the dataset. + split_infos (`list` of `[SplitInfo]`): Dataset splits information. + instruction ([`ReadInstruction`] or `str`): Reading instruction for a dataset. + filetype_suffix (`str`, *optional*): Suffix of dataset files, e.g. 'arrow' or 'parquet'. + prefix_path (`str`, *optional*): Prefix of dataset files, e.g. directory name. + + Returns: + [`FileInstructions`] + """ + if not isinstance(name, str): + raise TypeError(f"Expected str 'name', but got: {type(name).__name__}") + elif not name: + raise ValueError("Expected non-empty str 'name'") + name2len = {info.name: info.num_examples for info in split_infos} + name2shard_lengths = {info.name: info.shard_lengths for info in split_infos} + name2filenames = { + info.name: filenames_for_dataset_split( + path=prefix_path, + dataset_name=name, + split=info.name, + filetype_suffix=filetype_suffix, + shard_lengths=name2shard_lengths[info.name], + ) + for info in split_infos + } + if not isinstance(instruction, ReadInstruction): + instruction = ReadInstruction.from_spec(instruction) + # Create the absolute instruction (per split) + absolute_instructions = instruction.to_absolute(name2len) + + # For each split, return the files instruction (skip/take) + file_instructions = [] + num_examples = 0 + for abs_instr in absolute_instructions: + split_length = name2len[abs_instr.splitname] + filenames = name2filenames[abs_instr.splitname] + shard_lengths = name2shard_lengths[abs_instr.splitname] + from_ = 0 if abs_instr.from_ is None else abs_instr.from_ + to = split_length if abs_instr.to is None else abs_instr.to + if shard_lengths is None: # not sharded + for filename in filenames: + take = to - from_ + if take == 0: + continue + num_examples += take + file_instructions.append({"filename": filename, "skip": from_, "take": take}) + else: # sharded + index_start = 0 # Beginning (included) of moving window. + index_end = 0 # End (excluded) of moving window. + for filename, shard_length in zip(filenames, shard_lengths): + index_end += shard_length + if from_ < index_end and to > index_start: # There is something to take. + skip = from_ - index_start if from_ > index_start else 0 + take = to - index_start - skip if to < index_end else -1 + if take == 0: + continue + file_instructions.append({"filename": filename, "skip": skip, "take": take}) + num_examples += shard_length - skip if take == -1 else take + index_start += shard_length + return FileInstructions( + num_examples=num_examples, + file_instructions=file_instructions, + ) + + +class BaseReader: + """ + Build a Dataset object out of Instruction instance(s). + """ + + def __init__(self, path: str, info: Optional["DatasetInfo"]): + """Initializes ArrowReader. + + Args: + path (str): path where tfrecords are stored. + info (DatasetInfo): info about the dataset. + """ + self._path: str = path + self._info: Optional["DatasetInfo"] = info + self._filetype_suffix: Optional[str] = None + + def _get_table_from_filename(self, filename_skip_take, in_memory=False) -> Table: + """Returns a Dataset instance from given (filename, skip, take).""" + raise NotImplementedError + + def _read_files(self, files, in_memory=False) -> Table: + """Returns Dataset for given file instructions. + + Args: + files: List[dict(filename, skip, take)], the files information. + The filenames contain the absolute path, not relative. + skip/take indicates which example read in the file: `ds.slice(skip, take)` + in_memory (bool, default False): Whether to copy the data in-memory. + """ + if len(files) == 0 or not all(isinstance(f, dict) for f in files): + raise ValueError("please provide valid file informations") + files = copy.deepcopy(files) + for f in files: + f["filename"] = os.path.join(self._path, f["filename"]) + + pa_tables = thread_map( + partial(self._get_table_from_filename, in_memory=in_memory), + files, + tqdm_class=hf_tqdm, + desc="Loading dataset shards", + # set `disable=None` rather than `disable=False` by default to disable progress bar when no TTY attached + disable=len(files) <= 16 or None, + ) + pa_tables = [t for t in pa_tables if len(t) > 0] + if not pa_tables and (self._info is None or self._info.features is None): + raise ValueError( + "Tried to read an empty table. Please specify at least info.features to create an empty table with the right type." + ) + pa_tables = pa_tables or [InMemoryTable.from_batches([], schema=pa.schema(self._info.features.type))] + pa_table = concat_tables(pa_tables) if len(pa_tables) != 1 else pa_tables[0] + return pa_table + + def get_file_instructions(self, name, instruction, split_infos): + """Return list of dict {'filename': str, 'skip': int, 'take': int}""" + file_instructions = make_file_instructions( + name, split_infos, instruction, filetype_suffix=self._filetype_suffix, prefix_path=self._path + ) + files = file_instructions.file_instructions + return files + + def read( + self, + name, + instructions, + split_infos, + in_memory=False, + ): + """Returns Dataset instance(s). + + Args: + name (str): name of the dataset. + instructions (ReadInstruction): instructions to read. + Instruction can be string and will then be passed to the Instruction + constructor as it. + split_infos (list of SplitInfo proto): the available splits for dataset. + in_memory (bool, default False): Whether to copy the data in-memory. + + Returns: + kwargs to build a single Dataset instance. + """ + + files = self.get_file_instructions(name, instructions, split_infos) + if not files: + msg = f'Instruction "{instructions}" corresponds to no data!' + raise ValueError(msg) + return self.read_files(files=files, original_instructions=instructions, in_memory=in_memory) + + def read_files( + self, + files: list[dict], + original_instructions: Union[None, "ReadInstruction", "Split"] = None, + in_memory=False, + ): + """Returns single Dataset instance for the set of file instructions. + + Args: + files: List[dict(filename, skip, take)], the files information. + The filenames contains the relative path, not absolute. + skip/take indicates which example read in the file: `ds.skip().take()` + original_instructions: store the original instructions used to build the dataset split in the dataset. + in_memory (bool, default False): Whether to copy the data in-memory. + + Returns: + kwargs to build a Dataset instance. + """ + # Prepend path to filename + pa_table = self._read_files(files, in_memory=in_memory) + # If original_instructions is not None, convert it to a human-readable NamedSplit + if original_instructions is not None: + from .splits import Split # noqa + + split = Split(str(original_instructions)) + else: + split = None + dataset_kwargs = {"arrow_table": pa_table, "info": self._info, "split": split} + return dataset_kwargs + + +class ArrowReader(BaseReader): + """ + Build a Dataset object out of Instruction instance(s). + This Reader uses either memory mapping or file descriptors (in-memory) on arrow files. + """ + + def __init__(self, path: str, info: Optional["DatasetInfo"]): + """Initializes ArrowReader. + + Args: + path (str): path where Arrow files are stored. + info (DatasetInfo): info about the dataset. + """ + super().__init__(path, info) + self._filetype_suffix = "arrow" + + def _get_table_from_filename(self, filename_skip_take, in_memory=False) -> Table: + """Returns a Dataset instance from given (filename, skip, take).""" + filename, skip, take = ( + filename_skip_take["filename"], + filename_skip_take["skip"] if "skip" in filename_skip_take else None, + filename_skip_take["take"] if "take" in filename_skip_take else None, + ) + table = ArrowReader.read_table(filename, in_memory=in_memory) + if take == -1: + take = len(table) - skip + # here we don't want to slice an empty table, or it may segfault + if skip is not None and take is not None and not (skip == 0 and take == len(table)): + table = table.slice(skip, take) + return table + + @staticmethod + def read_table(filename, in_memory=False) -> Table: + """ + Read table from file. + + Args: + filename (str): File name of the table. + in_memory (bool, default=False): Whether to copy the data in-memory. + + Returns: + pyarrow.Table + """ + table_cls = InMemoryTable if in_memory else MemoryMappedTable + return table_cls.from_file(filename) + + +class ParquetReader(BaseReader): + """ + Build a Dataset object out of Instruction instance(s). + This Reader uses memory mapping on parquet files. + """ + + def __init__(self, path: str, info: Optional["DatasetInfo"]): + """Initializes ParquetReader. + + Args: + path (str): path where tfrecords are stored. + info (DatasetInfo): info about the dataset. + """ + super().__init__(path, info) + self._filetype_suffix = "parquet" + + def _get_table_from_filename(self, filename_skip_take, **kwargs): + """Returns a Dataset instance from given (filename, skip, take).""" + filename, skip, take = ( + filename_skip_take["filename"], + filename_skip_take["skip"] if "skip" in filename_skip_take else None, + filename_skip_take["take"] if "take" in filename_skip_take else None, + ) + # Parquet read_table always loads data in memory, independently of memory_map + pa_table = pq.read_table(filename, memory_map=True) + # here we don't want to slice an empty table, or it may segfault + if skip is not None and take is not None and not (skip == 0 and take == len(pa_table)): + pa_table = pa_table.slice(skip, take) + return pa_table + + +@dataclass(frozen=True) +class _AbsoluteInstruction: + """A machine friendly slice: defined absolute positive boundaries.""" + + splitname: str + from_: int # uint (starting index). + to: int # uint (ending index). + + +@dataclass(frozen=True) +class _RelativeInstruction: + """Represents a single parsed slicing instruction, can use % and negatives.""" + + splitname: str + from_: Optional[int] = None # int (starting index) or None if no lower boundary. + to: Optional[int] = None # int (ending index) or None if no upper boundary. + unit: Optional[str] = None + rounding: Optional[str] = None + + def __post_init__(self): + if self.unit is not None and self.unit not in ["%", "abs"]: + raise ValueError("unit must be either % or abs") + if self.rounding is not None and self.rounding not in ["closest", "pct1_dropremainder"]: + raise ValueError("rounding must be either closest or pct1_dropremainder") + if self.unit != "%" and self.rounding is not None: + raise ValueError("It is forbidden to specify rounding if not using percent slicing.") + if self.unit == "%" and self.from_ is not None and abs(self.from_) > 100: + raise ValueError("Percent slice boundaries must be > -100 and < 100.") + if self.unit == "%" and self.to is not None and abs(self.to) > 100: + raise ValueError("Percent slice boundaries must be > -100 and < 100.") + # Update via __dict__ due to instance being "frozen" + self.__dict__["rounding"] = "closest" if self.rounding is None and self.unit == "%" else self.rounding + + +def _str_to_read_instruction(spec): + """Returns ReadInstruction for given string.""" + res = _SUB_SPEC_RE.match(spec) + if not res: + raise ValueError(f"Unrecognized instruction format: {spec}") + unit = "%" if res.group("from_pct") or res.group("to_pct") else "abs" + return ReadInstruction( + split_name=res.group("split"), + rounding=res.group("rounding"), + from_=int(res.group("from")) if res.group("from") else None, + to=int(res.group("to")) if res.group("to") else None, + unit=unit, + ) + + +def _pct_to_abs_pct1(boundary, num_examples): + # Using math.trunc here, since -99.5% should give -99%, not -100%. + if num_examples < 100: + msg = ( + 'Using "pct1_dropremainder" rounding on a split with less than 100 ' + "elements is forbidden: it always results in an empty dataset." + ) + raise ValueError(msg) + return boundary * math.trunc(num_examples / 100.0) + + +def _pct_to_abs_closest(boundary, num_examples): + return int(round(boundary * num_examples / 100.0)) + + +def _rel_to_abs_instr(rel_instr, name2len): + """Returns _AbsoluteInstruction instance for given RelativeInstruction. + + Args: + rel_instr: RelativeInstruction instance. + name2len: dict {split_name: num_examples}. + """ + pct_to_abs = _pct_to_abs_closest if rel_instr.rounding == "closest" else _pct_to_abs_pct1 + split = rel_instr.splitname + if split not in name2len: + raise ValueError(f'Unknown split "{split}". Should be one of {list(name2len)}.') + num_examples = name2len[split] + from_ = rel_instr.from_ + to = rel_instr.to + if rel_instr.unit == "%": + from_ = 0 if from_ is None else pct_to_abs(from_, num_examples) + to = num_examples if to is None else pct_to_abs(to, num_examples) + else: + from_ = 0 if from_ is None else from_ + to = num_examples if to is None else to + if from_ < 0: + from_ = max(num_examples + from_, 0) + if to < 0: + to = max(num_examples + to, 0) + from_ = min(from_, num_examples) + to = min(to, num_examples) + return _AbsoluteInstruction(split, from_, to) + + +class ReadInstruction: + """Reading instruction for a dataset. + + Examples:: + + # The following lines are equivalent: + ds = datasets.load_dataset('mnist', split='test[:33%]') + ds = datasets.load_dataset('mnist', split=datasets.ReadInstruction.from_spec('test[:33%]')) + ds = datasets.load_dataset('mnist', split=datasets.ReadInstruction('test', to=33, unit='%')) + ds = datasets.load_dataset('mnist', split=datasets.ReadInstruction( + 'test', from_=0, to=33, unit='%')) + + # The following lines are equivalent: + ds = datasets.load_dataset('mnist', split='test[:33%]+train[1:-1]') + ds = datasets.load_dataset('mnist', split=datasets.ReadInstruction.from_spec( + 'test[:33%]+train[1:-1]')) + ds = datasets.load_dataset('mnist', split=( + datasets.ReadInstruction('test', to=33, unit='%') + + datasets.ReadInstruction('train', from_=1, to=-1, unit='abs'))) + + # The following lines are equivalent: + ds = datasets.load_dataset('mnist', split='test[:33%](pct1_dropremainder)') + ds = datasets.load_dataset('mnist', split=datasets.ReadInstruction.from_spec( + 'test[:33%](pct1_dropremainder)')) + ds = datasets.load_dataset('mnist', split=datasets.ReadInstruction( + 'test', from_=0, to=33, unit='%', rounding="pct1_dropremainder")) + + # 10-fold validation: + tests = datasets.load_dataset( + 'mnist', + [datasets.ReadInstruction('train', from_=k, to=k+10, unit='%') + for k in range(0, 100, 10)]) + trains = datasets.load_dataset( + 'mnist', + [datasets.ReadInstruction('train', to=k, unit='%') + datasets.ReadInstruction('train', from_=k+10, unit='%') + for k in range(0, 100, 10)]) + + """ + + def _init(self, relative_instructions): + # Private initializer. + self._relative_instructions = relative_instructions + + @classmethod + def _read_instruction_from_relative_instructions(cls, relative_instructions): + """Returns ReadInstruction obj initialized with relative_instructions.""" + # Use __new__ to bypass __init__ used by public API and not conveniant here. + result = cls.__new__(cls) + result._init(relative_instructions) # pylint: disable=protected-access + return result + + def __init__(self, split_name, rounding=None, from_=None, to=None, unit=None): + """Initialize ReadInstruction. + + Args: + split_name (str): name of the split to read. Eg: 'train'. + rounding (str, optional): The rounding behaviour to use when percent slicing is + used. Ignored when slicing with absolute indices. + Possible values: + - 'closest' (default): The specified percentages are rounded to the + closest value. Use this if you want specified percents to be as + much exact as possible. + - 'pct1_dropremainder': the specified percentages are treated as + multiple of 1%. Use this option if you want consistency. Eg: + len(5%) == 5 * len(1%). + Using this option, one might not be able to use the full set of + examples, if the number of those is not a multiple of 100. + from_ (int): + to (int): alternative way of specifying slicing boundaries. If any of + {from_, to, unit} argument is used, slicing cannot be specified as + string. + unit (str): optional, one of: + '%': to set the slicing unit as percents of the split size. + 'abs': to set the slicing unit as absolute numbers. + """ + # This constructor is not always called. See factory method + # `_read_instruction_from_relative_instructions`. Common init instructions + # MUST be placed in the _init method. + self._init([_RelativeInstruction(split_name, from_, to, unit, rounding)]) + + @classmethod + def from_spec(cls, spec): + """Creates a `ReadInstruction` instance out of a string spec. + + Args: + spec (`str`): + Split(s) + optional slice(s) to read + optional rounding + if percents are used as the slicing unit. A slice can be specified, + using absolute numbers (`int`) or percentages (`int`). + + Examples: + + ``` + test: test split. + test + validation: test split + validation split. + test[10:]: test split, minus its first 10 records. + test[:10%]: first 10% records of test split. + test[:20%](pct1_dropremainder): first 10% records, rounded with the pct1_dropremainder rounding. + test[:-5%]+train[40%:60%]: first 95% of test + middle 20% of train. + ``` + + Returns: + ReadInstruction instance. + """ + spec = str(spec) # Need to convert to str in case of NamedSplit instance. + subs = _ADDITION_SEP_RE.split(spec) + if not subs: + raise ValueError(f"No instructions could be built out of {spec}") + instruction = _str_to_read_instruction(subs[0]) + return sum((_str_to_read_instruction(sub) for sub in subs[1:]), instruction) + + def to_spec(self): + rel_instr_specs = [] + for rel_instr in self._relative_instructions: + rel_instr_spec = rel_instr.splitname + if rel_instr.from_ is not None or rel_instr.to is not None: + from_ = rel_instr.from_ + to = rel_instr.to + unit = rel_instr.unit + rounding = rel_instr.rounding + unit = unit if unit == "%" else "" + from_ = str(from_) + unit if from_ is not None else "" + to = str(to) + unit if to is not None else "" + slice_str = f"[{from_}:{to}]" + rounding_str = ( + f"({rounding})" if unit == "%" and rounding is not None and rounding != "closest" else "" + ) + rel_instr_spec += slice_str + rounding_str + rel_instr_specs.append(rel_instr_spec) + return "+".join(rel_instr_specs) + + def __add__(self, other): + """Returns a new ReadInstruction obj, result of appending other to self.""" + if not isinstance(other, ReadInstruction): + msg = "ReadInstruction can only be added to another ReadInstruction obj." + raise TypeError(msg) + self_ris = self._relative_instructions + other_ris = other._relative_instructions # pylint: disable=protected-access + if ( + self_ris[0].unit != "abs" + and other_ris[0].unit != "abs" + and self._relative_instructions[0].rounding != other_ris[0].rounding + ): + raise ValueError("It is forbidden to sum ReadInstruction instances with different rounding values.") + return self._read_instruction_from_relative_instructions(self_ris + other_ris) + + def __str__(self): + return self.to_spec() + + def __repr__(self): + return f"ReadInstruction({self._relative_instructions})" + + def to_absolute(self, name2len): + """Translate instruction into a list of absolute instructions. + + Those absolute instructions are then to be added together. + + Args: + name2len (`dict`): + Associating split names to number of examples. + + Returns: + list of _AbsoluteInstruction instances (corresponds to the + in spec). + """ + return [_rel_to_abs_instr(rel_instr, name2len) for rel_instr in self._relative_instructions] diff --git a/lib/python3.12/site-packages/datasets/arrow_writer.py b/lib/python3.12/site-packages/datasets/arrow_writer.py new file mode 100644 index 0000000000000000000000000000000000000000..64097b773f164e06a17085367e7a4e9b7e3fd1ed --- /dev/null +++ b/lib/python3.12/site-packages/datasets/arrow_writer.py @@ -0,0 +1,677 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +"""To write records into Parquet files.""" + +import json +import sys +from collections.abc import Iterable +from typing import Any, Optional, Union + +import fsspec +import numpy as np +import pyarrow as pa +import pyarrow.parquet as pq +from fsspec.core import url_to_fs + +from . import config +from .features import Audio, Features, Image, Pdf, Value, Video +from .features.features import ( + FeatureType, + _ArrayXDExtensionType, + _visit, + cast_to_python_objects, + generate_from_arrow_type, + get_nested_type, + list_of_np_array_to_pyarrow_listarray, + numpy_to_pyarrow_listarray, + to_pyarrow_listarray, +) +from .filesystems import is_remote_filesystem +from .info import DatasetInfo +from .keyhash import DuplicatedKeysError, KeyHasher +from .table import array_cast, cast_array_to_feature, embed_table_storage, table_cast +from .utils import logging +from .utils.py_utils import asdict, first_non_null_non_empty_value + + +logger = logging.get_logger(__name__) + +type_ = type # keep python's type function + + +def get_writer_batch_size(features: Optional[Features]) -> Optional[int]: + """ + Get the writer_batch_size that defines the maximum row group size in the parquet files. + The default in `datasets` is 1,000 but we lower it to 100 for image/audio datasets and 10 for videos. + This allows to optimize random access to parquet file, since accessing 1 row requires + to read its entire row group. + + This can be improved to get optimized size for querying/iterating + but at least it matches the dataset viewer expectations on HF. + + Args: + features (`datasets.Features` or `None`): + Dataset Features from `datasets`. + Returns: + writer_batch_size (`Optional[int]`): + Writer batch size to pass to a dataset builder. + If `None`, then it will use the `datasets` default. + """ + if not features: + return None + + batch_size = np.inf + + def set_batch_size(feature: FeatureType) -> None: + nonlocal batch_size + if isinstance(feature, Image): + batch_size = min(batch_size, config.PARQUET_ROW_GROUP_SIZE_FOR_IMAGE_DATASETS) + elif isinstance(feature, Audio): + batch_size = min(batch_size, config.PARQUET_ROW_GROUP_SIZE_FOR_AUDIO_DATASETS) + elif isinstance(feature, Video): + batch_size = min(batch_size, config.PARQUET_ROW_GROUP_SIZE_FOR_VIDEO_DATASETS) + elif isinstance(feature, Value) and feature.dtype == "binary": + batch_size = min(batch_size, config.PARQUET_ROW_GROUP_SIZE_FOR_BINARY_DATASETS) + + _visit(features, set_batch_size) + + return None if batch_size is np.inf else batch_size + + +class SchemaInferenceError(ValueError): + pass + + +class TypedSequence: + """ + This data container generalizes the typing when instantiating pyarrow arrays, tables or batches. + + More specifically it adds several features: + - Support extension types like ``datasets.features.Array2DExtensionType``: + By default pyarrow arrays don't return extension arrays. One has to call + ``pa.ExtensionArray.from_storage(type, pa.array(data, type.storage_type))`` + in order to get an extension array. + - Support for ``try_type`` parameter that can be used instead of ``type``: + When an array is transformed, we like to keep the same type as before if possible. + For example when calling :func:`datasets.Dataset.map`, we don't want to change the type + of each column by default. + - Better error message when a pyarrow array overflows. + + Example:: + + from datasets.features import Array2D, Array2DExtensionType, Value + from datasets.arrow_writer import TypedSequence + import pyarrow as pa + + arr = pa.array(TypedSequence([1, 2, 3], type=Value("int32"))) + assert arr.type == pa.int32() + + arr = pa.array(TypedSequence([1, 2, 3], try_type=Value("int32"))) + assert arr.type == pa.int32() + + arr = pa.array(TypedSequence(["foo", "bar"], try_type=Value("int32"))) + assert arr.type == pa.string() + + arr = pa.array(TypedSequence([[[1, 2, 3]]], type=Array2D((1, 3), "int64"))) + assert arr.type == Array2DExtensionType((1, 3), "int64") + + table = pa.Table.from_pydict({ + "image": TypedSequence([[[1, 2, 3]]], type=Array2D((1, 3), "int64")) + }) + assert table["image"].type == Array2DExtensionType((1, 3), "int64") + + """ + + def __init__( + self, + data: Iterable, + type: Optional[FeatureType] = None, + try_type: Optional[FeatureType] = None, + optimized_int_type: Optional[FeatureType] = None, + ): + # assert type is None or try_type is None, + if type is not None and try_type is not None: + raise ValueError("You cannot specify both type and try_type") + # set attributes + self.data = data + self.type = type + self.try_type = try_type # is ignored if it doesn't match the data + self.optimized_int_type = optimized_int_type + # when trying a type (is ignored if data is not compatible) + self.trying_type = self.try_type is not None + self.trying_int_optimization = optimized_int_type is not None and type is None and try_type is None + # used to get back the inferred type after __arrow_array__() is called once + self._inferred_type = None + + def get_inferred_type(self) -> FeatureType: + """Return the inferred feature type. + This is done by converting the sequence to an Arrow array, and getting the corresponding + feature type. + + Since building the Arrow array can be expensive, the value of the inferred type is cached + as soon as pa.array is called on the typed sequence. + + Returns: + FeatureType: inferred feature type of the sequence. + """ + if self._inferred_type is None: + self._inferred_type = generate_from_arrow_type(pa.array(self).type) + return self._inferred_type + + @staticmethod + def _infer_custom_type_and_encode(data: Iterable) -> tuple[Iterable, Optional[FeatureType]]: + """Implement type inference for custom objects like PIL.Image.Image -> Image type. + + This function is only used for custom python objects that can't be directly passed to build + an Arrow array. In such cases is infers the feature type to use, and it encodes the data so + that they can be passed to an Arrow array. + + Args: + data (Iterable): array of data to infer the type, e.g. a list of PIL images. + + Returns: + Tuple[Iterable, Optional[FeatureType]]: a tuple with: + - the (possibly encoded) array, if the inferred feature type requires encoding + - the inferred feature type if the array is made of supported custom objects like + PIL images, else None. + """ + if config.PIL_AVAILABLE and "PIL" in sys.modules: + import PIL.Image + + non_null_idx, non_null_value = first_non_null_non_empty_value(data) + if isinstance(non_null_value, PIL.Image.Image): + return [Image().encode_example(value) if value is not None else None for value in data], Image() + if isinstance(non_null_value, list) and isinstance(non_null_value[0], PIL.Image.Image): + return [[Image().encode_example(x) for x in value] if value is not None else None for value in data], [ + Image() + ] + if config.PDFPLUMBER_AVAILABLE and "pdfplumber" in sys.modules: + import pdfplumber + + non_null_idx, non_null_value = first_non_null_non_empty_value(data) + if isinstance(non_null_value, pdfplumber.pdf.PDF): + return [Pdf().encode_example(value) if value is not None else None for value in data], Pdf() + if isinstance(non_null_value, list) and isinstance(non_null_value[0], pdfplumber.pdf.PDF): + return [[Pdf().encode_example(x) for x in value] if value is not None else None for value in data], [ + Pdf() + ] + return data, None + + def __arrow_array__(self, type: Optional[pa.DataType] = None): + """This function is called when calling pa.array(typed_sequence)""" + + if type is not None: + raise ValueError("TypedSequence is supposed to be used with pa.array(typed_sequence, type=None)") + del type # make sure we don't use it + data = self.data + # automatic type inference for custom objects + if self.type is None and self.try_type is None: + data, self._inferred_type = self._infer_custom_type_and_encode(data) + if self._inferred_type is None: + type = self.try_type if self.trying_type else self.type + else: + type = self._inferred_type + pa_type = get_nested_type(type) if type is not None else None + optimized_int_pa_type = ( + get_nested_type(self.optimized_int_type) if self.optimized_int_type is not None else None + ) + trying_cast_to_python_objects = False + try: + # custom pyarrow types + if isinstance(pa_type, _ArrayXDExtensionType): + storage = to_pyarrow_listarray(data, pa_type) + return pa.ExtensionArray.from_storage(pa_type, storage) + + # efficient np array to pyarrow array + if isinstance(data, np.ndarray): + out = numpy_to_pyarrow_listarray(data) + elif isinstance(data, list) and data and isinstance(first_non_null_non_empty_value(data)[1], np.ndarray): + out = list_of_np_array_to_pyarrow_listarray(data) + else: + trying_cast_to_python_objects = True + out = pa.array(cast_to_python_objects(data, only_1d_for_numpy=True)) + # use smaller integer precisions if possible + if self.trying_int_optimization: + if pa.types.is_int64(out.type): + out = out.cast(optimized_int_pa_type) + elif pa.types.is_list(out.type): + if pa.types.is_int64(out.type.value_type): + out = array_cast(out, pa.list_(optimized_int_pa_type)) + elif pa.types.is_list(out.type.value_type) and pa.types.is_int64(out.type.value_type.value_type): + out = array_cast(out, pa.list_(pa.list_(optimized_int_pa_type))) + # otherwise we can finally use the user's type + elif type is not None: + # We use cast_array_to_feature to support casting to custom types like Audio and Image + # Also, when trying type "string", we don't want to convert integers or floats to "string". + # We only do it if trying_type is False - since this is what the user asks for. + out = cast_array_to_feature( + out, type, allow_primitive_to_str=not self.trying_type, allow_decimal_to_str=not self.trying_type + ) + return out + except ( + TypeError, + pa.lib.ArrowInvalid, + pa.lib.ArrowNotImplementedError, + ) as e: # handle type errors and overflows + # Ignore ArrowNotImplementedError caused by trying type, otherwise re-raise + if not self.trying_type and isinstance(e, pa.lib.ArrowNotImplementedError): + raise + + if self.trying_type: + try: # second chance + if isinstance(data, np.ndarray): + return numpy_to_pyarrow_listarray(data) + elif isinstance(data, list) and data and any(isinstance(value, np.ndarray) for value in data): + return list_of_np_array_to_pyarrow_listarray(data) + else: + trying_cast_to_python_objects = True + return pa.array(cast_to_python_objects(data, only_1d_for_numpy=True)) + except pa.lib.ArrowInvalid as e: + if "overflow" in str(e): + raise OverflowError( + f"There was an overflow with type {type_(data)}. Try to reduce writer_batch_size to have batches smaller than 2GB.\n({e})" + ) from None + elif self.trying_int_optimization and "not in range" in str(e): + optimized_int_pa_type_str = np.dtype(optimized_int_pa_type.to_pandas_dtype()).name + logger.info( + f"Failed to cast a sequence to {optimized_int_pa_type_str}. Falling back to int64." + ) + return out + elif trying_cast_to_python_objects and "Could not convert" in str(e): + out = pa.array( + cast_to_python_objects(data, only_1d_for_numpy=True, optimize_list_casting=False) + ) + if type is not None: + out = cast_array_to_feature( + out, type, allow_primitive_to_str=True, allow_decimal_to_str=True + ) + return out + else: + raise + elif "overflow" in str(e): + raise OverflowError( + f"There was an overflow with type {type_(data)}. Try to reduce writer_batch_size to have batches smaller than 2GB.\n({e})" + ) from None + elif self.trying_int_optimization and "not in range" in str(e): + optimized_int_pa_type_str = np.dtype(optimized_int_pa_type.to_pandas_dtype()).name + logger.info(f"Failed to cast a sequence to {optimized_int_pa_type_str}. Falling back to int64.") + return out + elif trying_cast_to_python_objects and "Could not convert" in str(e): + out = pa.array(cast_to_python_objects(data, only_1d_for_numpy=True, optimize_list_casting=False)) + if type is not None: + out = cast_array_to_feature(out, type, allow_primitive_to_str=True, allow_decimal_to_str=True) + return out + else: + raise + + +class OptimizedTypedSequence(TypedSequence): + def __init__( + self, + data, + type: Optional[FeatureType] = None, + try_type: Optional[FeatureType] = None, + col: Optional[str] = None, + optimized_int_type: Optional[FeatureType] = None, + ): + optimized_int_type_by_col = { + "attention_mask": Value("int8"), # binary tensor + "special_tokens_mask": Value("int8"), + "input_ids": Value("int32"), # typical vocab size: 0-50k (max ~500k, never > 1M) + "token_type_ids": Value( + "int8" + ), # binary mask; some (XLNetModel) use an additional token represented by a 2 + } + if type is None and try_type is None: + optimized_int_type = optimized_int_type_by_col.get(col, None) + super().__init__(data, type=type, try_type=try_type, optimized_int_type=optimized_int_type) + + +class ArrowWriter: + """Shuffles and writes Examples to Arrow files.""" + + _WRITER_CLASS = pa.RecordBatchStreamWriter + + def __init__( + self, + schema: Optional[pa.Schema] = None, + features: Optional[Features] = None, + path: Optional[str] = None, + stream: Optional[pa.NativeFile] = None, + fingerprint: Optional[str] = None, + writer_batch_size: Optional[int] = None, + hash_salt: Optional[str] = None, + check_duplicates: Optional[bool] = False, + disable_nullable: bool = False, + update_features: bool = False, + with_metadata: bool = True, + unit: str = "examples", + embed_local_files: bool = False, + storage_options: Optional[dict] = None, + ): + if path is None and stream is None: + raise ValueError("At least one of path and stream must be provided.") + if features is not None: + self._features = features + self._schema = None + elif schema is not None: + self._schema: pa.Schema = schema + self._features = Features.from_arrow_schema(self._schema) + else: + self._features = None + self._schema = None + + if hash_salt is not None: + # Create KeyHasher instance using split name as hash salt + self._hasher = KeyHasher(hash_salt) + else: + self._hasher = KeyHasher("") + + self._check_duplicates = check_duplicates + self._disable_nullable = disable_nullable + + if stream is None: + fs, path = url_to_fs(path, **(storage_options or {})) + self._fs: fsspec.AbstractFileSystem = fs + self._path = path if not is_remote_filesystem(self._fs) else self._fs.unstrip_protocol(path) + self.stream = self._fs.open(path, "wb") + self._closable_stream = True + else: + self._fs = None + self._path = None + self.stream = stream + self._closable_stream = False + + self.fingerprint = fingerprint + self.disable_nullable = disable_nullable + self.writer_batch_size = ( + writer_batch_size or get_writer_batch_size(self._features) or config.DEFAULT_MAX_BATCH_SIZE + ) + self.update_features = update_features + self.with_metadata = with_metadata + self.unit = unit + self.embed_local_files = embed_local_files + + self._num_examples = 0 + self._num_bytes = 0 + self.current_examples: list[tuple[dict[str, Any], str]] = [] + self.current_rows: list[pa.Table] = [] + self.pa_writer: Optional[pa.RecordBatchStreamWriter] = None + self.hkey_record = [] + + def __len__(self): + """Return the number of writed and staged examples""" + return self._num_examples + len(self.current_examples) + len(self.current_rows) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() + + def close(self): + # Try closing if opened; if closed: pyarrow.lib.ArrowInvalid: Invalid operation on closed file + if self.pa_writer: # it might be None + try: + self.pa_writer.close() + except Exception: # pyarrow.lib.ArrowInvalid, OSError + pass + if self._closable_stream and not self.stream.closed: + self.stream.close() # This also closes self.pa_writer if it is opened + + def _build_writer(self, inferred_schema: pa.Schema): + schema = self.schema + inferred_features = Features.from_arrow_schema(inferred_schema) + if self._features is not None: + if self.update_features: # keep original features it they match, or update them + fields = {field.name: field for field in self._features.type} + for inferred_field in inferred_features.type: + name = inferred_field.name + if name in fields: + if inferred_field == fields[name]: + inferred_features[name] = self._features[name] + self._features = inferred_features + schema: pa.Schema = inferred_schema + else: + self._features = inferred_features + schema: pa.Schema = inferred_features.arrow_schema + if self.disable_nullable: + schema = pa.schema(pa.field(field.name, field.type, nullable=False) for field in schema) + if self.with_metadata: + schema = schema.with_metadata(self._build_metadata(DatasetInfo(features=self._features), self.fingerprint)) + else: + schema = schema.with_metadata({}) + self._schema = schema + self.pa_writer = self._WRITER_CLASS(self.stream, schema) + + @property + def schema(self): + _schema = ( + self._schema + if self._schema is not None + else (pa.schema(self._features.type) if self._features is not None else None) + ) + if self._disable_nullable and _schema is not None: + _schema = pa.schema(pa.field(field.name, field.type, nullable=False) for field in _schema) + return _schema if _schema is not None else [] + + @staticmethod + def _build_metadata(info: DatasetInfo, fingerprint: Optional[str] = None) -> dict[str, str]: + info_keys = ["features"] # we can add support for more DatasetInfo keys in the future + info_as_dict = asdict(info) + metadata = {} + metadata["info"] = {key: info_as_dict[key] for key in info_keys} + if fingerprint is not None: + metadata["fingerprint"] = fingerprint + return {"huggingface": json.dumps(metadata)} + + def write_examples_on_file(self): + """Write stored examples from the write-pool of examples. It makes a table out of the examples and write it.""" + if not self.current_examples: + return + # preserve the order the columns + if self.schema: + schema_cols = set(self.schema.names) + examples_cols = self.current_examples[0][0].keys() # .keys() preserves the order (unlike set) + common_cols = [col for col in self.schema.names if col in examples_cols] + extra_cols = [col for col in examples_cols if col not in schema_cols] + cols = common_cols + extra_cols + else: + cols = list(self.current_examples[0][0]) + batch_examples = {} + for col in cols: + # We use row[0][col] since current_examples contains (example, key) tuples. + # Moreover, examples could be Arrow arrays of 1 element. + # This can happen in `.map()` when we want to re-write the same Arrow data + if all(isinstance(row[0][col], (pa.Array, pa.ChunkedArray)) for row in self.current_examples): + arrays = [row[0][col] for row in self.current_examples] + arrays = [ + chunk + for array in arrays + for chunk in (array.chunks if isinstance(array, pa.ChunkedArray) else [array]) + ] + batch_examples[col] = pa.concat_arrays(arrays) + else: + batch_examples[col] = [ + row[0][col].to_pylist()[0] if isinstance(row[0][col], (pa.Array, pa.ChunkedArray)) else row[0][col] + for row in self.current_examples + ] + self.write_batch(batch_examples=batch_examples) + self.current_examples = [] + + def write_rows_on_file(self): + """Write stored rows from the write-pool of rows. It concatenates the single-row tables and it writes the resulting table.""" + if not self.current_rows: + return + table = pa.concat_tables(self.current_rows) + self.write_table(table) + self.current_rows = [] + + def write( + self, + example: dict[str, Any], + key: Optional[Union[str, int, bytes]] = None, + writer_batch_size: Optional[int] = None, + ): + """Add a given (Example,Key) pair to the write-pool of examples which is written to file. + + Args: + example: the Example to add. + key: Optional, a unique identifier(str, int or bytes) associated with each example + """ + # Utilize the keys and duplicate checking when `self._check_duplicates` is passed True + if self._check_duplicates: + # Create unique hash from key and store as (key, example) pairs + hash = self._hasher.hash(key) + self.current_examples.append((example, hash)) + # Maintain record of keys and their respective hashes for checking duplicates + self.hkey_record.append((hash, key)) + else: + # Store example as a tuple so as to keep the structure of `self.current_examples` uniform + self.current_examples.append((example, "")) + + if writer_batch_size is None: + writer_batch_size = self.writer_batch_size + if writer_batch_size is not None and len(self.current_examples) >= writer_batch_size: + if self._check_duplicates: + self.check_duplicate_keys() + # Re-initializing to empty list for next batch + self.hkey_record = [] + + self.write_examples_on_file() + + def check_duplicate_keys(self): + """Raises error if duplicates found in a batch""" + tmp_record = set() + for hash, key in self.hkey_record: + if hash in tmp_record: + duplicate_key_indices = [ + str(self._num_examples + index) + for index, (duplicate_hash, _) in enumerate(self.hkey_record) + if duplicate_hash == hash + ] + + raise DuplicatedKeysError(key, duplicate_key_indices) + else: + tmp_record.add(hash) + + def write_row(self, row: pa.Table, writer_batch_size: Optional[int] = None): + """Add a given single-row Table to the write-pool of rows which is written to file. + + Args: + row: the row to add. + """ + if len(row) != 1: + raise ValueError(f"Only single-row pyarrow tables are allowed but got table with {len(row)} rows.") + self.current_rows.append(row) + if writer_batch_size is None: + writer_batch_size = self.writer_batch_size + if writer_batch_size is not None and len(self.current_rows) >= writer_batch_size: + self.write_rows_on_file() + + def write_batch( + self, + batch_examples: dict[str, list], + writer_batch_size: Optional[int] = None, + try_original_type: Optional[bool] = True, + ): + """Write a batch of Example to file. + Ignores the batch if it appears to be empty, + preventing a potential schema update of unknown types. + + Args: + batch_examples: the batch of examples to add. + try_original_type: use `try_type` when instantiating OptimizedTypedSequence if `True`, otherwise `try_type = None`. + """ + if batch_examples and len(next(iter(batch_examples.values()))) == 0: + return + features = None if self.pa_writer is None and self.update_features else self._features + try_features = self._features if self.pa_writer is None and self.update_features else None + arrays = [] + inferred_features = Features() + # preserve the order the columns + if self.schema: + schema_cols = set(self.schema.names) + batch_cols = batch_examples.keys() # .keys() preserves the order (unlike set) + common_cols = [col for col in self.schema.names if col in batch_cols] + extra_cols = [col for col in batch_cols if col not in schema_cols] + cols = common_cols + extra_cols + else: + cols = list(batch_examples) + for col in cols: + col_values = batch_examples[col] + col_type = features[col] if features else None + if isinstance(col_values, (pa.Array, pa.ChunkedArray)): + array = cast_array_to_feature(col_values, col_type) if col_type is not None else col_values + arrays.append(array) + inferred_features[col] = generate_from_arrow_type(col_values.type) + else: + col_try_type = ( + try_features[col] + if try_features is not None and col in try_features and try_original_type + else None + ) + typed_sequence = OptimizedTypedSequence(col_values, type=col_type, try_type=col_try_type, col=col) + arrays.append(pa.array(typed_sequence)) + inferred_features[col] = typed_sequence.get_inferred_type() + schema = inferred_features.arrow_schema if self.pa_writer is None else self.schema + pa_table = pa.Table.from_arrays(arrays, schema=schema) + self.write_table(pa_table, writer_batch_size) + + def write_table(self, pa_table: pa.Table, writer_batch_size: Optional[int] = None): + """Write a Table to file. + + Args: + example: the Table to add. + """ + if writer_batch_size is None: + writer_batch_size = self.writer_batch_size + if self.pa_writer is None: + self._build_writer(inferred_schema=pa_table.schema) + pa_table = pa_table.combine_chunks() + pa_table = table_cast(pa_table, self._schema) + if self.embed_local_files: + pa_table = embed_table_storage(pa_table) + self._num_bytes += pa_table.nbytes + self._num_examples += pa_table.num_rows + self.pa_writer.write_table(pa_table, writer_batch_size) + + def finalize(self, close_stream=True): + self.write_rows_on_file() + # In case current_examples < writer_batch_size, but user uses finalize() + if self._check_duplicates: + self.check_duplicate_keys() + # Re-initializing to empty list for next batch + self.hkey_record = [] + self.write_examples_on_file() + # If schema is known, infer features even if no examples were written + if self.pa_writer is None and self.schema: + self._build_writer(self.schema) + if self.pa_writer is not None: + self.pa_writer.close() + self.pa_writer = None + if close_stream: + self.stream.close() + else: + if close_stream: + self.stream.close() + raise SchemaInferenceError("Please pass `features` or at least one example when writing data") + logger.debug( + f"Done writing {self._num_examples} {self.unit} in {self._num_bytes} bytes {self._path if self._path else ''}." + ) + return self._num_examples, self._num_bytes + + +class ParquetWriter(ArrowWriter): + _WRITER_CLASS = pq.ParquetWriter diff --git a/lib/python3.12/site-packages/datasets/builder.py b/lib/python3.12/site-packages/datasets/builder.py new file mode 100644 index 0000000000000000000000000000000000000000..d6992b9e19db65e1c1512df6462d85f19c2be8e2 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/builder.py @@ -0,0 +1,1903 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +"""DatasetBuilder base class.""" + +import abc +import contextlib +import copy +import inspect +import os +import posixpath +import shutil +import textwrap +import time +import urllib +from collections.abc import Iterable, Mapping +from dataclasses import dataclass +from functools import partial +from pathlib import Path +from typing import TYPE_CHECKING, Optional, Union +from unittest.mock import patch + +import fsspec +from fsspec.core import url_to_fs +from multiprocess import Pool +from tqdm.contrib.concurrent import thread_map + +from . import config, utils +from .arrow_dataset import Dataset +from .arrow_reader import ( + ArrowReader, + ReadInstruction, +) +from .arrow_writer import ArrowWriter, ParquetWriter, SchemaInferenceError +from .data_files import DataFilesDict, DataFilesPatternsDict, sanitize_patterns +from .dataset_dict import DatasetDict, IterableDatasetDict +from .download.download_config import DownloadConfig +from .download.download_manager import DownloadManager, DownloadMode +from .download.streaming_download_manager import StreamingDownloadManager, xjoin +from .exceptions import DatasetGenerationCastError, DatasetGenerationError, FileFormatError, ManualDownloadError +from .features import Features +from .filesystems import ( + is_remote_filesystem, + rename, +) +from .fingerprint import Hasher +from .info import DatasetInfo, DatasetInfosDict, PostProcessedInfo +from .iterable_dataset import ArrowExamplesIterable, ExamplesIterable, IterableDataset +from .keyhash import DuplicatedKeysError +from .naming import INVALID_WINDOWS_CHARACTERS_IN_PATH, camelcase_to_snakecase +from .splits import Split, SplitDict, SplitGenerator, SplitInfo +from .streaming import extend_dataset_builder_for_streaming +from .table import CastError +from .utils import logging +from .utils import tqdm as hf_tqdm +from .utils._filelock import FileLock +from .utils.file_utils import is_remote_url +from .utils.info_utils import VerificationMode, get_size_checksum_dict, verify_checksums, verify_splits +from .utils.py_utils import ( + classproperty, + convert_file_size_to_int, + has_sufficient_disk_space, + iflatmap_unordered, + map_nested, + memoize, + size_str, + temporary_assignment, +) +from .utils.sharding import _number_of_shards_in_gen_kwargs, _split_gen_kwargs +from .utils.track import tracked_list + + +if TYPE_CHECKING: + from .load import DatasetModule + + +logger = logging.get_logger(__name__) + + +class InvalidConfigName(ValueError): + pass + + +@dataclass +class BuilderConfig: + """Base class for `DatasetBuilder` data configuration. + + `DatasetBuilder` subclasses with data configuration options should subclass + `BuilderConfig` and add their own properties. + + Attributes: + name (`str`, defaults to `default`): + The name of the configuration. + version (`Version` or `str`, defaults to `0.0.0`): + The version of the configuration. + data_dir (`str`, *optional*): + Path to the directory containing the source data. + data_files (`str` or `Sequence` or `Mapping`, *optional*): + Path(s) to source data file(s). + description (`str`, *optional*): + A human description of the configuration. + """ + + name: str = "default" + version: Optional[Union[utils.Version, str]] = utils.Version("0.0.0") + data_dir: Optional[str] = None + data_files: Optional[Union[DataFilesDict, DataFilesPatternsDict]] = None + description: Optional[str] = None + + def __post_init__(self): + # The config name is used to name the cache directory. + for invalid_char in INVALID_WINDOWS_CHARACTERS_IN_PATH: + if invalid_char in self.name: + raise InvalidConfigName( + f"Bad characters from black list '{INVALID_WINDOWS_CHARACTERS_IN_PATH}' found in '{self.name}'. " + f"They could create issues when creating a directory for this config on Windows filesystem." + ) + if self.data_files is not None and not isinstance(self.data_files, (DataFilesDict, DataFilesPatternsDict)): + raise ValueError(f"Expected a DataFilesDict in data_files but got {self.data_files}") + + def __eq__(self, o): + # we need to override the default dataclass __eq__ since it doesn't check for + # other attributes that the ones of the signature. + if set(self.__dict__.keys()) != set(o.__dict__.keys()): + return False + return all((k, getattr(self, k)) == (k, getattr(o, k)) for k in self.__dict__.keys()) + + def create_config_id( + self, + config_kwargs: dict, + custom_features: Optional[Features] = None, + ) -> str: + """ + The config id is used to build the cache directory. + By default it is equal to the config name. + However the name of a config is not sufficient to have a unique identifier for the dataset being generated + since it doesn't take into account: + - the config kwargs that can be used to overwrite attributes + - the custom features used to write the dataset + - the data_files for json/text/csv/pandas datasets + + Therefore the config id is just the config name with an optional suffix based on these. + """ + # Possibly add a suffix to the name to handle custom features/data_files/config_kwargs + suffix: Optional[str] = None + config_kwargs_to_add_to_suffix = config_kwargs.copy() + # name and version are already used to build the cache directory + config_kwargs_to_add_to_suffix.pop("name", None) + config_kwargs_to_add_to_suffix.pop("version", None) + # data dir handling (when specified it points to the manually downloaded data): + # it was previously ignored before the introduction of config id because we didn't want + # to change the config name. Now it's fine to take it into account for the config id. + # config_kwargs_to_add_to_suffix.pop("data_dir", None) + if "data_dir" in config_kwargs_to_add_to_suffix: + if config_kwargs_to_add_to_suffix["data_dir"] is None: + config_kwargs_to_add_to_suffix.pop("data_dir", None) + else: + # canonicalize the data dir to avoid two paths to the same location having different + # hashes + data_dir = config_kwargs_to_add_to_suffix["data_dir"] + data_dir = os.path.normpath(data_dir) + config_kwargs_to_add_to_suffix["data_dir"] = data_dir + if config_kwargs_to_add_to_suffix: + # we don't care about the order of the kwargs + config_kwargs_to_add_to_suffix = { + k: config_kwargs_to_add_to_suffix[k] for k in sorted(config_kwargs_to_add_to_suffix) + } + if all(isinstance(v, (str, bool, int, float)) for v in config_kwargs_to_add_to_suffix.values()): + suffix = ",".join( + str(k) + "=" + urllib.parse.quote_plus(str(v)) for k, v in config_kwargs_to_add_to_suffix.items() + ) + if len(suffix) > 32: # hash if too long + suffix = Hasher.hash(config_kwargs_to_add_to_suffix) + else: + suffix = Hasher.hash(config_kwargs_to_add_to_suffix) + + if custom_features is not None: + m = Hasher() + if suffix: + m.update(suffix) + m.update(custom_features) + suffix = m.hexdigest() + + if suffix: + config_id = self.name + "-" + suffix + if len(config_id) > config.MAX_DATASET_CONFIG_ID_READABLE_LENGTH: + config_id = self.name + "-" + Hasher.hash(suffix) + return config_id + else: + return self.name + + def _resolve_data_files(self, base_path: str, download_config: DownloadConfig) -> None: + if isinstance(self.data_files, DataFilesPatternsDict): + base_path = xjoin(base_path, self.data_dir) if self.data_dir else base_path + self.data_files = self.data_files.resolve(base_path, download_config) + + +class DatasetBuilder: + """Abstract base class for all datasets. + + `DatasetBuilder` has 3 key methods: + + - [`DatasetBuilder.info`]: Documents the dataset, including feature + names, types, shapes, version, splits, citation, etc. + - [`DatasetBuilder.download_and_prepare`]: Downloads the source data + and writes it to disk. + - [`DatasetBuilder.as_dataset`]: Generates a [`Dataset`]. + + Some `DatasetBuilder`s expose multiple variants of the + dataset by defining a [`BuilderConfig`] subclass and accepting a + config object (or name) on construction. Configurable datasets expose a + pre-defined set of configurations in [`DatasetBuilder.builder_configs`]. + + Args: + cache_dir (`str`, *optional*): + Directory to cache data. Defaults to `"~/.cache/huggingface/datasets"`. + dataset_name (`str`, *optional*): + Name of the dataset, if different from the builder name. Useful for packaged builders + like csv, imagefolder, audiofolder, etc. to reflect the difference between datasets + that use the same packaged builder. + config_name (`str`, *optional*): + Name of the dataset configuration. + It affects the data generated on disk. Different configurations will have their own subdirectories and + versions. + If not provided, the default configuration is used (if it exists). + + + + Parameter `name` was renamed to `config_name`. + + + hash (`str`, *optional*): + Hash specific to the dataset code. Used to update the caching directory when the + dataset loading script code is updated (to avoid reusing old data). + The typical caching directory (defined in `self._relative_data_dir`) is `name/version/hash/`. + base_path (`str`, *optional*): + Base path for relative paths that are used to download files. + This can be a remote URL. + features ([`Features`], *optional*): + Features types to use with this dataset. + It can be used to change the [`Features`] types of a dataset, for example. + token (`str` or `bool`, *optional*): + String or boolean to use as Bearer token for remote files on the + Datasets Hub. If `True`, will get token from `"~/.huggingface"`. + repo_id (`str`, *optional*): + ID of the dataset repository. + Used to distinguish builders with the same name but not coming from the same namespace, for example "rajpurkar/squad" + and "lhoestq/squad" repo IDs. In the latter, the builder name would be "lhoestq___squad". + data_files (`str` or `Sequence` or `Mapping`, *optional*): + Path(s) to source data file(s). + For builders like "csv" or "json" that need the user to specify data files. They can be either + local or remote files. For convenience, you can use a `DataFilesDict`. + data_dir (`str`, *optional*): + Path to directory containing source data file(s). + Use only if `data_files` is not passed, in which case it is equivalent to passing + `os.path.join(data_dir, "**")` as `data_files`. + For builders that require manual download, it must be the path to the local directory containing the + manually downloaded data. + storage_options (`dict`, *optional*): + Key/value pairs to be passed on to the dataset file-system backend, if any. + writer_batch_size (`int`, *optional*): + Batch size used by the ArrowWriter. + It defines the number of samples that are kept in memory before writing them + and also the length of the arrow chunks. + None means that the ArrowWriter will use its default value. + **config_kwargs (additional keyword arguments): Keyword arguments to be passed to the corresponding builder + configuration class, set on the class attribute [`DatasetBuilder.BUILDER_CONFIG_CLASS`]. The builder + configuration class is [`BuilderConfig`] or a subclass of it. + """ + + # Default version + VERSION = None # Default version set in BuilderConfig + + # Class for the builder config. + BUILDER_CONFIG_CLASS = BuilderConfig + + # Named configurations that modify the data generated by download_and_prepare. + BUILDER_CONFIGS = [] + + # Optional default config name to be used when name is None + DEFAULT_CONFIG_NAME = None + + # Default batch size used by the ArrowWriter + # It defines the number of samples that are kept in memory before writing them + # and also the length of the arrow chunks + # None means that the ArrowWriter will use its default value + DEFAULT_WRITER_BATCH_SIZE = None + + def __init__( + self, + cache_dir: Optional[str] = None, + dataset_name: Optional[str] = None, + config_name: Optional[str] = None, + hash: Optional[str] = None, + base_path: Optional[str] = None, + info: Optional[DatasetInfo] = None, + features: Optional[Features] = None, + token: Optional[Union[bool, str]] = None, + repo_id: Optional[str] = None, + data_files: Optional[Union[str, list, dict, DataFilesDict]] = None, + data_dir: Optional[str] = None, + storage_options: Optional[dict] = None, + writer_batch_size: Optional[int] = None, + **config_kwargs, + ): + # DatasetBuilder name + self.name: str = camelcase_to_snakecase(self.__module__.split(".")[-1]) + self.hash: Optional[str] = hash + self.base_path = base_path + self.token = token + self.repo_id = repo_id + self.storage_options = storage_options or {} + self.dataset_name = camelcase_to_snakecase(dataset_name) if dataset_name else self.name + self._writer_batch_size = writer_batch_size or self.DEFAULT_WRITER_BATCH_SIZE + + if data_files is not None and not isinstance(data_files, DataFilesDict): + data_files = DataFilesDict.from_patterns( + sanitize_patterns(data_files), + base_path=base_path, + download_config=DownloadConfig(token=token, storage_options=self.storage_options), + ) + + # Prepare config: DatasetConfig contains name, version and description but can be extended by each dataset + if "features" in inspect.signature(self.BUILDER_CONFIG_CLASS.__init__).parameters and features is not None: + config_kwargs["features"] = features + if data_files is not None: + config_kwargs["data_files"] = data_files + if data_dir is not None: + config_kwargs["data_dir"] = data_dir + self.config_kwargs = config_kwargs + self.config, self.config_id = self._create_builder_config( + config_name=config_name, + custom_features=features, + **config_kwargs, + ) + + # prepare info: DatasetInfo are a standardized dataclass across all datasets + # Prefill datasetinfo + if info is None: + # TODO FOR PACKAGED MODULES IT IMPORTS DATA FROM src/packaged_modules which doesn't make sense + info = self.get_exported_dataset_info() + info.update(self._info()) + info.builder_name = self.name + info.dataset_name = self.dataset_name + info.config_name = self.config.name + info.version = self.config.version + self.info = info + # update info with user specified infos + if features is not None: + self.info.features = features + + # Prepare data dirs: + # cache_dir can be a remote bucket on GCS or S3 + self._cache_dir_root = str(cache_dir or config.HF_DATASETS_CACHE) + self._cache_dir_root = ( + self._cache_dir_root if is_remote_url(self._cache_dir_root) else os.path.expanduser(self._cache_dir_root) + ) + self._cache_downloaded_dir = ( + posixpath.join(self._cache_dir_root, config.DOWNLOADED_DATASETS_DIR) + if cache_dir + else str(config.DOWNLOADED_DATASETS_PATH) + ) + self._cache_downloaded_dir = ( + self._cache_downloaded_dir + if is_remote_url(self._cache_downloaded_dir) + else os.path.expanduser(self._cache_downloaded_dir) + ) + + # In case there exists a legacy cache directory + self._legacy_relative_data_dir = None + + self._cache_dir = self._build_cache_dir() + if not is_remote_url(self._cache_dir_root): + os.makedirs(self._cache_dir_root, exist_ok=True) + lock_path = os.path.join( + self._cache_dir_root, Path(self._cache_dir).as_posix().replace("/", "_") + ".lock" + ) + with FileLock(lock_path): + if os.path.exists(self._cache_dir): # check if data exist + if len(os.listdir(self._cache_dir)) > 0: + if os.path.exists(os.path.join(self._cache_dir, config.DATASET_INFO_FILENAME)): + logger.info("Overwrite dataset info from restored data version if exists.") + self.info = DatasetInfo.from_directory(self._cache_dir) + else: # dir exists but no data, remove the empty dir as data aren't available anymore + logger.warning( + f"Old caching folder {self._cache_dir} for dataset {self.dataset_name} exists but no data were found. Removing it. " + ) + os.rmdir(self._cache_dir) + + # Store in the cache by default unless the user specifies a custom output_dir to download_and_prepare + self._output_dir = self._cache_dir + self._fs: fsspec.AbstractFileSystem = fsspec.filesystem("file") + + # Set download manager + self.dl_manager = None + + # Set to True by "datasets-cli test" to generate file checksums for (deprecated) dataset_infos.json independently of verification_mode value. + self._record_infos = False + + # Set in `.download_and_prepare` once the format of the generated dataset is known + self._file_format = None + + # Enable streaming (e.g. it patches "open" to work with remote files) + extend_dataset_builder_for_streaming(self) + + def __getstate__(self): + return self.__dict__ + + def __setstate__(self, d): + self.__dict__ = d + # Re-enable streaming, since patched functions are not kept when pickling + extend_dataset_builder_for_streaming(self) + + # Must be set for datasets that use 'data_dir' functionality - the ones + # that require users to do additional steps to download the data + # (this is usually due to some external regulations / rules). + # This field should contain a string with user instructions, including + # the list of files that should be present. It will be + # displayed in the dataset documentation. + @property + def manual_download_instructions(self) -> Optional[str]: + return None + + def _check_legacy_cache(self) -> Optional[str]: + """Check for the old cache directory template {cache_dir}/{namespace}___{builder_name} from 2.13""" + if ( + self.__module__.startswith("datasets.") + and not is_remote_url(self._cache_dir_root) + and self.config.name == "default" + ): + from .packaged_modules import _PACKAGED_DATASETS_MODULES + + namespace = self.repo_id.split("/")[0] if self.repo_id and self.repo_id.count("/") > 0 else None + config_name = self.repo_id.replace("/", "--") if self.repo_id is not None else self.dataset_name + config_id = config_name + self.config_id[len(self.config.name) :] + hash = _PACKAGED_DATASETS_MODULES.get(self.name, "missing")[1] + legacy_relative_data_dir = posixpath.join( + self.dataset_name if namespace is None else f"{namespace}___{self.dataset_name}", + config_id, + "0.0.0", + hash, + ) + legacy_cache_dir = posixpath.join(self._cache_dir_root, legacy_relative_data_dir) + if os.path.isdir(legacy_cache_dir): + return legacy_relative_data_dir + + def _check_legacy_cache2(self, dataset_module: "DatasetModule") -> Optional[str]: + """Check for the old cache directory template {cache_dir}/{namespace}___{dataset_name}/{config_name}-xxx from 2.14 and 2.15""" + if ( + self.__module__.startswith("datasets.") + and not is_remote_url(self._cache_dir_root) + and not (set(self.config_kwargs) - {"data_files", "data_dir"}) + ): + from .packaged_modules import _PACKAGED_DATASETS_MODULES_2_15_HASHES + from .utils._dill import Pickler + + def update_hash_with_config_parameters(hash: str, config_parameters: dict) -> str: + """ + Used to update hash of packaged modules which is used for creating unique cache directories to reflect + different config parameters which are passed in metadata from readme. + """ + params_to_exclude = {"config_name", "version", "description"} + params_to_add_to_hash = { + param: value + for param, value in sorted(config_parameters.items()) + if param not in params_to_exclude + } + m = Hasher() + m.update(hash) + m.update(params_to_add_to_hash) + return m.hexdigest() + + namespace = self.repo_id.split("/")[0] if self.repo_id and self.repo_id.count("/") > 0 else None + with patch.object(Pickler, "_legacy_no_dict_keys_sorting", True): + config_id = self.config.name + "-" + Hasher.hash({"data_files": self.config.data_files}) + hash = _PACKAGED_DATASETS_MODULES_2_15_HASHES.get(self.name, "missing") + if ( + dataset_module.builder_configs_parameters.metadata_configs + and self.config.name in dataset_module.builder_configs_parameters.metadata_configs + ): + hash = update_hash_with_config_parameters( + hash, dataset_module.builder_configs_parameters.metadata_configs[self.config.name] + ) + legacy_relative_data_dir = posixpath.join( + self.dataset_name if namespace is None else f"{namespace}___{self.dataset_name}", + config_id, + "0.0.0", + hash, + ) + legacy_cache_dir = posixpath.join(self._cache_dir_root, legacy_relative_data_dir) + if os.path.isdir(legacy_cache_dir): + return legacy_relative_data_dir + + @classmethod + def get_all_exported_dataset_infos(cls) -> DatasetInfosDict: + """Empty dict if doesn't exist + + Example: + + ```py + >>> from datasets import load_dataset_builder + >>> ds_builder = load_dataset_builder('vivos') + >>> ds_builder.get_all_exported_dataset_infos() + {'default': DatasetInfo(description='', citation='', homepage='', license='', features={'speaker_id': Value(dtype='string', id=None), 'path': Value(dtype='string', id=None), 'audio': Audio(sampling_rate=16000, mono=True, decode=True, id=None), 'sentence': Value(dtype='string', id=None)}, post_processed=None, supervised_keys=None, builder_name=None, dataset_name=None, config_name='default', version=None, splits={'train': SplitInfo(name='train', num_bytes=1722002133, num_examples=11660, shard_lengths=None, dataset_name=None), 'test': SplitInfo(name='test', num_bytes=86120227, num_examples=760, shard_lengths=None, dataset_name=None)}, download_checksums=None, download_size=1475540500, post_processing_size=None, dataset_size=1808122360, size_in_bytes=None)} + ``` + """ + return DatasetInfosDict.from_directory(cls.get_imported_module_dir()) + + def get_exported_dataset_info(self) -> DatasetInfo: + """Empty `DatasetInfo` if doesn't exist + + Example: + + ```py + >>> from datasets import load_dataset_builder + >>> ds_builder = load_dataset_builder('cornell-movie-review-data/rotten_tomatoes') + >>> ds_builder.get_exported_dataset_info() + DatasetInfo(description='', citation='', homepage='', license='', features={'speaker_id': Value(dtype='string', id=None), 'path': Value(dtype='string', id=None), 'audio': Audio(sampling_rate=16000, mono=True, decode=True, id=None), 'sentence': Value(dtype='string', id=None)}, post_processed=None, supervised_keys=None, builder_name=None, dataset_name=None, config_name='default', version=None, splits={'train': SplitInfo(name='train', num_bytes=1722002133, num_examples=11660, shard_lengths=None, dataset_name=None), 'test': SplitInfo(name='test', num_bytes=86120227, num_examples=760, shard_lengths=None, dataset_name=None)}, download_checksums=None, download_size=1475540500, post_processing_size=None, dataset_size=1808122360, size_in_bytes=None) + ``` + """ + return self.get_all_exported_dataset_infos().get(self.config.name, DatasetInfo()) + + def _create_builder_config( + self, config_name=None, custom_features=None, **config_kwargs + ) -> tuple[BuilderConfig, str]: + """Create and validate BuilderConfig object as well as a unique config id for this config. + Raises ValueError if there are multiple builder configs and config_name and DEFAULT_CONFIG_NAME are None. + config_kwargs override the defaults kwargs in config + """ + builder_config = None + + # try default config + if config_name is None and self.BUILDER_CONFIGS: + if self.DEFAULT_CONFIG_NAME is not None: + builder_config = self.builder_configs.get(self.DEFAULT_CONFIG_NAME) + logger.info(f"No config specified, defaulting to: {self.dataset_name}/{builder_config.name}") + else: + if len(self.BUILDER_CONFIGS) > 1: + if not config_kwargs: + example_of_usage = ( + f"load_dataset('{self.repo_id or self.dataset_name}', '{self.BUILDER_CONFIGS[0].name}')" + ) + raise ValueError( + "Config name is missing." + f"\nPlease pick one among the available configs: {list(self.builder_configs.keys())}" + + f"\nExample of usage:\n\t`{example_of_usage}`" + ) + else: + builder_config = self.BUILDER_CONFIGS[0] + logger.info( + f"No config specified, defaulting to the single config: {self.dataset_name}/{builder_config.name}" + ) + + # try to get config by name + if isinstance(config_name, str): + builder_config = self.builder_configs.get(config_name) + if builder_config is None and self.BUILDER_CONFIGS: + raise ValueError( + f"BuilderConfig '{config_name}' not found. Available: {list(self.builder_configs.keys())}" + ) + + # if not using an existing config, then create a new config on the fly + if not builder_config: + if config_name is not None: + config_kwargs["name"] = config_name + elif self.DEFAULT_CONFIG_NAME and not config_kwargs: + # Use DEFAULT_CONFIG_NAME only if no config_kwargs are passed + config_kwargs["name"] = self.DEFAULT_CONFIG_NAME + if "version" not in config_kwargs and hasattr(self, "VERSION") and self.VERSION: + config_kwargs["version"] = self.VERSION + builder_config = self.BUILDER_CONFIG_CLASS(**config_kwargs) + + # otherwise use the config_kwargs to overwrite the attributes + else: + builder_config = copy.deepcopy(builder_config) if config_kwargs else builder_config + for key, value in config_kwargs.items(): + if value is not None: + if not hasattr(builder_config, key): + raise ValueError(f"BuilderConfig {builder_config} doesn't have a '{key}' key.") + setattr(builder_config, key, value) + + if not builder_config.name: + raise ValueError(f"BuilderConfig must have a name, got {builder_config.name}") + + # resolve data files if needed + builder_config._resolve_data_files( + base_path=self.base_path, + download_config=DownloadConfig(token=self.token, storage_options=self.storage_options), + ) + + # compute the config id that is going to be used for caching + config_id = builder_config.create_config_id( + config_kwargs, + custom_features=custom_features, + ) + is_custom = (config_id not in self.builder_configs) and config_id != "default" + if is_custom: + logger.info(f"Using custom data configuration {config_id}") + else: + if ( + builder_config.name in self.builder_configs + and builder_config != self.builder_configs[builder_config.name] + ): + raise ValueError( + "Cannot name a custom BuilderConfig the same as an available " + f"BuilderConfig. Change the name. Available BuilderConfigs: {list(self.builder_configs.keys())}" + ) + if not builder_config.version: + raise ValueError(f"BuilderConfig {builder_config.name} must have a version") + + return builder_config, config_id + + @classproperty + @classmethod + @memoize() + def builder_configs(cls) -> dict[str, BuilderConfig]: + """Dictionary of pre-defined configurations for this builder class.""" + configs = {config.name: config for config in cls.BUILDER_CONFIGS} + if len(configs) != len(cls.BUILDER_CONFIGS): + names = [config.name for config in cls.BUILDER_CONFIGS] + raise ValueError(f"Names in BUILDER_CONFIGS must not be duplicated. Got {names}") + return configs + + @property + def cache_dir(self): + return self._cache_dir + + def _use_legacy_cache_dir_if_possible(self, dataset_module: "DatasetModule"): + # Check for the legacy cache directory template (datasets<3.0.0) + self._legacy_relative_data_dir = ( + self._check_legacy_cache2(dataset_module) or self._check_legacy_cache() or None + ) + self._cache_dir = self._build_cache_dir() + self._output_dir = self._cache_dir + + def _relative_data_dir(self, with_version=True, with_hash=True) -> str: + """Relative path of this dataset in cache_dir: + Will be: + self.dataset_name/self.config.version/self.hash/ + or if a repo_id with a namespace has been specified: + self.namespace___self.dataset_name/self.config.version/self.hash/ + If any of these element is missing or if ``with_version=False`` the corresponding subfolders are dropped. + """ + if self._legacy_relative_data_dir is not None and with_version and with_hash: + return self._legacy_relative_data_dir + + namespace = self.repo_id.split("/")[0] if self.repo_id and self.repo_id.count("/") > 0 else None + builder_data_dir = self.dataset_name if namespace is None else f"{namespace}___{self.dataset_name}" + builder_data_dir = posixpath.join(builder_data_dir, self.config_id) + if with_version: + builder_data_dir = posixpath.join(builder_data_dir, str(self.config.version)) + if with_hash and self.hash and isinstance(self.hash, str): + builder_data_dir = posixpath.join(builder_data_dir, self.hash) + return builder_data_dir + + def _build_cache_dir(self): + """Return the data directory for the current version.""" + builder_data_dir = posixpath.join(self._cache_dir_root, self._relative_data_dir(with_version=False)) + version_data_dir = posixpath.join(self._cache_dir_root, self._relative_data_dir(with_version=True)) + + def _other_versions_on_disk(): + """Returns previous versions on disk.""" + if not os.path.exists(builder_data_dir): + return [] + + version_dirnames = [] + for dir_name in os.listdir(builder_data_dir): + try: + version_dirnames.append((utils.Version(dir_name), dir_name)) + except ValueError: # Invalid version (ex: incomplete data dir) + pass + version_dirnames.sort(reverse=True) + return version_dirnames + + # Check and warn if other versions exist + if not is_remote_url(builder_data_dir): + version_dirs = _other_versions_on_disk() + if version_dirs: + other_version = version_dirs[0][0] + if other_version != self.config.version: + warn_msg = ( + f"Found a different version {str(other_version)} of dataset {self.dataset_name} in " + f"cache_dir {self._cache_dir_root}. Using currently defined version " + f"{str(self.config.version)}." + ) + logger.warning(warn_msg) + + return version_data_dir + + @abc.abstractmethod + def _info(self) -> DatasetInfo: + """Construct the DatasetInfo object. See `DatasetInfo` for details. + + Warning: This function is only called once and the result is cached for all + following .info() calls. + + Returns: + info: (DatasetInfo) The dataset information + """ + raise NotImplementedError + + @classmethod + def get_imported_module_dir(cls): + """Return the path of the module of this class or subclass.""" + return os.path.dirname(inspect.getfile(inspect.getmodule(cls))) + + def _rename(self, src: str, dst: str): + rename(self._fs, src, dst) + + def download_and_prepare( + self, + output_dir: Optional[str] = None, + download_config: Optional[DownloadConfig] = None, + download_mode: Optional[Union[DownloadMode, str]] = None, + verification_mode: Optional[Union[VerificationMode, str]] = None, + dl_manager: Optional[DownloadManager] = None, + base_path: Optional[str] = None, + file_format: str = "arrow", + max_shard_size: Optional[Union[int, str]] = None, + num_proc: Optional[int] = None, + storage_options: Optional[dict] = None, + **download_and_prepare_kwargs, + ): + """Downloads and prepares dataset for reading. + + Args: + output_dir (`str`, *optional*): + Output directory for the dataset. + Default to this builder's `cache_dir`, which is inside `~/.cache/huggingface/datasets` by default. + + + download_config (`DownloadConfig`, *optional*): + Specific download configuration parameters. + download_mode ([`DownloadMode`] or `str`, *optional*): + Select the download/generate mode, default to `REUSE_DATASET_IF_EXISTS`. + verification_mode ([`VerificationMode`] or `str`, defaults to `BASIC_CHECKS`): + Verification mode determining the checks to run on the downloaded/processed dataset information (checksums/size/splits/...). + + + dl_manager (`DownloadManager`, *optional*): + Specific `DownloadManger` to use. + base_path (`str`, *optional*): + Base path for relative paths that are used to download files. This can be a remote url. + If not specified, the value of the `base_path` attribute (`self.base_path`) will be used instead. + file_format (`str`, *optional*): + Format of the data files in which the dataset will be written. + Supported formats: "arrow", "parquet". Default to "arrow" format. + If the format is "parquet", then image and audio data are embedded into the Parquet files instead of pointing to local files. + + + max_shard_size (`Union[str, int]`, *optional*): + Maximum number of bytes written per shard, default is "500MB". + The size is based on uncompressed data size, so in practice your shard files may be smaller than + `max_shard_size` thanks to Parquet compression for example. + + + num_proc (`int`, *optional*, defaults to `None`): + Number of processes when downloading and generating the dataset locally. + Multiprocessing is disabled by default. + + + storage_options (`dict`, *optional*): + Key/value pairs to be passed on to the caching file-system backend, if any. + + + **download_and_prepare_kwargs (additional keyword arguments): Keyword arguments. + + Example: + + Download and prepare the dataset as Arrow files that can be loaded as a Dataset using `builder.as_dataset()`: + + ```py + >>> from datasets import load_dataset_builder + >>> builder = load_dataset_builder("cornell-movie-review-data/rotten_tomatoes") + >>> builder.download_and_prepare() + ``` + + Download and prepare the dataset as sharded Parquet files locally: + + ```py + >>> from datasets import load_dataset_builder + >>> builder = load_dataset_builder("cornell-movie-review-data/rotten_tomatoes") + >>> builder.download_and_prepare("./output_dir", file_format="parquet") + ``` + + Download and prepare the dataset as sharded Parquet files in a cloud storage: + + ```py + >>> from datasets import load_dataset_builder + >>> storage_options = {"key": aws_access_key_id, "secret": aws_secret_access_key} + >>> builder = load_dataset_builder("cornell-movie-review-data/rotten_tomatoes") + >>> builder.download_and_prepare("s3://my-bucket/my_rotten_tomatoes", storage_options=storage_options, file_format="parquet") + ``` + """ + output_dir = output_dir if output_dir is not None else self._cache_dir + # output_dir can be a remote bucket on GCS or S3 + fs, output_dir = url_to_fs(output_dir, **(storage_options or {})) + self._fs = fs + self._output_dir = output_dir if not is_remote_filesystem(self._fs) else self._fs.unstrip_protocol(output_dir) + + download_mode = DownloadMode(download_mode or DownloadMode.REUSE_DATASET_IF_EXISTS) + verification_mode = VerificationMode(verification_mode or VerificationMode.BASIC_CHECKS) + base_path = base_path if base_path is not None else self.base_path + + if file_format is not None and file_format not in ["arrow", "parquet"]: + raise ValueError(f"Unsupported file_format: {file_format}. Expected 'arrow' or 'parquet'") + self._file_format = file_format + + if self._fs._strip_protocol(self._output_dir) == "": + # We don't support the root directory, because it has no dirname, + # and we need a dirname to use a .incomplete directory + # when the dataset is being written + raise RuntimeError( + f"Unable to download and prepare the dataset at the root {self._output_dir}. " + f"Please specify a subdirectory, e.g. '{self._output_dir + self.dataset_name}'" + ) + + if dl_manager is None: + if download_config is None: + download_config = DownloadConfig( + cache_dir=self._cache_downloaded_dir, + force_download=download_mode == DownloadMode.FORCE_REDOWNLOAD, + force_extract=download_mode == DownloadMode.FORCE_REDOWNLOAD, + use_etag=False, + num_proc=num_proc, + token=self.token, + storage_options=self.storage_options, + ) # We don't use etag for data files to speed up the process + + dl_manager = DownloadManager( + dataset_name=self.dataset_name, + download_config=download_config, + data_dir=self.config.data_dir, + base_path=base_path, + record_checksums=(self._record_infos or verification_mode == VerificationMode.ALL_CHECKS), + ) + + is_local = not is_remote_filesystem(self._fs) + self.dl_manager = dl_manager + + # Prevent parallel local disk operations + if is_local: + # Create parent directory of the output_dir to put the lock file in there + Path(self._output_dir).parent.mkdir(parents=True, exist_ok=True) + lock_path = self._output_dir + "_builder.lock" + + # File locking only with local paths; no file locking on GCS or S3 + with FileLock(lock_path) if is_local else contextlib.nullcontext(): + # Check if the data already exists + data_exists = self._fs.exists(posixpath.join(self._output_dir, config.DATASET_INFO_FILENAME)) + if data_exists and download_mode == DownloadMode.REUSE_DATASET_IF_EXISTS: + logger.info(f"Found cached dataset {self.dataset_name} ({self._output_dir})") + # We need to update the info in case some splits were added in the meantime + # for example when calling load_dataset from multiple workers. + self.info = self._load_info() + self.download_post_processing_resources(dl_manager) + return + + logger.info(f"Generating dataset {self.dataset_name} ({self._output_dir})") + if is_local: # if cache dir is local, check for available space + if not has_sufficient_disk_space( + self.info.size_in_bytes or 0, directory=Path(self._output_dir).parent + ): + raise OSError( + f"Not enough disk space. Needed: {size_str(self.info.size_in_bytes or 0)} (download: {size_str(self.info.download_size or 0)}, generated: {size_str(self.info.dataset_size or 0)}, post-processed: {size_str(self.info.post_processing_size or 0)})" + ) + + @contextlib.contextmanager + def incomplete_dir(dirname): + """Create temporary dir for dirname and rename on exit.""" + if not is_local: + self._fs.makedirs(dirname, exist_ok=True) + yield dirname + else: + tmp_dir = dirname + ".incomplete" + os.makedirs(tmp_dir, exist_ok=True) + try: + yield tmp_dir + if os.path.isdir(dirname): + shutil.rmtree(dirname) + # LocalFileSystem.mv does copy + rm, it is more efficient to simply rename a local directory + shutil.move(tmp_dir, dirname) + finally: + if os.path.exists(tmp_dir): + shutil.rmtree(tmp_dir) + + # Print is intentional: we want this to always go to stdout so user has + # information needed to cancel download/preparation if needed. + # This comes right before the progress bar. + if self.info.size_in_bytes: + logger.info( + f"Downloading and preparing dataset {self.dataset_name}/{self.config.name} " + f"(download: {size_str(self.info.download_size)}, generated: {size_str(self.info.dataset_size)}, " + f"post-processed: {size_str(self.info.post_processing_size)}, " + f"total: {size_str(self.info.size_in_bytes)}) to {self._output_dir}..." + ) + else: + _dest = self._fs._strip_protocol(self._output_dir) if is_local else self._output_dir + logger.info(f"Downloading and preparing dataset {self.dataset_name}/{self.config.name} to {_dest}...") + + self._check_manual_download(dl_manager) + + # Create a tmp dir and rename to self._output_dir on successful exit. + with incomplete_dir(self._output_dir) as tmp_output_dir: + # Temporarily assign _output_dir to tmp_data_dir to avoid having to forward + # it to every sub function. + with temporary_assignment(self, "_output_dir", tmp_output_dir): + prepare_split_kwargs = {"file_format": file_format} + if max_shard_size is not None: + prepare_split_kwargs["max_shard_size"] = max_shard_size + if num_proc is not None: + prepare_split_kwargs["num_proc"] = num_proc + self._download_and_prepare( + dl_manager=dl_manager, + verification_mode=verification_mode, + **prepare_split_kwargs, + **download_and_prepare_kwargs, + ) + # Sync info + self.info.dataset_size = sum(split.num_bytes for split in self.info.splits.values()) + self.info.download_checksums = dl_manager.get_recorded_sizes_checksums() + if self.info.download_size is not None: + self.info.size_in_bytes = self.info.dataset_size + self.info.download_size + # Save info + self._save_info() + + # Download post processing resources + self.download_post_processing_resources(dl_manager) + + logger.info( + f"Dataset {self.dataset_name} downloaded and prepared to {self._output_dir}. " + f"Subsequent calls will reuse this data." + ) + + def _check_manual_download(self, dl_manager): + if self.manual_download_instructions is not None and dl_manager.manual_dir is None: + raise ManualDownloadError( + textwrap.dedent( + f"""\ + The dataset {self.dataset_name} with config {self.config.name} requires manual data. + Please follow the manual download instructions: + {self.manual_download_instructions} + Manual data can be loaded with: + datasets.load_dataset("{self.repo_id or self.dataset_name}", data_dir="")""" + ) + ) + + def _download_and_prepare(self, dl_manager, verification_mode, **prepare_split_kwargs): + """Downloads and prepares dataset for reading. + + This is the internal implementation to overwrite called when user calls + `download_and_prepare`. It should download all required data and generate + the pre-processed datasets files. + + Args: + dl_manager ([`DownloadManager`]): + `DownloadManager` used to download and cache data. + verification_mode ([`VerificationMode`]): + if `ALL_CHECKS`, perform all the verifications including checksums. + if `BASIC_CHECKS`, do not perform checksums, only perform split tests. + if `NO_CHECKS`, do not perform any verification. + prepare_split_kwargs: Additional options, such as `file_format`, `max_shard_size` + """ + # Generating data for all splits + split_dict = SplitDict(dataset_name=self.dataset_name) + split_generators_kwargs = self._make_split_generators_kwargs(prepare_split_kwargs) + split_generators = self._split_generators(dl_manager, **split_generators_kwargs) + + # Checksums verification + if verification_mode == VerificationMode.ALL_CHECKS and dl_manager.record_checksums: + verify_checksums( + self.info.download_checksums, dl_manager.get_recorded_sizes_checksums(), "dataset source files" + ) + + # Build splits + for split_generator in split_generators: + if str(split_generator.split_info.name).lower() == "all": + raise ValueError( + "`all` is a special split keyword corresponding to the " + "union of all splits, so cannot be used as key in " + "._split_generator()." + ) + + logger.info(f"Generating {split_generator.split_info.name} split") + split_dict.add(split_generator.split_info) + + try: + # Prepare split will record examples associated to the split + self._prepare_split(split_generator, **prepare_split_kwargs) + except OSError as e: + raise OSError( + "Cannot find data file. " + + (self.manual_download_instructions or "") + + "\nOriginal error:\n" + + str(e) + ) from None + # If check_duplicates is set to True , then except DuplicatedKeysError + except DuplicatedKeysError as e: + raise DuplicatedKeysError( + e.key, + e.duplicate_key_indices, + fix_msg=f"To avoid duplicate keys, please fix the dataset script {self.name}.py", + ) from None + dl_manager.manage_extracted_files() + + if verification_mode == VerificationMode.BASIC_CHECKS or verification_mode == VerificationMode.ALL_CHECKS: + verify_splits(self.info.splits, split_dict) + + # Update the info object with the splits. + self.info.splits = split_dict + self.info.download_size = dl_manager.downloaded_size + + def download_post_processing_resources(self, dl_manager): + for split in self.info.splits or []: + for resource_name, resource_file_name in self._post_processing_resources(split).items(): + if not not is_remote_filesystem(self._fs): + raise NotImplementedError(f"Post processing is not supported on filesystem {self._fs}") + if os.sep in resource_file_name: + raise ValueError(f"Resources shouldn't be in a sub-directory: {resource_file_name}") + resource_path = os.path.join(self._output_dir, resource_file_name) + if not os.path.exists(resource_path): + downloaded_resource_path = self._download_post_processing_resources( + split, resource_name, dl_manager + ) + if downloaded_resource_path: + logger.info(f"Downloaded post-processing resource {resource_name} as {resource_file_name}") + shutil.move(downloaded_resource_path, resource_path) + + def _load_info(self) -> DatasetInfo: + return DatasetInfo.from_directory(self._output_dir, storage_options=self._fs.storage_options) + + def _save_info(self): + file_lock = ( + FileLock(self._output_dir + "_info.lock") + if not is_remote_filesystem(self._fs) + else contextlib.nullcontext() + ) + with file_lock: + self.info.write_to_directory(self._output_dir, storage_options=self._fs.storage_options) + + def _save_infos(self): + file_lock = ( + FileLock(self._output_dir + "_infos.lock") + if not is_remote_filesystem(self._fs) + else contextlib.nullcontext() + ) + with file_lock: + DatasetInfosDict(**{self.config.name: self.info}).write_to_directory(self.get_imported_module_dir()) + + def _make_split_generators_kwargs(self, prepare_split_kwargs): + """Get kwargs for `self._split_generators()` from `prepare_split_kwargs`.""" + del prepare_split_kwargs + return {} + + def as_dataset( + self, + split: Optional[Split] = None, + run_post_process=True, + verification_mode: Optional[Union[VerificationMode, str]] = None, + in_memory=False, + ) -> Union[Dataset, DatasetDict]: + """Return a Dataset for the specified split. + + Args: + split (`datasets.Split`): + Which subset of the data to return. + run_post_process (`bool`, defaults to `True`): + Whether to run post-processing dataset transforms and/or add + indexes. + verification_mode ([`VerificationMode`] or `str`, defaults to `BASIC_CHECKS`): + Verification mode determining the checks to run on the + downloaded/processed dataset information (checksums/size/splits/...). + + + in_memory (`bool`, defaults to `False`): + Whether to copy the data in-memory. + + Returns: + datasets.Dataset + + Example: + + ```py + >>> from datasets import load_dataset_builder + >>> builder = load_dataset_builder('cornell-movie-review-data/rotten_tomatoes') + >>> builder.download_and_prepare() + >>> ds = builder.as_dataset(split='train') + >>> ds + Dataset({ + features: ['text', 'label'], + num_rows: 8530 + }) + ``` + """ + if self._file_format is not None and self._file_format != "arrow": + raise FileFormatError('Loading a dataset not written in the "arrow" format is not supported.') + if is_remote_filesystem(self._fs): + raise NotImplementedError(f"Loading a dataset cached in a {type(self._fs).__name__} is not supported.") + if not os.path.exists(self._output_dir): + raise FileNotFoundError( + f"Dataset {self.dataset_name}: could not find data in {self._output_dir}. Please make sure to call " + "builder.download_and_prepare(), or use " + "datasets.load_dataset() before trying to access the Dataset object." + ) + + logger.debug(f"Constructing Dataset for split {split or ', '.join(self.info.splits)}, from {self._output_dir}") + + # By default, return all splits + if split is None: + split = {s: s for s in self.info.splits} + + verification_mode = VerificationMode(verification_mode or VerificationMode.BASIC_CHECKS) + + # Create a dataset for each of the given splits + datasets = map_nested( + partial( + self._build_single_dataset, + run_post_process=run_post_process, + verification_mode=verification_mode, + in_memory=in_memory, + ), + split, + map_tuple=True, + disable_tqdm=True, + ) + if isinstance(datasets, dict): + datasets = DatasetDict(datasets) + return datasets + + def _build_single_dataset( + self, + split: Union[str, ReadInstruction, Split], + run_post_process: bool, + verification_mode: VerificationMode, + in_memory: bool = False, + ): + """as_dataset for a single split.""" + if not isinstance(split, ReadInstruction): + split = str(split) + if split == "all": + split = "+".join(self.info.splits.keys()) + split = Split(split) + + # Build base dataset + ds = self._as_dataset( + split=split, + in_memory=in_memory, + ) + if run_post_process: + for resource_file_name in self._post_processing_resources(split).values(): + if os.sep in resource_file_name: + raise ValueError(f"Resources shouldn't be in a sub-directory: {resource_file_name}") + resources_paths = { + resource_name: os.path.join(self._output_dir, resource_file_name) + for resource_name, resource_file_name in self._post_processing_resources(split).items() + } + post_processed = self._post_process(ds, resources_paths) + if post_processed is not None: + ds = post_processed + recorded_checksums = {} + record_checksums = False + for resource_name, resource_path in resources_paths.items(): + size_checksum = get_size_checksum_dict(resource_path) + recorded_checksums[resource_name] = size_checksum + if verification_mode == VerificationMode.ALL_CHECKS and record_checksums: + if self.info.post_processed is None or self.info.post_processed.resources_checksums is None: + expected_checksums = None + else: + expected_checksums = self.info.post_processed.resources_checksums.get(split) + verify_checksums(expected_checksums, recorded_checksums, "post processing resources") + if self.info.post_processed is None: + self.info.post_processed = PostProcessedInfo() + if self.info.post_processed.resources_checksums is None: + self.info.post_processed.resources_checksums = {} + self.info.post_processed.resources_checksums[str(split)] = recorded_checksums + self.info.post_processing_size = sum( + checksums_dict["num_bytes"] + for split_checksums_dicts in self.info.post_processed.resources_checksums.values() + for checksums_dict in split_checksums_dicts.values() + ) + if self.info.dataset_size is not None and self.info.download_size is not None: + self.info.size_in_bytes = ( + self.info.dataset_size + self.info.download_size + self.info.post_processing_size + ) + self._save_info() + ds._info.post_processed = self.info.post_processed + ds._info.post_processing_size = self.info.post_processing_size + ds._info.size_in_bytes = self.info.size_in_bytes + if self.info.post_processed.features is not None: + if self.info.post_processed.features.type != ds.features.type: + raise ValueError( + f"Post-processed features info don't match the dataset:\nGot\n{self.info.post_processed.features}\nbut expected something like\n{ds.features}" + ) + else: + ds.info.features = self.info.post_processed.features + + return ds + + def _as_dataset(self, split: Union[ReadInstruction, Split] = Split.TRAIN, in_memory: bool = False) -> Dataset: + """Constructs a `Dataset`. + + This is the internal implementation to overwrite called when user calls + `as_dataset`. It should read the pre-processed datasets files and generate + the `Dataset` object. + + Args: + split (`datasets.Split`): + which subset of the data to read. + in_memory (`bool`, defaults to `False`): + Whether to copy the data in-memory. + + Returns: + `Dataset` + """ + cache_dir = self._fs._strip_protocol(self._output_dir) + dataset_name = self.dataset_name + if self._check_legacy_cache(): + dataset_name = self.name + dataset_kwargs = ArrowReader(cache_dir, self.info).read( + name=dataset_name, + instructions=split, + split_infos=self.info.splits.values(), + in_memory=in_memory, + ) + fingerprint = self._get_dataset_fingerprint(split) + return Dataset(fingerprint=fingerprint, **dataset_kwargs) + + def _get_dataset_fingerprint(self, split: Union[ReadInstruction, Split]) -> str: + """The dataset fingerprint is the hash of the relative directory dataset_name/config_name/version/hash, as well as the split specs.""" + hasher = Hasher() + hasher.update(Path(self._relative_data_dir()).as_posix()) + hasher.update(str(split)) # for example: train, train+test, train[:10%], test[:33%](pct1_dropremainder) + fingerprint = hasher.hexdigest() + return fingerprint + + def as_streaming_dataset( + self, + split: Optional[str] = None, + base_path: Optional[str] = None, + ) -> Union[dict[str, IterableDataset], IterableDataset]: + if is_remote_filesystem(self._fs): + raise NotImplementedError( + f"Loading a streaming dataset cached in a {type(self._fs).__name__} is not supported yet." + ) + + dl_manager = StreamingDownloadManager( + base_path=base_path or self.base_path, + download_config=DownloadConfig(token=self.token, storage_options=self.storage_options), + dataset_name=self.dataset_name, + data_dir=self.config.data_dir, + ) + self._check_manual_download(dl_manager) + splits_generators = {sg.name: sg for sg in self._split_generators(dl_manager)} + # By default, return all splits + if split is None: + splits_generator = splits_generators + elif split in splits_generators: + splits_generator = splits_generators[split] + else: + raise ValueError(f"Bad split: {split}. Available splits: {list(splits_generators)}") + + # Create a dataset for each of the given splits + datasets = map_nested( + self._as_streaming_dataset_single, + splits_generator, + map_tuple=True, + ) + if isinstance(datasets, dict): + datasets = IterableDatasetDict(datasets) + return datasets + + def _as_streaming_dataset_single( + self, + splits_generator, + ) -> IterableDataset: + ex_iterable = self._get_examples_iterable_for_split(splits_generator) + # add auth to be able to access and decode audio/image files from private repositories. + token_per_repo_id = {self.repo_id: self.token} if self.repo_id else {} + return IterableDataset( + ex_iterable, info=self.info, split=splits_generator.name, token_per_repo_id=token_per_repo_id + ) + + def _post_process(self, dataset: Dataset, resources_paths: Mapping[str, str]) -> Optional[Dataset]: + """Run dataset transforms or add indexes""" + return None + + def _post_processing_resources(self, split: str) -> dict[str, str]: + """Mapping resource_name -> resource_file_name""" + return {} + + def _download_post_processing_resources( + self, split: str, resource_name: str, dl_manager: DownloadManager + ) -> Optional[str]: + """Download the resource using the download manager and return the downloaded path.""" + return None + + @abc.abstractmethod + def _split_generators(self, dl_manager: Union[DownloadManager, StreamingDownloadManager]): + """Specify feature dictionary generators and dataset splits. + + This function returns a list of `SplitGenerator`s defining how to generate + data and what splits to use. + + Example: + + return [ + datasets.SplitGenerator( + name=datasets.Split.TRAIN, + gen_kwargs={'file': 'train_data.zip'}, + ), + datasets.SplitGenerator( + name=datasets.Split.TEST, + gen_kwargs={'file': 'test_data.zip'}, + ), + ] + + The above code will first call `_generate_examples(file='train_data.zip')` + to write the train data, then `_generate_examples(file='test_data.zip')` to + write the test data. + + Datasets are typically split into different subsets to be used at various + stages of training and evaluation. + + Note that for datasets without a `VALIDATION` split, you can use a + fraction of the `TRAIN` data for evaluation as you iterate on your model + so as not to overfit to the `TEST` data. + + For downloads and extractions, use the given `download_manager`. + Note that the `DownloadManager` caches downloads, so it is fine to have each + generator attempt to download the source data. + + A good practice is to download all data in this function, and then + distribute the relevant parts to each split with the `gen_kwargs` argument + + Args: + dl_manager (`Union[DownloadManager, StreamingDownloadManager]`): + Download manager to download the data + + Returns: + `list`. + """ + raise NotImplementedError() + + @abc.abstractmethod + def _prepare_split( + self, + split_generator: SplitGenerator, + file_format: str = "arrow", + max_shard_size: Optional[Union[str, int]] = None, + num_proc: Optional[int] = None, + **kwargs, + ): + """Generate the examples and record them on disk. + + Args: + split_generator (`SplitGenerator`): + Split generator to process + file_format (`str`, *optional*): + format of the data files in which the dataset will be written. + Supported formats: "arrow", "parquet". Default to "arrow" format. + max_shard_size (`Union[str, int]`, *optional*): + Maximum number of bytes written per shard, default is "500MB". + The size is based on uncompressed data size, so in practice your shard files may be smaller than + `max_shard_size` thanks to Parquet compression for example. + num_proc (`int`, *optional*, defaults to `None`): + Number of processes when downloading and generating the dataset locally. + Multiprocessing is disabled by default. + + + **kwargs: Additional kwargs forwarded from _download_and_prepare + """ + raise NotImplementedError() + + def _get_examples_iterable_for_split(self, split_generator: SplitGenerator) -> ExamplesIterable: + """Generate the examples on the fly. + + Args: + split_generator (`SplitGenerator`): + Split generator to process + """ + raise NotImplementedError() + + +class GeneratorBasedBuilder(DatasetBuilder): + """Base class for datasets with data generation based on dict generators. + + `GeneratorBasedBuilder` is a convenience class that abstracts away much + of the data writing and reading of `DatasetBuilder`. It expects subclasses to + implement generators of feature dictionaries across the dataset splits + (`_split_generators`). See the method docstrings for details. + """ + + @abc.abstractmethod + def _generate_examples(self, **kwargs): + """Default function generating examples for each `SplitGenerator`. + + This function preprocess the examples from the raw data to the preprocessed + dataset files. + This function is called once for each `SplitGenerator` defined in + `_split_generators`. The examples yielded here will be written on + disk. + + Args: + **kwargs (additional keyword arguments): + Arguments forwarded from the SplitGenerator.gen_kwargs + + Yields: + key: `str` or `int`, a unique deterministic example identification key. + * Unique: An error will be raised if two examples are yield with the + same key. + * Deterministic: When generating the dataset twice, the same example + should have the same key. + Good keys can be the image id, or line number if examples are extracted + from a text file. + The key will be hashed and sorted to shuffle examples deterministically, + such as generating the dataset multiple times keep examples in the + same order. + example: `dict`, a feature dictionary + ready to be encoded and written to disk. The example will be + encoded with `self.info.features.encode_example({...})`. + """ + raise NotImplementedError() + + def _prepare_split( + self, + split_generator: SplitGenerator, + check_duplicate_keys: bool, + file_format="arrow", + num_proc: Optional[int] = None, + max_shard_size: Optional[Union[int, str]] = None, + ): + max_shard_size = convert_file_size_to_int(max_shard_size or config.MAX_SHARD_SIZE) + + if self.info.splits is not None: + split_info = self.info.splits[split_generator.name] + else: + split_info = split_generator.split_info + + SUFFIX = "-JJJJJ-SSSSS-of-NNNNN" + fname = f"{self.dataset_name}-{split_generator.name}{SUFFIX}.{file_format}" + fpath = posixpath.join(self._output_dir, fname) + + if num_proc and num_proc > 1: + num_input_shards = _number_of_shards_in_gen_kwargs(split_generator.gen_kwargs) + if num_input_shards <= 1: + logger.warning( + f"Setting num_proc from {num_proc} back to 1 for the {split_info.name} split to disable multiprocessing as it only contains one shard." + ) + num_proc = 1 + elif num_input_shards < num_proc: + logger.warning( + f"Setting num_proc from {num_proc} to {num_input_shards} for the {split_info.name} split as it only contains {num_input_shards} shards." + ) + num_proc = num_input_shards + + pbar = hf_tqdm( + unit=" examples", + total=split_info.num_examples, + desc=f"Generating {split_info.name} split", + ) + + _prepare_split_args = { + "fpath": fpath, + "file_format": file_format, + "max_shard_size": max_shard_size, + "split_info": split_info, + "check_duplicate_keys": check_duplicate_keys, + } + + if num_proc is None or num_proc == 1: + result = None + gen_kwargs = split_generator.gen_kwargs + job_id = 0 + with pbar: + for job_id, done, content in self._prepare_split_single( + gen_kwargs=gen_kwargs, job_id=job_id, **_prepare_split_args + ): + if done: + result = content + else: + pbar.update(content) + # wrapping everything into lists for consistency with the multiprocessed code path + assert result is not None, "Failed to retrieve results from prepare_split" + examples_per_job, bytes_per_job, features_per_job, shards_per_job, shard_lengths_per_job = ( + [item] for item in result + ) + else: + kwargs_per_job = [ + {"gen_kwargs": gen_kwargs, "job_id": job_id, **_prepare_split_args} + for job_id, gen_kwargs in enumerate( + _split_gen_kwargs(split_generator.gen_kwargs, max_num_jobs=num_proc) + ) + ] + num_jobs = len(kwargs_per_job) + + examples_per_job = [None] * num_jobs + bytes_per_job = [None] * num_jobs + features_per_job = [None] * num_jobs + shards_per_job = [None] * num_jobs + shard_lengths_per_job = [None] * num_jobs + + with Pool(num_proc) as pool: + with pbar: + for job_id, done, content in iflatmap_unordered( + pool, self._prepare_split_single, kwargs_iterable=kwargs_per_job + ): + if done: + # the content is the result of the job + ( + examples_per_job[job_id], + bytes_per_job[job_id], + features_per_job[job_id], + shards_per_job[job_id], + shard_lengths_per_job[job_id], + ) = content + else: + # the content is the number of examples progress update + pbar.update(content) + + assert None not in examples_per_job, ( + f"Failed to retrieve results from prepare_split: result list {examples_per_job} still contains None - at least one worker failed to return its results" + ) + + total_shards = sum(shards_per_job) + total_num_examples = sum(examples_per_job) + total_num_bytes = sum(bytes_per_job) + features = features_per_job[0] + + split_generator.split_info.num_examples = total_num_examples + split_generator.split_info.num_bytes = total_num_bytes + + # should rename everything at the end + logger.debug(f"Renaming {total_shards} shards.") + if total_shards > 1: + # use the -SSSSS-of-NNNNN pattern + + def _rename_shard(shard_and_job: tuple[int]): + shard_id, job_id = shard_and_job + global_shard_id = sum(shards_per_job[:job_id]) + shard_id + self._rename( + fpath.replace("SSSSS", f"{shard_id:05d}").replace("JJJJJ", f"{job_id:05d}"), + fpath.replace("JJJJJ-SSSSS", f"{global_shard_id:05d}").replace("NNNNN", f"{total_shards:05d}"), + ) + + shards_and_jobs = [ + (shard_id, job_id) + for job_id, num_shards in enumerate(shards_per_job) + for shard_id in range(num_shards) + ] + thread_map(_rename_shard, shards_and_jobs, disable=True, max_workers=64) + + split_generator.split_info.shard_lengths = [ + shard_length for shard_lengths in shard_lengths_per_job for shard_length in shard_lengths + ] + else: + # don't use any pattern + shard_id, job_id = 0, 0 + self._rename( + fpath.replace("SSSSS", f"{shard_id:05d}").replace("JJJJJ", f"{job_id:05d}"), + fpath.replace(SUFFIX, ""), + ) + + if self.info.features is None: + self.info.features = features + + def _prepare_split_single( + self, + gen_kwargs: dict, + fpath: str, + file_format: str, + max_shard_size: int, + split_info: SplitInfo, + check_duplicate_keys: bool, + job_id: int, + ) -> Iterable[tuple[int, bool, Union[int, tuple]]]: + generator = self._generate_examples(**gen_kwargs) + writer_class = ParquetWriter if file_format == "parquet" else ArrowWriter + embed_local_files = file_format == "parquet" + shard_lengths = [] + total_num_examples, total_num_bytes = 0, 0 + + shard_id = 0 + num_examples_progress_update = 0 + try: + writer = writer_class( + features=self.info.features, + path=fpath.replace("SSSSS", f"{shard_id:05d}").replace("JJJJJ", f"{job_id:05d}"), + writer_batch_size=self._writer_batch_size, + hash_salt=split_info.name, + check_duplicates=check_duplicate_keys, + storage_options=self._fs.storage_options, + embed_local_files=embed_local_files, + ) + try: + _time = time.time() + for key, record in generator: + if max_shard_size is not None and writer._num_bytes > max_shard_size: + num_examples, num_bytes = writer.finalize() + writer.close() + shard_lengths.append(num_examples) + total_num_examples += num_examples + total_num_bytes += num_bytes + shard_id += 1 + writer = writer_class( + features=writer._features, + path=fpath.replace("SSSSS", f"{shard_id:05d}").replace("JJJJJ", f"{job_id:05d}"), + writer_batch_size=self._writer_batch_size, + hash_salt=split_info.name, + check_duplicates=check_duplicate_keys, + storage_options=self._fs.storage_options, + embed_local_files=embed_local_files, + ) + example = self.info.features.encode_example(record) if self.info.features is not None else record + writer.write(example, key) + num_examples_progress_update += 1 + if time.time() > _time + config.PBAR_REFRESH_TIME_INTERVAL: + _time = time.time() + yield job_id, False, num_examples_progress_update + num_examples_progress_update = 0 + finally: + yield job_id, False, num_examples_progress_update + num_shards = shard_id + 1 + num_examples, num_bytes = writer.finalize() + writer.close() + shard_lengths.append(num_examples) + total_num_examples += num_examples + total_num_bytes += num_bytes + except Exception as e: + # Ignore the writer's error for no examples written to the file if this error was caused by the error in _generate_examples before the first example was yielded + if isinstance(e, SchemaInferenceError) and e.__context__ is not None: + e = e.__context__ + raise DatasetGenerationError("An error occurred while generating the dataset") from e + + yield job_id, True, (total_num_examples, total_num_bytes, writer._features, num_shards, shard_lengths) + + def _download_and_prepare(self, dl_manager, verification_mode, **prepare_splits_kwargs): + super()._download_and_prepare( + dl_manager, + verification_mode, + check_duplicate_keys=verification_mode == VerificationMode.BASIC_CHECKS + or verification_mode == VerificationMode.ALL_CHECKS, + **prepare_splits_kwargs, + ) + + def _get_examples_iterable_for_split(self, split_generator: SplitGenerator) -> ExamplesIterable: + return ExamplesIterable(self._generate_examples, split_generator.gen_kwargs) + + +class ArrowBasedBuilder(DatasetBuilder): + """Base class for datasets with data generation based on Arrow loading functions (CSV/JSON/Parquet).""" + + @abc.abstractmethod + def _generate_tables(self, **kwargs): + """Default function generating examples for each `SplitGenerator`. + + This function preprocess the examples from the raw data to the preprocessed + dataset files. + This function is called once for each `SplitGenerator` defined in + `_split_generators`. The examples yielded here will be written on + disk. + + Args: + **kwargs (additional keyword arguments): + Arguments forwarded from the SplitGenerator.gen_kwargs + + Yields: + key: `str` or `int`, a unique deterministic example identification key. + * Unique: An error will be raised if two examples are yield with the + same key. + * Deterministic: When generating the dataset twice, the same example + should have the same key. + Good keys can be the image id, or line number if examples are extracted + from a text file. + The key will be hashed and sorted to shuffle examples deterministically, + such as generating the dataset multiple times keep examples in the + same order. + example: `pyarrow.Table`, a feature table + ready to be encoded and written to disk. + """ + raise NotImplementedError() + + def _prepare_split( + self, + split_generator: SplitGenerator, + file_format: str = "arrow", + num_proc: Optional[int] = None, + max_shard_size: Optional[Union[str, int]] = None, + ): + max_shard_size = convert_file_size_to_int(max_shard_size or config.MAX_SHARD_SIZE) + + try: + split_info = self.info.splits[split_generator.name] + except Exception: + split_info = split_generator.split_info + + SUFFIX = "-JJJJJ-SSSSS-of-NNNNN" + fname = f"{self.dataset_name}-{split_generator.name}{SUFFIX}.{file_format}" + fpath = posixpath.join(self._output_dir, fname) + + if num_proc and num_proc > 1: + num_input_shards = _number_of_shards_in_gen_kwargs(split_generator.gen_kwargs) + if num_input_shards <= 1: + logger.warning( + f"Setting num_proc from {num_proc} back to 1 for the {split_info.name} split to disable multiprocessing as it only contains one shard." + ) + num_proc = 1 + elif num_input_shards < num_proc: + logger.warning( + f"Setting num_proc from {num_proc} to {num_input_shards} for the {split_info.name} split as it only contains {num_input_shards} shards." + ) + num_proc = num_input_shards + + pbar = hf_tqdm( + unit=" examples", + total=split_info.num_examples, + desc=f"Generating {split_info.name} split", + ) + + _prepare_split_args = { + "fpath": fpath, + "file_format": file_format, + "max_shard_size": max_shard_size, + } + + if num_proc is None or num_proc == 1: + result = None + gen_kwargs = split_generator.gen_kwargs + job_id = 0 + with pbar: + for job_id, done, content in self._prepare_split_single( + gen_kwargs=gen_kwargs, job_id=job_id, **_prepare_split_args + ): + if done: + result = content + else: + pbar.update(content) + # wrapping everything into lists for consistency with the multiprocessed code path + assert result is not None, "Failed to retrieve results from prepare_split" + examples_per_job, bytes_per_job, features_per_job, shards_per_job, shard_lengths_per_job = ( + [item] for item in result + ) + else: + kwargs_per_job = [ + {"gen_kwargs": gen_kwargs, "job_id": job_id, **_prepare_split_args} + for job_id, gen_kwargs in enumerate( + _split_gen_kwargs(split_generator.gen_kwargs, max_num_jobs=num_proc) + ) + ] + num_jobs = len(kwargs_per_job) + + examples_per_job = [None] * num_jobs + bytes_per_job = [None] * num_jobs + features_per_job = [None] * num_jobs + shards_per_job = [None] * num_jobs + shard_lengths_per_job = [None] * num_jobs + + with Pool(num_proc) as pool: + with pbar: + for job_id, done, content in iflatmap_unordered( + pool, self._prepare_split_single, kwargs_iterable=kwargs_per_job + ): + if done: + # the content is the result of the job + ( + examples_per_job[job_id], + bytes_per_job[job_id], + features_per_job[job_id], + shards_per_job[job_id], + shard_lengths_per_job[job_id], + ) = content + else: + # the content is the number of examples progress update + pbar.update(content) + + assert None not in examples_per_job, ( + f"Failed to retrieve results from prepare_split: result list {examples_per_job} still contains None - at least one worker failed to return its results" + ) + + total_shards = sum(shards_per_job) + total_num_examples = sum(examples_per_job) + total_num_bytes = sum(bytes_per_job) + features = features_per_job[0] + + split_generator.split_info.num_examples = total_num_examples + split_generator.split_info.num_bytes = total_num_bytes + + # should rename everything at the end + logger.debug(f"Renaming {total_shards} shards.") + if total_shards > 1: + # use the -SSSSS-of-NNNNN pattern + + def _rename_shard(shard_id_and_job: tuple[int]): + shard_id, job_id = shard_id_and_job + global_shard_id = sum(shards_per_job[:job_id]) + shard_id + self._rename( + fpath.replace("SSSSS", f"{shard_id:05d}").replace("JJJJJ", f"{job_id:05d}"), + fpath.replace("JJJJJ-SSSSS", f"{global_shard_id:05d}").replace("NNNNN", f"{total_shards:05d}"), + ) + + shard_ids_and_jobs = [ + (shard_id, job_id) + for job_id, num_shards in enumerate(shards_per_job) + for shard_id in range(num_shards) + ] + thread_map(_rename_shard, shard_ids_and_jobs, disable=True, max_workers=64) + + split_generator.split_info.shard_lengths = [ + shard_length for shard_lengths in shard_lengths_per_job for shard_length in shard_lengths + ] + else: + # don't use any pattern + shard_id, job_id = 0, 0 + self._rename( + fpath.replace("SSSSS", f"{shard_id:05d}").replace("JJJJJ", f"{job_id:05d}"), + fpath.replace(SUFFIX, ""), + ) + + if self.info.features is None: + self.info.features = features + + def _prepare_split_single( + self, gen_kwargs: dict, fpath: str, file_format: str, max_shard_size: int, job_id: int + ) -> Iterable[tuple[int, bool, Union[int, tuple]]]: + gen_kwargs = {k: tracked_list(v) if isinstance(v, list) else v for k, v in gen_kwargs.items()} + generator = self._generate_tables(**gen_kwargs) + writer_class = ParquetWriter if file_format == "parquet" else ArrowWriter + embed_local_files = file_format == "parquet" + shard_lengths = [] + total_num_examples, total_num_bytes = 0, 0 + + shard_id = 0 + num_examples_progress_update = 0 + try: + writer = writer_class( + features=self.info.features, + path=fpath.replace("SSSSS", f"{shard_id:05d}").replace("JJJJJ", f"{job_id:05d}"), + writer_batch_size=self._writer_batch_size, + storage_options=self._fs.storage_options, + embed_local_files=embed_local_files, + ) + try: + _time = time.time() + for _, table in generator: + if max_shard_size is not None and writer._num_bytes > max_shard_size: + num_examples, num_bytes = writer.finalize() + writer.close() + shard_lengths.append(num_examples) + total_num_examples += num_examples + total_num_bytes += num_bytes + shard_id += 1 + writer = writer_class( + features=writer._features, + path=fpath.replace("SSSSS", f"{shard_id:05d}").replace("JJJJJ", f"{job_id:05d}"), + writer_batch_size=self._writer_batch_size, + storage_options=self._fs.storage_options, + embed_local_files=embed_local_files, + ) + try: + writer.write_table(table) + except CastError as cast_error: + raise DatasetGenerationCastError.from_cast_error( + cast_error=cast_error, + builder_name=self.info.builder_name, + gen_kwargs=gen_kwargs, + token=self.token, + ) + num_examples_progress_update += len(table) + if time.time() > _time + config.PBAR_REFRESH_TIME_INTERVAL: + _time = time.time() + yield job_id, False, num_examples_progress_update + num_examples_progress_update = 0 + finally: + yield job_id, False, num_examples_progress_update + num_shards = shard_id + 1 + num_examples, num_bytes = writer.finalize() + writer.close() + shard_lengths.append(num_examples) + total_num_examples += num_examples + total_num_bytes += num_bytes + except Exception as e: + # Ignore the writer's error for no examples written to the file if this error was caused by the error in _generate_examples before the first example was yielded + if isinstance(e, SchemaInferenceError) and e.__context__ is not None: + e = e.__context__ + if isinstance(e, DatasetGenerationError): + raise + raise DatasetGenerationError("An error occurred while generating the dataset") from e + + yield job_id, True, (total_num_examples, total_num_bytes, writer._features, num_shards, shard_lengths) + + def _get_examples_iterable_for_split(self, split_generator: SplitGenerator) -> ExamplesIterable: + return ArrowExamplesIterable(self._generate_tables, kwargs=split_generator.gen_kwargs) diff --git a/lib/python3.12/site-packages/datasets/combine.py b/lib/python3.12/site-packages/datasets/combine.py new file mode 100644 index 0000000000000000000000000000000000000000..3b253e240c5520f53f599a261b3228b8c618a52f --- /dev/null +++ b/lib/python3.12/site-packages/datasets/combine.py @@ -0,0 +1,215 @@ +from typing import Optional, TypeVar + +from .arrow_dataset import Dataset, _concatenate_map_style_datasets, _interleave_map_style_datasets +from .dataset_dict import DatasetDict, IterableDatasetDict +from .info import DatasetInfo +from .iterable_dataset import IterableDataset, _concatenate_iterable_datasets, _interleave_iterable_datasets +from .splits import NamedSplit +from .utils import logging +from .utils.py_utils import Literal + + +logger = logging.get_logger(__name__) + + +DatasetType = TypeVar("DatasetType", Dataset, IterableDataset) + + +def interleave_datasets( + datasets: list[DatasetType], + probabilities: Optional[list[float]] = None, + seed: Optional[int] = None, + info: Optional[DatasetInfo] = None, + split: Optional[NamedSplit] = None, + stopping_strategy: Literal["first_exhausted", "all_exhausted"] = "first_exhausted", +) -> DatasetType: + """ + Interleave several datasets (sources) into a single dataset. + The new dataset is constructed by alternating between the sources to get the examples. + + You can use this function on a list of [`Dataset`] objects, or on a list of [`IterableDataset`] objects. + + - If `probabilities` is `None` (default) the new dataset is constructed by cycling between each source to get the examples. + - If `probabilities` is not `None`, the new dataset is constructed by getting examples from a random source at a time according to the provided probabilities. + + The resulting dataset ends when one of the source datasets runs out of examples except when `oversampling` is `True`, + in which case, the resulting dataset ends when all datasets have ran out of examples at least one time. + + Note for iterable datasets: + + In a distributed setup or in PyTorch DataLoader workers, the stopping strategy is applied per process. + Therefore the "first_exhausted" strategy on an sharded iterable dataset can generate less samples in total (up to 1 missing sample per subdataset per worker). + + Args: + datasets (`List[Dataset]` or `List[IterableDataset]`): + List of datasets to interleave. + probabilities (`List[float]`, *optional*, defaults to `None`): + If specified, the new dataset is constructed by sampling + examples from one source at a time according to these probabilities. + seed (`int`, *optional*, defaults to `None`): + The random seed used to choose a source for each example. + info ([`DatasetInfo`], *optional*): + Dataset information, like description, citation, etc. + + split ([`NamedSplit`], *optional*): + Name of the dataset split. + + stopping_strategy (`str`, defaults to `first_exhausted`): + Two strategies are proposed right now, `first_exhausted` and `all_exhausted`. + By default, `first_exhausted` is an undersampling strategy, i.e the dataset construction is stopped as soon as one dataset has ran out of samples. + If the strategy is `all_exhausted`, we use an oversampling strategy, i.e the dataset construction is stopped as soon as every samples of every dataset has been added at least once. + Note that if the strategy is `all_exhausted`, the interleaved dataset size can get enormous: + - with no probabilities, the resulting dataset will have `max_length_datasets*nb_dataset` samples. + - with given probabilities, the resulting dataset will have more samples if some datasets have really low probability of visiting. + Returns: + [`Dataset`] or [`IterableDataset`]: Return type depends on the input `datasets` + parameter. `Dataset` if the input is a list of `Dataset`, `IterableDataset` if the input is a list of + `IterableDataset`. + + Example: + + For regular datasets (map-style): + + ```python + >>> from datasets import Dataset, interleave_datasets + >>> d1 = Dataset.from_dict({"a": [0, 1, 2]}) + >>> d2 = Dataset.from_dict({"a": [10, 11, 12]}) + >>> d3 = Dataset.from_dict({"a": [20, 21, 22]}) + >>> dataset = interleave_datasets([d1, d2, d3], probabilities=[0.7, 0.2, 0.1], seed=42, stopping_strategy="all_exhausted") + >>> dataset["a"] + [10, 0, 11, 1, 2, 20, 12, 10, 0, 1, 2, 21, 0, 11, 1, 2, 0, 1, 12, 2, 10, 0, 22] + >>> dataset = interleave_datasets([d1, d2, d3], probabilities=[0.7, 0.2, 0.1], seed=42) + >>> dataset["a"] + [10, 0, 11, 1, 2] + >>> dataset = interleave_datasets([d1, d2, d3]) + >>> dataset["a"] + [0, 10, 20, 1, 11, 21, 2, 12, 22] + >>> dataset = interleave_datasets([d1, d2, d3], stopping_strategy="all_exhausted") + >>> dataset["a"] + [0, 10, 20, 1, 11, 21, 2, 12, 22] + >>> d1 = Dataset.from_dict({"a": [0, 1, 2]}) + >>> d2 = Dataset.from_dict({"a": [10, 11, 12, 13]}) + >>> d3 = Dataset.from_dict({"a": [20, 21, 22, 23, 24]}) + >>> dataset = interleave_datasets([d1, d2, d3]) + >>> dataset["a"] + [0, 10, 20, 1, 11, 21, 2, 12, 22] + >>> dataset = interleave_datasets([d1, d2, d3], stopping_strategy="all_exhausted") + >>> dataset["a"] + [0, 10, 20, 1, 11, 21, 2, 12, 22, 0, 13, 23, 1, 10, 24] + >>> dataset = interleave_datasets([d1, d2, d3], probabilities=[0.7, 0.2, 0.1], seed=42) + >>> dataset["a"] + [10, 0, 11, 1, 2] + >>> dataset = interleave_datasets([d1, d2, d3], probabilities=[0.7, 0.2, 0.1], seed=42, stopping_strategy="all_exhausted") + >>> dataset["a"] + [10, 0, 11, 1, 2, 20, 12, 13, ..., 0, 1, 2, 0, 24] + For datasets in streaming mode (iterable): + + >>> from datasets import interleave_datasets + >>> d1 = load_dataset('allenai/c4', 'es', split='train', streaming=True) + >>> d2 = load_dataset('allenai/c4', 'fr', split='train', streaming=True) + >>> dataset = interleave_datasets([d1, d2]) + >>> iterator = iter(dataset) + >>> next(iterator) + {'text': 'Comprar Zapatillas para niña en chancla con goma por...'} + >>> next(iterator) + {'text': 'Le sacre de philippe ier, 23 mai 1059 - Compte Rendu...' + ``` + """ + from .arrow_dataset import Dataset + from .iterable_dataset import IterableDataset + + if not datasets: + raise ValueError("Unable to interleave an empty list of datasets.") + for i, dataset in enumerate(datasets): + if not isinstance(dataset, (Dataset, IterableDataset)): + if isinstance(dataset, (DatasetDict, IterableDatasetDict)): + if not dataset: + raise ValueError( + f"Expected a list of Dataset objects or a list of IterableDataset objects, but element at position {i} " + "is an empty dataset dictionary." + ) + raise ValueError( + f"Dataset at position {i} has at least one split: {list(dataset)}\n" + f"Please pick one to interleave with the other datasets, for example: dataset['{next(iter(dataset))}']" + ) + raise ValueError( + f"Expected a list of Dataset objects or a list of IterableDataset objects, but element at position {i} is a {type(dataset).__name__}." + ) + if i == 0: + dataset_type, other_type = ( + (Dataset, IterableDataset) if isinstance(dataset, Dataset) else (IterableDataset, Dataset) + ) + elif not isinstance(dataset, dataset_type): + raise ValueError( + f"Unable to interleave a {dataset_type.__name__} (at position 0) with a {other_type.__name__} (at position {i}). Expected a list of Dataset objects or a list of IterableDataset objects." + ) + if stopping_strategy not in ["first_exhausted", "all_exhausted"]: + raise ValueError(f"{stopping_strategy} is not supported. Please enter a valid stopping_strategy.") + if dataset_type is Dataset: + return _interleave_map_style_datasets( + datasets, probabilities, seed, info=info, split=split, stopping_strategy=stopping_strategy + ) + else: + return _interleave_iterable_datasets( + datasets, probabilities, seed, info=info, split=split, stopping_strategy=stopping_strategy + ) + + +def concatenate_datasets( + dsets: list[DatasetType], + info: Optional[DatasetInfo] = None, + split: Optional[NamedSplit] = None, + axis: int = 0, +) -> DatasetType: + """ + Converts a list of [`Dataset`] with the same schema into a single [`Dataset`]. + + Args: + dsets (`List[datasets.Dataset]`): + List of Datasets to concatenate. + info (`DatasetInfo`, *optional*): + Dataset information, like description, citation, etc. + split (`NamedSplit`, *optional*): + Name of the dataset split. + axis (`{0, 1}`, defaults to `0`): + Axis to concatenate over, where `0` means over rows (vertically) and `1` means over columns + (horizontally). + + + + Example: + + ```py + >>> ds3 = concatenate_datasets([ds1, ds2]) + ``` + """ + + if not dsets: + raise ValueError("Unable to concatenate an empty list of datasets.") + for i, dataset in enumerate(dsets): + if not isinstance(dataset, (Dataset, IterableDataset)): + if isinstance(dataset, (DatasetDict, IterableDatasetDict)): + if not dataset: + raise ValueError( + f"Expected a list of Dataset objects or a list of IterableDataset objects, but element at position {i} " + "is an empty dataset dictionary." + ) + raise ValueError( + f"Dataset at position {i} has at least one split: {list(dataset)}\n" + f"Please pick one to interleave with the other datasets, for example: dataset['{next(iter(dataset))}']" + ) + raise ValueError( + f"Expected a list of Dataset objects or a list of IterableDataset objects, but element at position {i} is a {type(dataset).__name__}." + ) + if i == 0: + dataset_type, other_type = ( + (Dataset, IterableDataset) if isinstance(dataset, Dataset) else (IterableDataset, Dataset) + ) + elif not isinstance(dataset, dataset_type): + raise ValueError( + f"Unable to interleave a {dataset_type.__name__} (at position 0) with a {other_type.__name__} (at position {i}). Expected a list of Dataset objects or a list of IterableDataset objects." + ) + if dataset_type is Dataset: + return _concatenate_map_style_datasets(dsets, info=info, split=split, axis=axis) + else: + return _concatenate_iterable_datasets(dsets, info=info, split=split, axis=axis) diff --git a/lib/python3.12/site-packages/datasets/commands/__init__.py b/lib/python3.12/site-packages/datasets/commands/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..905e753955a348a8e486302e1b6f5e8f53ec7bf4 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/commands/__init__.py @@ -0,0 +1,13 @@ +from abc import ABC, abstractmethod +from argparse import ArgumentParser + + +class BaseDatasetsCLICommand(ABC): + @staticmethod + @abstractmethod + def register_subcommand(parser: ArgumentParser): + raise NotImplementedError() + + @abstractmethod + def run(self): + raise NotImplementedError() diff --git a/lib/python3.12/site-packages/datasets/commands/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/datasets/commands/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5d8c608a365e13f33ae454bed89685bb4cd481a3 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/commands/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/commands/__pycache__/convert.cpython-312.pyc b/lib/python3.12/site-packages/datasets/commands/__pycache__/convert.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b61542a84140ea193a44c8840989f75c77659af2 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/commands/__pycache__/convert.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/commands/__pycache__/convert_to_parquet.cpython-312.pyc b/lib/python3.12/site-packages/datasets/commands/__pycache__/convert_to_parquet.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e643619aecd0fbf295208e454a1f639bcf6df922 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/commands/__pycache__/convert_to_parquet.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/commands/__pycache__/datasets_cli.cpython-312.pyc b/lib/python3.12/site-packages/datasets/commands/__pycache__/datasets_cli.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8dda745292651db45fe653fdb8596922d0aedaf2 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/commands/__pycache__/datasets_cli.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/commands/__pycache__/delete_from_hub.cpython-312.pyc b/lib/python3.12/site-packages/datasets/commands/__pycache__/delete_from_hub.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..36fcf88f2b5b2f91ca404254f34cfac727833e1c Binary files /dev/null and b/lib/python3.12/site-packages/datasets/commands/__pycache__/delete_from_hub.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/commands/__pycache__/env.cpython-312.pyc b/lib/python3.12/site-packages/datasets/commands/__pycache__/env.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..733edf0f4dd9b282c5115630557980c553717ac6 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/commands/__pycache__/env.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/commands/__pycache__/test.cpython-312.pyc b/lib/python3.12/site-packages/datasets/commands/__pycache__/test.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..065ba655154fe3ee634b513267ce7f95587dda41 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/commands/__pycache__/test.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/commands/convert.py b/lib/python3.12/site-packages/datasets/commands/convert.py new file mode 100644 index 0000000000000000000000000000000000000000..f82547efdee094ab84882e19d219e831b60e17f1 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/commands/convert.py @@ -0,0 +1,195 @@ +import os +import re +import shutil +from argparse import ArgumentParser, Namespace + +from datasets.commands import BaseDatasetsCLICommand +from datasets.utils.logging import get_logger + + +HIGHLIGHT_MESSAGE_PRE = """<<<<<<< This should probably be modified because it mentions: """ + +HIGHLIGHT_MESSAGE_POST = """======= +>>>>>>> +""" + +TO_HIGHLIGHT = [ + "TextEncoderConfig", + "ByteTextEncoder", + "SubwordTextEncoder", + "encoder_config", + "maybe_build_from_corpus", + "manual_dir", +] + +TO_CONVERT = [ + # (pattern, replacement) + # Order is important here for some replacements + (r"tfds\.core", r"datasets"), + (r"tf\.io\.gfile\.GFile", r"open"), + (r"tf\.([\w\d]+)", r"datasets.Value('\1')"), + (r"tfds\.features\.Text\(\)", r"datasets.Value('string')"), + (r"tfds\.features\.Text\(", r"datasets.Value('string'),"), + (r"features\s*=\s*tfds.features.FeaturesDict\(", r"features=datasets.Features("), + (r"tfds\.features\.FeaturesDict\(", r"dict("), + (r"The TensorFlow Datasets Authors", r"The TensorFlow Datasets Authors and the HuggingFace Datasets Authors"), + (r"tfds\.", r"datasets."), + (r"dl_manager\.manual_dir", r"self.config.data_dir"), + (r"self\.builder_config", r"self.config"), +] + + +def convert_command_factory(args: Namespace): + """ + Factory function used to convert a model TF 1.0 checkpoint in a PyTorch checkpoint. + + Returns: ConvertCommand + """ + return ConvertCommand(args.tfds_path, args.datasets_directory) + + +class ConvertCommand(BaseDatasetsCLICommand): + @staticmethod + def register_subcommand(parser: ArgumentParser): + """ + Register this command to argparse so it's available for the datasets-cli + + Args: + parser: Root parser to register command-specific arguments + """ + train_parser = parser.add_parser( + "convert", + help="Convert a TensorFlow Datasets dataset to a HuggingFace Datasets dataset.", + ) + train_parser.add_argument( + "--tfds_path", + type=str, + required=True, + help="Path to a TensorFlow Datasets folder to convert or a single tfds file to convert.", + ) + train_parser.add_argument( + "--datasets_directory", type=str, required=True, help="Path to the HuggingFace Datasets folder." + ) + train_parser.set_defaults(func=convert_command_factory) + + def __init__(self, tfds_path: str, datasets_directory: str, *args): + self._logger = get_logger("datasets-cli/converting") + + self._tfds_path = tfds_path + self._datasets_directory = datasets_directory + + def run(self): + if os.path.isdir(self._tfds_path): + abs_tfds_path = os.path.abspath(self._tfds_path) + elif os.path.isfile(self._tfds_path): + abs_tfds_path = os.path.dirname(self._tfds_path) + else: + raise ValueError("--tfds_path is neither a directory nor a file. Please check path.") + + abs_datasets_path = os.path.abspath(self._datasets_directory) + + self._logger.info(f"Converting datasets from {abs_tfds_path} to {abs_datasets_path}") + + utils_files = [] + with_manual_update = [] + imports_to_builder_map = {} + + if os.path.isdir(self._tfds_path): + file_names = os.listdir(abs_tfds_path) + else: + file_names = [os.path.basename(self._tfds_path)] + + for f_name in file_names: + self._logger.info(f"Looking at file {f_name}") + input_file = os.path.join(abs_tfds_path, f_name) + output_file = os.path.join(abs_datasets_path, f_name) + + if not os.path.isfile(input_file) or "__init__" in f_name or "_test" in f_name or ".py" not in f_name: + self._logger.info("Skipping file") + continue + + with open(input_file, encoding="utf-8") as f: + lines = f.readlines() + + out_lines = [] + is_builder = False + needs_manual_update = False + tfds_imports = [] + for line in lines: + out_line = line + + # Convert imports + if "import tensorflow.compat.v2 as tf" in out_line: + continue + elif "@tfds.core" in out_line: + continue + elif "builder=self" in out_line: + continue + elif "import tensorflow_datasets.public_api as tfds" in out_line: + out_line = "import datasets\n" + elif "import tensorflow" in out_line: + # order is important here + out_line = "" + continue + elif "from absl import logging" in out_line: + out_line = "from datasets import logging\n" + elif "getLogger" in out_line: + out_line = out_line.replace("getLogger", "get_logger") + elif any(expression in out_line for expression in TO_HIGHLIGHT): + needs_manual_update = True + to_remove = list(filter(lambda e: e in out_line, TO_HIGHLIGHT)) + out_lines.append(HIGHLIGHT_MESSAGE_PRE + str(to_remove) + "\n") + out_lines.append(out_line) + out_lines.append(HIGHLIGHT_MESSAGE_POST) + continue + else: + for pattern, replacement in TO_CONVERT: + out_line = re.sub(pattern, replacement, out_line) + + # Take care of saving utilities (to later move them together with main script) + if "tensorflow_datasets" in out_line: + match = re.match(r"from\stensorflow_datasets.*import\s([^\.\r\n]+)", out_line) + tfds_imports.extend(imp.strip() for imp in match.group(1).split(",")) + out_line = "from . import " + match.group(1) + + # Check we have not forget anything + if "tf." in out_line or "tfds." in out_line or "tensorflow_datasets" in out_line: + raise ValueError(f"Error converting {out_line.strip()}") + + if "GeneratorBasedBuilder" in out_line: + is_builder = True + out_lines.append(out_line) + + if is_builder or "wmt" in f_name: + # We create a new directory for each dataset + dir_name = f_name.replace(".py", "") + output_dir = os.path.join(abs_datasets_path, dir_name) + output_file = os.path.join(output_dir, f_name) + os.makedirs(output_dir, exist_ok=True) + self._logger.info(f"Adding directory {output_dir}") + imports_to_builder_map.update(dict.fromkeys(tfds_imports, output_dir)) + else: + # Utilities will be moved at the end + utils_files.append(output_file) + + if needs_manual_update: + with_manual_update.append(output_file) + + with open(output_file, "w", encoding="utf-8") as f: + f.writelines(out_lines) + self._logger.info(f"Converted in {output_file}") + + for utils_file in utils_files: + try: + f_name = os.path.basename(utils_file) + dest_folder = imports_to_builder_map[f_name.replace(".py", "")] + self._logger.info(f"Moving {dest_folder} to {utils_file}") + shutil.copy(utils_file, dest_folder) + except KeyError: + self._logger.error(f"Cannot find destination folder for {utils_file}. Please copy manually.") + + if with_manual_update: + for file_path in with_manual_update: + self._logger.warning( + f"You need to manually update file {file_path} to remove configurations using 'TextEncoderConfig'." + ) diff --git a/lib/python3.12/site-packages/datasets/commands/convert_to_parquet.py b/lib/python3.12/site-packages/datasets/commands/convert_to_parquet.py new file mode 100644 index 0000000000000000000000000000000000000000..43e82de802bf3687678ddee537edf662d7639bb7 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/commands/convert_to_parquet.py @@ -0,0 +1,46 @@ +from argparse import ArgumentParser +from typing import Optional + +from datasets.commands import BaseDatasetsCLICommand +from datasets.hub import convert_to_parquet + + +def _command_factory(args): + return ConvertToParquetCommand( + args.dataset_id, + args.token, + args.revision, + args.trust_remote_code, + ) + + +class ConvertToParquetCommand(BaseDatasetsCLICommand): + @staticmethod + def register_subcommand(parser): + parser: ArgumentParser = parser.add_parser("convert_to_parquet", help="Convert dataset to Parquet") + parser.add_argument( + "dataset_id", help="source dataset ID, e.g. USERNAME/DATASET_NAME or ORGANIZATION/DATASET_NAME" + ) + parser.add_argument("--token", help="access token to the Hugging Face Hub (defaults to logged-in user's one)") + parser.add_argument("--revision", help="source revision") + parser.add_argument( + "--trust_remote_code", action="store_true", help="whether to trust the code execution of the load script" + ) + parser.set_defaults(func=_command_factory) + + def __init__( + self, + dataset_id: str, + token: Optional[str], + revision: Optional[str], + trust_remote_code: bool, + ): + self._dataset_id = dataset_id + self._token = token + self._revision = revision + self._trust_remote_code = trust_remote_code + + def run(self) -> None: + _ = convert_to_parquet( + self._dataset_id, revision=self._revision, token=self._token, trust_remote_code=self._trust_remote_code + ) diff --git a/lib/python3.12/site-packages/datasets/commands/datasets_cli.py b/lib/python3.12/site-packages/datasets/commands/datasets_cli.py new file mode 100644 index 0000000000000000000000000000000000000000..c8f02d9633ac4fbb6d98f90e7a53bedf45f006e3 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/commands/datasets_cli.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python +from argparse import ArgumentParser + +from datasets.commands.convert import ConvertCommand +from datasets.commands.convert_to_parquet import ConvertToParquetCommand +from datasets.commands.delete_from_hub import DeleteFromHubCommand +from datasets.commands.env import EnvironmentCommand +from datasets.commands.test import TestCommand +from datasets.utils.logging import set_verbosity_info + + +def parse_unknown_args(unknown_args): + return {key.lstrip("-"): value for key, value in zip(unknown_args[::2], unknown_args[1::2])} + + +def main(): + parser = ArgumentParser( + "HuggingFace Datasets CLI tool", usage="datasets-cli []", allow_abbrev=False + ) + commands_parser = parser.add_subparsers(help="datasets-cli command helpers") + set_verbosity_info() + + # Register commands + ConvertCommand.register_subcommand(commands_parser) + EnvironmentCommand.register_subcommand(commands_parser) + TestCommand.register_subcommand(commands_parser) + ConvertToParquetCommand.register_subcommand(commands_parser) + DeleteFromHubCommand.register_subcommand(commands_parser) + + # Parse args + args, unknown_args = parser.parse_known_args() + if not hasattr(args, "func"): + parser.print_help() + exit(1) + kwargs = parse_unknown_args(unknown_args) + + # Run + service = args.func(args, **kwargs) + service.run() + + +if __name__ == "__main__": + main() diff --git a/lib/python3.12/site-packages/datasets/commands/delete_from_hub.py b/lib/python3.12/site-packages/datasets/commands/delete_from_hub.py new file mode 100644 index 0000000000000000000000000000000000000000..126a029c246ab584df47c4ba1b908a798b9003b6 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/commands/delete_from_hub.py @@ -0,0 +1,42 @@ +from argparse import ArgumentParser +from typing import Optional + +from datasets.commands import BaseDatasetsCLICommand +from datasets.hub import delete_from_hub + + +def _command_factory(args): + return DeleteFromHubCommand( + args.dataset_id, + args.config_name, + args.token, + args.revision, + ) + + +class DeleteFromHubCommand(BaseDatasetsCLICommand): + @staticmethod + def register_subcommand(parser): + parser: ArgumentParser = parser.add_parser("delete_from_hub", help="Delete dataset config from the Hub") + parser.add_argument( + "dataset_id", help="source dataset ID, e.g. USERNAME/DATASET_NAME or ORGANIZATION/DATASET_NAME" + ) + parser.add_argument("config_name", help="config name to delete") + parser.add_argument("--token", help="access token to the Hugging Face Hub") + parser.add_argument("--revision", help="source revision") + parser.set_defaults(func=_command_factory) + + def __init__( + self, + dataset_id: str, + config_name: str, + token: Optional[str], + revision: Optional[str], + ): + self._dataset_id = dataset_id + self._config_name = config_name + self._token = token + self._revision = revision + + def run(self) -> None: + _ = delete_from_hub(self._dataset_id, self._config_name, revision=self._revision, token=self._token) diff --git a/lib/python3.12/site-packages/datasets/commands/env.py b/lib/python3.12/site-packages/datasets/commands/env.py new file mode 100644 index 0000000000000000000000000000000000000000..40b2a3654c8f1c6c080222a80b12d108972a5dc9 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/commands/env.py @@ -0,0 +1,41 @@ +import platform +from argparse import ArgumentParser + +import fsspec +import huggingface_hub +import pandas +import pyarrow + +from datasets import __version__ as version +from datasets.commands import BaseDatasetsCLICommand + + +def info_command_factory(_): + return EnvironmentCommand() + + +class EnvironmentCommand(BaseDatasetsCLICommand): + @staticmethod + def register_subcommand(parser: ArgumentParser): + download_parser = parser.add_parser("env", help="Print relevant system environment info.") + download_parser.set_defaults(func=info_command_factory) + + def run(self): + info = { + "`datasets` version": version, + "Platform": platform.platform(), + "Python version": platform.python_version(), + "`huggingface_hub` version": huggingface_hub.__version__, + "PyArrow version": pyarrow.__version__, + "Pandas version": pandas.__version__, + "`fsspec` version": fsspec.__version__, + } + + print("\nCopy-and-paste the text below in your GitHub issue.\n") + print(self.format_dict(info)) + + return info + + @staticmethod + def format_dict(d): + return "\n".join([f"- {prop}: {val}" for prop, val in d.items()]) + "\n" diff --git a/lib/python3.12/site-packages/datasets/commands/test.py b/lib/python3.12/site-packages/datasets/commands/test.py new file mode 100644 index 0000000000000000000000000000000000000000..3789873708e74c6202f24cb566bb9e9eaea7ceed --- /dev/null +++ b/lib/python3.12/site-packages/datasets/commands/test.py @@ -0,0 +1,207 @@ +import logging +import os +from argparse import ArgumentParser +from collections.abc import Generator +from pathlib import Path +from shutil import copyfile, rmtree +from typing import Optional + +import datasets.config +from datasets.builder import DatasetBuilder +from datasets.commands import BaseDatasetsCLICommand +from datasets.download.download_manager import DownloadMode +from datasets.load import dataset_module_factory, import_main_class +from datasets.utils.info_utils import VerificationMode +from datasets.utils.logging import ERROR, get_logger + + +logger = get_logger(__name__) + + +def _test_command_factory(args): + return TestCommand( + args.dataset, + args.name, + args.cache_dir, + args.data_dir, + args.all_configs, + args.save_info or args.save_infos, + args.ignore_verifications, + args.force_redownload, + args.clear_cache, + args.num_proc, + args.trust_remote_code, + ) + + +class TestCommand(BaseDatasetsCLICommand): + __test__ = False # to tell pytest it's not a test class + + @staticmethod + def register_subcommand(parser: ArgumentParser): + test_parser = parser.add_parser("test", help="Test dataset implementation.") + test_parser.add_argument("--name", type=str, default=None, help="Dataset processing name") + test_parser.add_argument( + "--cache_dir", + type=str, + default=None, + help="Cache directory where the datasets are stored.", + ) + test_parser.add_argument( + "--data_dir", + type=str, + default=None, + help="Can be used to specify a manual directory to get the files from.", + ) + test_parser.add_argument("--all_configs", action="store_true", help="Test all dataset configurations") + test_parser.add_argument( + "--save_info", action="store_true", help="Save the dataset infos in the dataset card (README.md)" + ) + test_parser.add_argument( + "--ignore_verifications", + action="store_true", + help="Run the test without checksums and splits checks.", + ) + test_parser.add_argument("--force_redownload", action="store_true", help="Force dataset redownload") + test_parser.add_argument( + "--clear_cache", + action="store_true", + help="Remove downloaded files and cached datasets after each config test", + ) + test_parser.add_argument("--num_proc", type=int, default=None, help="Number of processes") + test_parser.add_argument( + "--trust_remote_code", action="store_true", help="whether to trust the code execution of the load script" + ) + # aliases + test_parser.add_argument("--save_infos", action="store_true", help="alias to save_info") + test_parser.add_argument("dataset", type=str, help="Name of the dataset to download") + test_parser.set_defaults(func=_test_command_factory) + + def __init__( + self, + dataset: str, + name: str, + cache_dir: str, + data_dir: str, + all_configs: bool, + save_infos: bool, + ignore_verifications: bool, + force_redownload: bool, + clear_cache: bool, + num_proc: int, + trust_remote_code: Optional[bool], + ): + self._dataset = dataset + self._name = name + self._cache_dir = cache_dir + self._data_dir = data_dir + self._all_configs = all_configs + self._save_infos = save_infos + self._ignore_verifications = ignore_verifications + self._force_redownload = force_redownload + self._clear_cache = clear_cache + self._num_proc = num_proc + self._trust_remote_code = trust_remote_code + if clear_cache and not cache_dir: + print( + "When --clear_cache is used, specifying a cache directory is mandatory.\n" + "The 'download' folder of the cache directory and the dataset builder cache will be deleted after each configuration test.\n" + "Please provide a --cache_dir that will be used to test the dataset script." + ) + exit(1) + if save_infos: + self._ignore_verifications = True + + def run(self): + logging.getLogger("filelock").setLevel(ERROR) + if self._name is not None and self._all_configs: + print("Both parameters `config` and `all_configs` can't be used at once.") + exit(1) + path, config_name = self._dataset, self._name + module = dataset_module_factory(path, trust_remote_code=self._trust_remote_code) + builder_cls = import_main_class(module.module_path) + n_builders = len(builder_cls.BUILDER_CONFIGS) if self._all_configs and builder_cls.BUILDER_CONFIGS else 1 + + def get_builders() -> Generator[DatasetBuilder, None, None]: + if self._all_configs and builder_cls.BUILDER_CONFIGS: + for i, config in enumerate(builder_cls.BUILDER_CONFIGS): + if "config_name" in module.builder_kwargs: + yield builder_cls( + cache_dir=self._cache_dir, + data_dir=self._data_dir, + **module.builder_kwargs, + ) + else: + yield builder_cls( + config_name=config.name, + cache_dir=self._cache_dir, + data_dir=self._data_dir, + **module.builder_kwargs, + ) + else: + if "config_name" in module.builder_kwargs: + yield builder_cls(cache_dir=self._cache_dir, data_dir=self._data_dir, **module.builder_kwargs) + else: + yield builder_cls( + config_name=config_name, + cache_dir=self._cache_dir, + data_dir=self._data_dir, + **module.builder_kwargs, + ) + + for j, builder in enumerate(get_builders()): + print(f"Testing builder '{builder.config.name}' ({j + 1}/{n_builders})") + builder._record_infos = os.path.exists( + os.path.join(builder.get_imported_module_dir(), datasets.config.DATASETDICT_INFOS_FILENAME) + ) # record checksums only if we need to update a (deprecated) dataset_infos.json + builder.download_and_prepare( + download_mode=DownloadMode.REUSE_CACHE_IF_EXISTS + if not self._force_redownload + else DownloadMode.FORCE_REDOWNLOAD, + verification_mode=VerificationMode.NO_CHECKS + if self._ignore_verifications + else VerificationMode.ALL_CHECKS, + num_proc=self._num_proc, + ) + builder.as_dataset() + if self._save_infos: + builder._save_infos() + + # If save_infos=True, the dataset card (README.md) is created next to the loaded module file. + # The dataset_infos are saved in the YAML part of the README.md + + # Let's move it to the original directory of the dataset script, to allow the user to + # upload them on S3 at the same time afterwards. + if self._save_infos: + dataset_readme_path = os.path.join( + builder_cls.get_imported_module_dir(), datasets.config.REPOCARD_FILENAME + ) + name = Path(path).name + ".py" + combined_path = os.path.join(path, name) + if os.path.isfile(path): + dataset_dir = os.path.dirname(path) + elif os.path.isfile(combined_path): + dataset_dir = path + elif os.path.isdir(path): # for local directories containing only data files + dataset_dir = path + else: # in case of a remote dataset + dataset_dir = None + print(f"Dataset card saved at {dataset_readme_path}") + + # Move dataset_info back to the user + if dataset_dir is not None: + user_dataset_readme_path = os.path.join(dataset_dir, datasets.config.REPOCARD_FILENAME) + copyfile(dataset_readme_path, user_dataset_readme_path) + print(f"Dataset card saved at {user_dataset_readme_path}") + + # If clear_cache=True, the download folder and the dataset builder cache directory are deleted + if self._clear_cache: + if os.path.isdir(builder._cache_dir): + logger.warning(f"Clearing cache at {builder._cache_dir}") + rmtree(builder._cache_dir) + download_dir = os.path.join(self._cache_dir, datasets.config.DOWNLOADED_DATASETS_DIR) + if os.path.isdir(download_dir): + logger.warning(f"Clearing cache at {download_dir}") + rmtree(download_dir) + + print("Test successful.") diff --git a/lib/python3.12/site-packages/datasets/config.py b/lib/python3.12/site-packages/datasets/config.py new file mode 100644 index 0000000000000000000000000000000000000000..326db94da68758589f15cc2efe89db273801c531 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/config.py @@ -0,0 +1,269 @@ +import importlib +import importlib.metadata +import logging +import os +import platform +from pathlib import Path +from typing import Optional + +from huggingface_hub import constants +from packaging import version + + +logger = logging.getLogger(__name__.split(".", 1)[0]) # to avoid circular import from .utils.logging + +# Datasets +S3_DATASETS_BUCKET_PREFIX = "https://s3.amazonaws.com/datasets.huggingface.co/datasets/datasets" +CLOUDFRONT_DATASETS_DISTRIB_PREFIX = "https://cdn-datasets.huggingface.co/datasets/datasets" +REPO_DATASETS_URL = "https://raw.githubusercontent.com/huggingface/datasets/{revision}/datasets/{path}/{name}" + +# Hub +HF_ENDPOINT = os.environ.get("HF_ENDPOINT", "https://huggingface.co") +HUB_DATASETS_URL = HF_ENDPOINT + "/datasets/{repo_id}/resolve/{revision}/{path}" +HUB_DATASETS_HFFS_URL = "hf://datasets/{repo_id}@{revision}/{path}" +HUB_DEFAULT_VERSION = "main" + +PY_VERSION = version.parse(platform.python_version()) + +# General environment variables accepted values for booleans +ENV_VARS_TRUE_VALUES = {"1", "ON", "YES", "TRUE"} +ENV_VARS_FALSE_VALUES = {"0", "OFF", "NO", "FALSE"} +ENV_VARS_TRUE_AND_AUTO_VALUES = ENV_VARS_TRUE_VALUES.union({"AUTO"}) +ENV_VARS_FALSE_AND_AUTO_VALUES = ENV_VARS_FALSE_VALUES.union({"AUTO"}) + + +# Imports +DILL_VERSION = version.parse(importlib.metadata.version("dill")) +FSSPEC_VERSION = version.parse(importlib.metadata.version("fsspec")) +PANDAS_VERSION = version.parse(importlib.metadata.version("pandas")) +PYARROW_VERSION = version.parse(importlib.metadata.version("pyarrow")) +HF_HUB_VERSION = version.parse(importlib.metadata.version("huggingface_hub")) + +USE_TF = os.environ.get("USE_TF", "AUTO").upper() +USE_TORCH = os.environ.get("USE_TORCH", "AUTO").upper() +USE_JAX = os.environ.get("USE_JAX", "AUTO").upper() + +TORCH_VERSION = "N/A" +TORCH_AVAILABLE = False + +if USE_TORCH in ENV_VARS_TRUE_AND_AUTO_VALUES and USE_TF not in ENV_VARS_TRUE_VALUES: + TORCH_AVAILABLE = importlib.util.find_spec("torch") is not None + if TORCH_AVAILABLE: + try: + TORCH_VERSION = version.parse(importlib.metadata.version("torch")) + logger.info(f"PyTorch version {TORCH_VERSION} available.") + except importlib.metadata.PackageNotFoundError: + pass +else: + logger.info("Disabling PyTorch because USE_TF is set") + +POLARS_VERSION = "N/A" +POLARS_AVAILABLE = importlib.util.find_spec("polars") is not None + +if POLARS_AVAILABLE: + try: + POLARS_VERSION = version.parse(importlib.metadata.version("polars")) + logger.info(f"Polars version {POLARS_VERSION} available.") + except importlib.metadata.PackageNotFoundError: + pass + + +DUCKDB_VERSION = "N/A" +DUCKDB_AVAILABLE = importlib.util.find_spec("duckdb") is not None + +if DUCKDB_AVAILABLE: + try: + DUCKDB_VERSION = version.parse(importlib.metadata.version("duckdb")) + logger.info(f"Duckdb version {DUCKDB_VERSION} available.") + except importlib.metadata.PackageNotFoundError: + pass + +TF_VERSION = "N/A" +TF_AVAILABLE = False + +if USE_TF in ENV_VARS_TRUE_AND_AUTO_VALUES and USE_TORCH not in ENV_VARS_TRUE_VALUES: + TF_AVAILABLE = importlib.util.find_spec("tensorflow") is not None + if TF_AVAILABLE: + # For the metadata, we have to look for both tensorflow and tensorflow-cpu + for package in [ + "tensorflow", + "tensorflow-cpu", + "tensorflow-gpu", + "tf-nightly", + "tf-nightly-cpu", + "tf-nightly-gpu", + "intel-tensorflow", + "tensorflow-rocm", + "tensorflow-macos", + ]: + try: + TF_VERSION = version.parse(importlib.metadata.version(package)) + except importlib.metadata.PackageNotFoundError: + continue + else: + break + else: + TF_AVAILABLE = False + if TF_AVAILABLE: + if TF_VERSION.major < 2: + logger.info(f"TensorFlow found but with version {TF_VERSION}. `datasets` requires version 2 minimum.") + TF_AVAILABLE = False + else: + logger.info(f"TensorFlow version {TF_VERSION} available.") +else: + logger.info("Disabling Tensorflow because USE_TORCH is set") + + +JAX_VERSION = "N/A" +JAX_AVAILABLE = False + +if USE_JAX in ENV_VARS_TRUE_AND_AUTO_VALUES: + JAX_AVAILABLE = importlib.util.find_spec("jax") is not None and importlib.util.find_spec("jaxlib") is not None + if JAX_AVAILABLE: + try: + JAX_VERSION = version.parse(importlib.metadata.version("jax")) + logger.info(f"JAX version {JAX_VERSION} available.") + except importlib.metadata.PackageNotFoundError: + pass +else: + logger.info("Disabling JAX because USE_JAX is set to False") + + +# Optional tools for data loading +SQLALCHEMY_AVAILABLE = importlib.util.find_spec("sqlalchemy") is not None + +# Optional tools for feature decoding +PIL_AVAILABLE = importlib.util.find_spec("PIL") is not None +IS_OPUS_SUPPORTED = importlib.util.find_spec("soundfile") is not None and version.parse( + importlib.import_module("soundfile").__libsndfile_version__ +) >= version.parse("1.0.31") +IS_MP3_SUPPORTED = importlib.util.find_spec("soundfile") is not None and version.parse( + importlib.import_module("soundfile").__libsndfile_version__ +) >= version.parse("1.1.0") +TORCHVISION_AVAILABLE = importlib.util.find_spec("torchvision") is not None +PDFPLUMBER_AVAILABLE = importlib.util.find_spec("pdfplumber") is not None + +# Optional compression tools +RARFILE_AVAILABLE = importlib.util.find_spec("rarfile") is not None +ZSTANDARD_AVAILABLE = importlib.util.find_spec("zstandard") is not None +LZ4_AVAILABLE = importlib.util.find_spec("lz4") is not None +PY7ZR_AVAILABLE = importlib.util.find_spec("py7zr") is not None + +# Cache location +DEFAULT_XDG_CACHE_HOME = "~/.cache" +XDG_CACHE_HOME = os.getenv("XDG_CACHE_HOME", DEFAULT_XDG_CACHE_HOME) +DEFAULT_HF_CACHE_HOME = os.path.join(XDG_CACHE_HOME, "huggingface") +HF_CACHE_HOME = os.path.expanduser(os.getenv("HF_HOME", DEFAULT_HF_CACHE_HOME)) + +DEFAULT_HF_DATASETS_CACHE = os.path.join(HF_CACHE_HOME, "datasets") +HF_DATASETS_CACHE = Path(os.getenv("HF_DATASETS_CACHE", DEFAULT_HF_DATASETS_CACHE)) + +DEFAULT_HF_MODULES_CACHE = os.path.join(HF_CACHE_HOME, "modules") +HF_MODULES_CACHE = Path(os.getenv("HF_MODULES_CACHE", DEFAULT_HF_MODULES_CACHE)) + +DOWNLOADED_DATASETS_DIR = "downloads" +DEFAULT_DOWNLOADED_DATASETS_PATH = os.path.join(HF_DATASETS_CACHE, DOWNLOADED_DATASETS_DIR) +DOWNLOADED_DATASETS_PATH = Path(os.getenv("HF_DATASETS_DOWNLOADED_DATASETS_PATH", DEFAULT_DOWNLOADED_DATASETS_PATH)) + +EXTRACTED_DATASETS_DIR = "extracted" +DEFAULT_EXTRACTED_DATASETS_PATH = os.path.join(DEFAULT_DOWNLOADED_DATASETS_PATH, EXTRACTED_DATASETS_DIR) +EXTRACTED_DATASETS_PATH = Path(os.getenv("HF_DATASETS_EXTRACTED_DATASETS_PATH", DEFAULT_EXTRACTED_DATASETS_PATH)) + +# Download count for the website +HF_UPDATE_DOWNLOAD_COUNTS = ( + os.environ.get("HF_UPDATE_DOWNLOAD_COUNTS", "AUTO").upper() in ENV_VARS_TRUE_AND_AUTO_VALUES +) + +# For downloads and to check remote files metadata +HF_DATASETS_MULTITHREADING_MAX_WORKERS = 16 + +# Remote dataset scripts support +__HF_DATASETS_TRUST_REMOTE_CODE = os.environ.get("HF_DATASETS_TRUST_REMOTE_CODE", "ask") +HF_DATASETS_TRUST_REMOTE_CODE: Optional[bool] = ( + True + if __HF_DATASETS_TRUST_REMOTE_CODE.upper() in ENV_VARS_TRUE_VALUES + else False + if __HF_DATASETS_TRUST_REMOTE_CODE.upper() in ENV_VARS_FALSE_VALUES + else None +) +TIME_OUT_REMOTE_CODE = 15 + +# Dataset viewer API +USE_PARQUET_EXPORT = True + +# Batch size constants. For more info, see: +# https://github.com/apache/arrow/blob/master/docs/source/cpp/arrays.rst#size-limitations-and-recommendations) +DEFAULT_MAX_BATCH_SIZE = 1000 + +# Size of the preloaded record batch in `Dataset.__iter__` +ARROW_READER_BATCH_SIZE_IN_DATASET_ITER = 10 + +# Max shard size in bytes (e.g. to shard parquet datasets in push_to_hub or download_and_prepare) +MAX_SHARD_SIZE = "500MB" + +# Parquet configuration +PARQUET_ROW_GROUP_SIZE_FOR_AUDIO_DATASETS = 100 +PARQUET_ROW_GROUP_SIZE_FOR_IMAGE_DATASETS = 100 +PARQUET_ROW_GROUP_SIZE_FOR_BINARY_DATASETS = 100 +PARQUET_ROW_GROUP_SIZE_FOR_VIDEO_DATASETS = 10 + +# Offline mode +_offline = os.environ.get("HF_DATASETS_OFFLINE") +HF_HUB_OFFLINE = constants.HF_HUB_OFFLINE if _offline is None else _offline.upper() in ENV_VARS_TRUE_VALUES +HF_DATASETS_OFFLINE = HF_HUB_OFFLINE # kept for backward-compatibility + +# Here, `True` will disable progress bars globally without possibility of enabling it +# programmatically. `False` will enable them without possibility of disabling them. +# If environment variable is not set (None), then the user is free to enable/disable +# them programmatically. +# TL;DR: env variable has priority over code +__HF_DATASETS_DISABLE_PROGRESS_BARS = os.environ.get("HF_DATASETS_DISABLE_PROGRESS_BARS") +HF_DATASETS_DISABLE_PROGRESS_BARS: Optional[bool] = ( + __HF_DATASETS_DISABLE_PROGRESS_BARS.upper() in ENV_VARS_TRUE_VALUES + if __HF_DATASETS_DISABLE_PROGRESS_BARS is not None + else None +) + +# In-memory +DEFAULT_IN_MEMORY_MAX_SIZE = 0 # Disabled +IN_MEMORY_MAX_SIZE = float(os.environ.get("HF_DATASETS_IN_MEMORY_MAX_SIZE", DEFAULT_IN_MEMORY_MAX_SIZE)) + +# File names +DATASET_ARROW_FILENAME = "dataset.arrow" +DATASET_INDICES_FILENAME = "indices.arrow" +DATASET_STATE_JSON_FILENAME = "state.json" +DATASET_INFO_FILENAME = "dataset_info.json" +DATASETDICT_INFOS_FILENAME = "dataset_infos.json" +LICENSE_FILENAME = "LICENSE" +DATASETDICT_JSON_FILENAME = "dataset_dict.json" +METADATA_CONFIGS_FIELD = "configs" +REPOCARD_FILENAME = "README.md" +REPOYAML_FILENAME = ".huggingface.yaml" + +MODULE_NAME_FOR_DYNAMIC_MODULES = "datasets_modules" + +MAX_DATASET_CONFIG_ID_READABLE_LENGTH = 255 + +# Temporary cache directory prefix +TEMP_CACHE_DIR_PREFIX = "hf_datasets-" + +# Streaming +STREAMING_READ_MAX_RETRIES = 20 +STREAMING_READ_RETRY_INTERVAL = 5 + +# Datasets without script +DATA_FILES_MAX_NUMBER_FOR_MODULE_INFERENCE = 200 +GLOBBED_DATA_FILES_MAX_NUMBER_FOR_MODULE_INFERENCE = 10 +ARCHIVED_DATA_FILES_MAX_NUMBER_FOR_MODULE_INFERENCE = 200 + +# Async map functions +MAX_NUM_RUNNING_ASYNC_MAP_FUNCTIONS_IN_PARALLEL = 1000 + +# Progress bars +PBAR_REFRESH_TIME_INTERVAL = 0.05 # 20 progress updates per sec + +# Maximum number of uploaded files per commit +UPLOADS_MAX_NUMBER_PER_COMMIT = 50 + +# Backward compatibility +MAX_TABLE_NBYTES_FOR_PICKLING = 4 << 30 diff --git a/lib/python3.12/site-packages/datasets/data_files.py b/lib/python3.12/site-packages/datasets/data_files.py new file mode 100644 index 0000000000000000000000000000000000000000..6c35a9a061e4f8d656446f225a347dc435578df8 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/data_files.py @@ -0,0 +1,795 @@ +import os +import re +from functools import partial +from glob import has_magic +from pathlib import Path, PurePath +from typing import Callable, Optional, Union + +import huggingface_hub +from fsspec.core import url_to_fs +from huggingface_hub import HfFileSystem +from packaging import version +from tqdm.contrib.concurrent import thread_map + +from . import config +from .download import DownloadConfig +from .naming import _split_re +from .splits import Split +from .utils import logging +from .utils import tqdm as hf_tqdm +from .utils.file_utils import _prepare_path_and_storage_options, is_local_path, is_relative_path, xbasename, xjoin +from .utils.py_utils import glob_pattern_to_regex, string_to_dict + + +SingleOriginMetadata = Union[tuple[str, str], tuple[str], tuple[()]] + + +SANITIZED_DEFAULT_SPLIT = str(Split.TRAIN) + + +logger = logging.get_logger(__name__) + + +class Url(str): + pass + + +class EmptyDatasetError(FileNotFoundError): + pass + + +SPLIT_PATTERN_SHARDED = "data/{split}-[0-9][0-9][0-9][0-9][0-9]-of-[0-9][0-9][0-9][0-9][0-9]*.*" + +SPLIT_KEYWORDS = { + Split.TRAIN: ["train", "training"], + Split.VALIDATION: ["validation", "valid", "dev", "val"], + Split.TEST: ["test", "testing", "eval", "evaluation"], +} +NON_WORDS_CHARS = "-._ 0-9" +if config.FSSPEC_VERSION < version.parse("2023.9.0"): + KEYWORDS_IN_FILENAME_BASE_PATTERNS = ["**[{sep}/]{keyword}[{sep}]*", "{keyword}[{sep}]*"] + KEYWORDS_IN_DIR_NAME_BASE_PATTERNS = [ + "{keyword}/**", + "{keyword}[{sep}]*/**", + "**[{sep}/]{keyword}/**", + "**[{sep}/]{keyword}[{sep}]*/**", + ] +elif config.FSSPEC_VERSION < version.parse("2023.12.0"): + KEYWORDS_IN_FILENAME_BASE_PATTERNS = ["**/*[{sep}/]{keyword}[{sep}]*", "{keyword}[{sep}]*"] + KEYWORDS_IN_DIR_NAME_BASE_PATTERNS = [ + "{keyword}/**/*", + "{keyword}[{sep}]*/**/*", + "**/*[{sep}/]{keyword}/**/*", + "**/*[{sep}/]{keyword}[{sep}]*/**/*", + ] +else: + KEYWORDS_IN_FILENAME_BASE_PATTERNS = ["**/{keyword}[{sep}]*", "**/*[{sep}]{keyword}[{sep}]*"] + KEYWORDS_IN_DIR_NAME_BASE_PATTERNS = [ + "**/{keyword}/**", + "**/{keyword}[{sep}]*/**", + "**/*[{sep}]{keyword}/**", + "**/*[{sep}]{keyword}[{sep}]*/**", + ] + +DEFAULT_SPLITS = [Split.TRAIN, Split.VALIDATION, Split.TEST] +DEFAULT_PATTERNS_SPLIT_IN_FILENAME = { + split: [ + pattern.format(keyword=keyword, sep=NON_WORDS_CHARS) + for keyword in SPLIT_KEYWORDS[split] + for pattern in KEYWORDS_IN_FILENAME_BASE_PATTERNS + ] + for split in DEFAULT_SPLITS +} +DEFAULT_PATTERNS_SPLIT_IN_DIR_NAME = { + split: [ + pattern.format(keyword=keyword, sep=NON_WORDS_CHARS) + for keyword in SPLIT_KEYWORDS[split] + for pattern in KEYWORDS_IN_DIR_NAME_BASE_PATTERNS + ] + for split in DEFAULT_SPLITS +} + + +DEFAULT_PATTERNS_ALL = { + Split.TRAIN: ["**"], +} + +ALL_SPLIT_PATTERNS = [SPLIT_PATTERN_SHARDED] +ALL_DEFAULT_PATTERNS = [ + DEFAULT_PATTERNS_SPLIT_IN_DIR_NAME, + DEFAULT_PATTERNS_SPLIT_IN_FILENAME, + DEFAULT_PATTERNS_ALL, +] +WILDCARD_CHARACTERS = "*[]" +FILES_TO_IGNORE = [ + "README.md", + "config.json", + "dataset_info.json", + "dataset_infos.json", + "dummy_data.zip", + "dataset_dict.json", +] + + +def contains_wildcards(pattern: str) -> bool: + return any(wildcard_character in pattern for wildcard_character in WILDCARD_CHARACTERS) + + +def sanitize_patterns(patterns: Union[dict, list, str]) -> dict[str, Union[list[str], "DataFilesList"]]: + """ + Take the data_files patterns from the user, and format them into a dictionary. + Each key is the name of the split, and each value is a list of data files patterns (paths or urls). + The default split is "train". + + Returns: + patterns: dictionary of split_name -> list of patterns + """ + if isinstance(patterns, dict): + return {str(key): value if isinstance(value, list) else [value] for key, value in patterns.items()} + elif isinstance(patterns, str): + return {SANITIZED_DEFAULT_SPLIT: [patterns]} + elif isinstance(patterns, list): + if any(isinstance(pattern, dict) for pattern in patterns): + for pattern in patterns: + if not ( + isinstance(pattern, dict) + and len(pattern) == 2 + and "split" in pattern + and isinstance(pattern.get("path"), (str, list)) + ): + raise ValueError( + f"Expected each split to have a 'path' key which can be a string or a list of strings, but got {pattern}" + ) + splits = [pattern["split"] for pattern in patterns] + if len(set(splits)) != len(splits): + raise ValueError(f"Some splits are duplicated in data_files: {splits}") + return { + str(pattern["split"]): pattern["path"] if isinstance(pattern["path"], list) else [pattern["path"]] + for pattern in patterns + } + else: + return {SANITIZED_DEFAULT_SPLIT: patterns} + else: + return sanitize_patterns(list(patterns)) + + +def _is_inside_unrequested_special_dir(matched_rel_path: str, pattern: str) -> bool: + """ + When a path matches a pattern, we additionally check if it's inside a special directory + we ignore by default (if it starts with a double underscore). + + Users can still explicitly request a filepath inside such a directory if "__pycache__" is + mentioned explicitly in the requested pattern. + + Some examples: + + base directory: + + ./ + └── __pycache__ + └── b.txt + + >>> _is_inside_unrequested_special_dir("__pycache__/b.txt", "**") + True + >>> _is_inside_unrequested_special_dir("__pycache__/b.txt", "*/b.txt") + True + >>> _is_inside_unrequested_special_dir("__pycache__/b.txt", "__pycache__/*") + False + >>> _is_inside_unrequested_special_dir("__pycache__/b.txt", "__*/*") + False + """ + # We just need to check if every special directories from the path is present explicitly in the pattern. + # Since we assume that the path matches the pattern, it's equivalent to counting that both + # the parent path and the parent pattern have the same number of special directories. + data_dirs_to_ignore_in_path = [part for part in PurePath(matched_rel_path).parent.parts if part.startswith("__")] + data_dirs_to_ignore_in_pattern = [part for part in PurePath(pattern).parent.parts if part.startswith("__")] + return len(data_dirs_to_ignore_in_path) != len(data_dirs_to_ignore_in_pattern) + + +def _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(matched_rel_path: str, pattern: str) -> bool: + """ + When a path matches a pattern, we additionally check if it's a hidden file or if it's inside + a hidden directory we ignore by default, i.e. if the file name or a parent directory name starts with a dot. + + Users can still explicitly request a filepath that is hidden or is inside a hidden directory + if the hidden part is mentioned explicitly in the requested pattern. + + Some examples: + + base directory: + + ./ + └── .hidden_file.txt + + >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_file.txt", "**") + True + >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_file.txt", ".*") + False + + base directory: + + ./ + └── .hidden_dir + └── a.txt + + >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/a.txt", "**") + True + >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/a.txt", ".*/*") + False + >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/a.txt", ".hidden_dir/*") + False + + base directory: + + ./ + └── .hidden_dir + └── .hidden_file.txt + + >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/.hidden_file.txt", "**") + True + >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/.hidden_file.txt", ".*/*") + True + >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/.hidden_file.txt", ".*/.*") + False + >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/.hidden_file.txt", ".hidden_dir/*") + True + >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/.hidden_file.txt", ".hidden_dir/.*") + False + """ + # We just need to check if every hidden part from the path is present explicitly in the pattern. + # Since we assume that the path matches the pattern, it's equivalent to counting that both + # the path and the pattern have the same number of hidden parts. + hidden_directories_in_path = [ + part for part in PurePath(matched_rel_path).parts if part.startswith(".") and not set(part) == {"."} + ] + hidden_directories_in_pattern = [ + part for part in PurePath(pattern).parts if part.startswith(".") and not set(part) == {"."} + ] + return len(hidden_directories_in_path) != len(hidden_directories_in_pattern) + + +def _get_data_files_patterns(pattern_resolver: Callable[[str], list[str]]) -> dict[str, list[str]]: + """ + Get the default pattern from a directory or repository by testing all the supported patterns. + The first patterns to return a non-empty list of data files is returned. + + In order, it first tests if SPLIT_PATTERN_SHARDED works, otherwise it tests the patterns in ALL_DEFAULT_PATTERNS. + """ + # first check the split patterns like data/{split}-00000-of-00001.parquet + for split_pattern in ALL_SPLIT_PATTERNS: + pattern = split_pattern.replace("{split}", "*") + try: + data_files = pattern_resolver(pattern) + except FileNotFoundError: + continue + if len(data_files) > 0: + splits: set[str] = set() + for p in data_files: + p_parts = string_to_dict(xbasename(p), glob_pattern_to_regex(xbasename(split_pattern))) + assert p_parts is not None + splits.add(p_parts["split"]) + + if any(not re.match(_split_re, split) for split in splits): + raise ValueError(f"Split name should match '{_split_re}'' but got '{splits}'.") + sorted_splits = [str(split) for split in DEFAULT_SPLITS if split in splits] + sorted( + splits - {str(split) for split in DEFAULT_SPLITS} + ) + return {split: [split_pattern.format(split=split)] for split in sorted_splits} + # then check the default patterns based on train/valid/test splits + for patterns_dict in ALL_DEFAULT_PATTERNS: + non_empty_splits = [] + for split, patterns in patterns_dict.items(): + for pattern in patterns: + try: + data_files = pattern_resolver(pattern) + except FileNotFoundError: + continue + if len(data_files) > 0: + non_empty_splits.append(split) + break + if non_empty_splits: + return {split: patterns_dict[split] for split in non_empty_splits} + raise FileNotFoundError(f"Couldn't resolve pattern {pattern} with resolver {pattern_resolver}") + + +def resolve_pattern( + pattern: str, + base_path: str, + allowed_extensions: Optional[list[str]] = None, + download_config: Optional[DownloadConfig] = None, +) -> list[str]: + """ + Resolve the paths and URLs of the data files from the pattern passed by the user. + + You can use patterns to resolve multiple local files. Here are a few examples: + - *.csv to match all the CSV files at the first level + - **.csv to match all the CSV files at any level + - data/* to match all the files inside "data" + - data/** to match all the files inside "data" and its subdirectories + + The patterns are resolved using the fsspec glob. In fsspec>=2023.12.0 this is equivalent to + Python's glob.glob, Path.glob, Path.match and fnmatch where ** is unsupported with a prefix/suffix + other than a forward slash /. + + More generally: + - '*' matches any character except a forward-slash (to match just the file or directory name) + - '**' matches any character including a forward-slash / + + Hidden files and directories (i.e. whose names start with a dot) are ignored, unless they are explicitly requested. + The same applies to special directories that start with a double underscore like "__pycache__". + You can still include one if the pattern explicitly mentions it: + - to include a hidden file: "*/.hidden.txt" or "*/.*" + - to include a hidden directory: ".hidden/*" or ".*/*" + - to include a special directory: "__special__/*" or "__*/*" + + Example:: + + >>> from datasets.data_files import resolve_pattern + >>> base_path = "." + >>> resolve_pattern("docs/**/*.py", base_path) + [/Users/mariosasko/Desktop/projects/datasets/docs/source/_config.py'] + + Args: + pattern (str): Unix pattern or paths or URLs of the data files to resolve. + The paths can be absolute or relative to base_path. + Remote filesystems using fsspec are supported, e.g. with the hf:// protocol. + base_path (str): Base path to use when resolving relative paths. + allowed_extensions (Optional[list], optional): White-list of file extensions to use. Defaults to None (all extensions). + For example: allowed_extensions=[".csv", ".json", ".txt", ".parquet"] + download_config ([`DownloadConfig`], *optional*): Specific download configuration parameters. + Returns: + List[str]: List of paths or URLs to the local or remote files that match the patterns. + """ + if is_relative_path(pattern): + pattern = xjoin(base_path, pattern) + elif is_local_path(pattern): + base_path = os.path.splitdrive(pattern)[0] + os.sep + else: + base_path = "" + pattern, storage_options = _prepare_path_and_storage_options(pattern, download_config=download_config) + fs, fs_pattern = url_to_fs(pattern, **storage_options) + files_to_ignore = set(FILES_TO_IGNORE) - {xbasename(pattern)} + protocol = fs.protocol if isinstance(fs.protocol, str) else fs.protocol[0] + protocol_prefix = protocol + "://" if protocol != "file" else "" + glob_kwargs = {} + if protocol == "hf" and config.HF_HUB_VERSION >= version.parse("0.20.0"): + # 10 times faster glob with detail=True (ignores costly info like lastCommit) + glob_kwargs["expand_info"] = False + matched_paths = [ + filepath if filepath.startswith(protocol_prefix) else protocol_prefix + filepath + for filepath, info in fs.glob(pattern, detail=True, **glob_kwargs).items() + if (info["type"] == "file" or (info.get("islink") and os.path.isfile(os.path.realpath(filepath)))) + and (xbasename(filepath) not in files_to_ignore) + and not _is_inside_unrequested_special_dir(filepath, fs_pattern) + and not _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(filepath, fs_pattern) + ] # ignore .ipynb and __pycache__, but keep /../ + if allowed_extensions is not None: + out = [ + filepath + for filepath in matched_paths + if any("." + suffix in allowed_extensions for suffix in xbasename(filepath).split(".")[1:]) + ] + if len(out) < len(matched_paths): + invalid_matched_files = list(set(matched_paths) - set(out)) + logger.info( + f"Some files matched the pattern '{pattern}' but don't have valid data file extensions: {invalid_matched_files}" + ) + else: + out = matched_paths + if not out: + error_msg = f"Unable to find '{pattern}'" + if allowed_extensions is not None: + error_msg += f" with any supported extension {list(allowed_extensions)}" + raise FileNotFoundError(error_msg) + return out + + +def get_data_patterns(base_path: str, download_config: Optional[DownloadConfig] = None) -> dict[str, list[str]]: + """ + Get the default pattern from a directory testing all the supported patterns. + The first patterns to return a non-empty list of data files is returned. + + Some examples of supported patterns: + + Input: + + my_dataset_repository/ + ├── README.md + └── dataset.csv + + Output: + + {'train': ['**']} + + Input: + + my_dataset_repository/ + ├── README.md + ├── train.csv + └── test.csv + + my_dataset_repository/ + ├── README.md + └── data/ + ├── train.csv + └── test.csv + + my_dataset_repository/ + ├── README.md + ├── train_0.csv + ├── train_1.csv + ├── train_2.csv + ├── train_3.csv + ├── test_0.csv + └── test_1.csv + + Output: + + {'train': ['**/train[-._ 0-9]*', '**/*[-._ 0-9]train[-._ 0-9]*', '**/training[-._ 0-9]*', '**/*[-._ 0-9]training[-._ 0-9]*'], + 'test': ['**/test[-._ 0-9]*', '**/*[-._ 0-9]test[-._ 0-9]*', '**/testing[-._ 0-9]*', '**/*[-._ 0-9]testing[-._ 0-9]*', ...]} + + Input: + + my_dataset_repository/ + ├── README.md + └── data/ + ├── train/ + │ ├── shard_0.csv + │ ├── shard_1.csv + │ ├── shard_2.csv + │ └── shard_3.csv + └── test/ + ├── shard_0.csv + └── shard_1.csv + + Output: + + {'train': ['**/train/**', '**/train[-._ 0-9]*/**', '**/*[-._ 0-9]train/**', '**/*[-._ 0-9]train[-._ 0-9]*/**', ...], + 'test': ['**/test/**', '**/test[-._ 0-9]*/**', '**/*[-._ 0-9]test/**', '**/*[-._ 0-9]test[-._ 0-9]*/**', ...]} + + Input: + + my_dataset_repository/ + ├── README.md + └── data/ + ├── train-00000-of-00003.csv + ├── train-00001-of-00003.csv + ├── train-00002-of-00003.csv + ├── test-00000-of-00001.csv + ├── random-00000-of-00003.csv + ├── random-00001-of-00003.csv + └── random-00002-of-00003.csv + + Output: + + {'train': ['data/train-[0-9][0-9][0-9][0-9][0-9]-of-[0-9][0-9][0-9][0-9][0-9]*.*'], + 'test': ['data/test-[0-9][0-9][0-9][0-9][0-9]-of-[0-9][0-9][0-9][0-9][0-9]*.*'], + 'random': ['data/random-[0-9][0-9][0-9][0-9][0-9]-of-[0-9][0-9][0-9][0-9][0-9]*.*']} + + In order, it first tests if SPLIT_PATTERN_SHARDED works, otherwise it tests the patterns in ALL_DEFAULT_PATTERNS. + """ + resolver = partial(resolve_pattern, base_path=base_path, download_config=download_config) + try: + return _get_data_files_patterns(resolver) + except FileNotFoundError: + raise EmptyDatasetError(f"The directory at {base_path} doesn't contain any data files") from None + + +def _get_single_origin_metadata( + data_file: str, + download_config: Optional[DownloadConfig] = None, +) -> SingleOriginMetadata: + data_file, storage_options = _prepare_path_and_storage_options(data_file, download_config=download_config) + fs, *_ = url_to_fs(data_file, **storage_options) + if isinstance(fs, HfFileSystem): + resolved_path = fs.resolve_path(data_file) + return resolved_path.repo_id, resolved_path.revision + elif data_file.startswith(config.HF_ENDPOINT): + hffs = HfFileSystem(endpoint=config.HF_ENDPOINT, token=download_config.token) + data_file = "hf://" + data_file[len(config.HF_ENDPOINT) + 1 :].replace("/resolve/", "@", 1) + resolved_path = hffs.resolve_path(data_file) + return resolved_path.repo_id, resolved_path.revision + info = fs.info(data_file) + # s3fs uses "ETag", gcsfs uses "etag", and for local we simply check mtime + for key in ["ETag", "etag", "mtime"]: + if key in info: + return (str(info[key]),) + return () + + +def _get_origin_metadata( + data_files: list[str], + download_config: Optional[DownloadConfig] = None, + max_workers: Optional[int] = None, +) -> list[SingleOriginMetadata]: + max_workers = max_workers if max_workers is not None else config.HF_DATASETS_MULTITHREADING_MAX_WORKERS + return thread_map( + partial(_get_single_origin_metadata, download_config=download_config), + data_files, + max_workers=max_workers, + tqdm_class=hf_tqdm, + desc="Resolving data files", + # set `disable=None` rather than `disable=False` by default to disable progress bar when no TTY attached + disable=len(data_files) <= 16 or None, + ) + + +class DataFilesList(list[str]): + """ + List of data files (absolute local paths or URLs). + It has two construction methods given the user's data files patterns: + - ``from_hf_repo``: resolve patterns inside a dataset repository + - ``from_local_or_remote``: resolve patterns from a local path + + Moreover, DataFilesList has an additional attribute ``origin_metadata``. + It can store: + - the last modified time of local files + - ETag of remote files + - commit sha of a dataset repository + + Thanks to this additional attribute, it is possible to hash the list + and get a different hash if and only if at least one file changed. + This is useful for caching Dataset objects that are obtained from a list of data files. + """ + + def __init__(self, data_files: list[str], origin_metadata: list[SingleOriginMetadata]) -> None: + super().__init__(data_files) + self.origin_metadata = origin_metadata + + def __add__(self, other: "DataFilesList") -> "DataFilesList": + return DataFilesList([*self, *other], self.origin_metadata + other.origin_metadata) + + @classmethod + def from_hf_repo( + cls, + patterns: list[str], + dataset_info: huggingface_hub.hf_api.DatasetInfo, + base_path: Optional[str] = None, + allowed_extensions: Optional[list[str]] = None, + download_config: Optional[DownloadConfig] = None, + ) -> "DataFilesList": + base_path = f"hf://datasets/{dataset_info.id}@{dataset_info.sha}/{base_path or ''}".rstrip("/") + return cls.from_patterns( + patterns, base_path=base_path, allowed_extensions=allowed_extensions, download_config=download_config + ) + + @classmethod + def from_local_or_remote( + cls, + patterns: list[str], + base_path: Optional[str] = None, + allowed_extensions: Optional[list[str]] = None, + download_config: Optional[DownloadConfig] = None, + ) -> "DataFilesList": + base_path = base_path if base_path is not None else Path().resolve().as_posix() + return cls.from_patterns( + patterns, base_path=base_path, allowed_extensions=allowed_extensions, download_config=download_config + ) + + @classmethod + def from_patterns( + cls, + patterns: list[str], + base_path: Optional[str] = None, + allowed_extensions: Optional[list[str]] = None, + download_config: Optional[DownloadConfig] = None, + ) -> "DataFilesList": + base_path = base_path if base_path is not None else Path().resolve().as_posix() + data_files = [] + for pattern in patterns: + try: + data_files.extend( + resolve_pattern( + pattern, + base_path=base_path, + allowed_extensions=allowed_extensions, + download_config=download_config, + ) + ) + except FileNotFoundError: + if not has_magic(pattern): + raise + origin_metadata = _get_origin_metadata(data_files, download_config=download_config) + return cls(data_files, origin_metadata) + + def filter( + self, *, extensions: Optional[list[str]] = None, file_names: Optional[list[str]] = None + ) -> "DataFilesList": + patterns = [] + if extensions: + ext_pattern = "|".join(re.escape(ext) for ext in extensions) + patterns.append(re.compile(f".*({ext_pattern})(\\..+)?$")) + if file_names: + fn_pattern = "|".join(re.escape(fn) for fn in file_names) + patterns.append(re.compile(rf".*[\/]?({fn_pattern})$")) + if patterns: + return DataFilesList( + [data_file for data_file in self if any(pattern.match(data_file) for pattern in patterns)], + origin_metadata=self.origin_metadata, + ) + else: + return DataFilesList(list(self), origin_metadata=self.origin_metadata) + + +class DataFilesDict(dict[str, DataFilesList]): + """ + Dict of split_name -> list of data files (absolute local paths or URLs). + It has two construction methods given the user's data files patterns : + - ``from_hf_repo``: resolve patterns inside a dataset repository + - ``from_local_or_remote``: resolve patterns from a local path + + Moreover, each list is a DataFilesList. It is possible to hash the dictionary + and get a different hash if and only if at least one file changed. + For more info, see [`DataFilesList`]. + + This is useful for caching Dataset objects that are obtained from a list of data files. + + Changing the order of the keys of this dictionary also doesn't change its hash. + """ + + @classmethod + def from_local_or_remote( + cls, + patterns: dict[str, Union[list[str], DataFilesList]], + base_path: Optional[str] = None, + allowed_extensions: Optional[list[str]] = None, + download_config: Optional[DownloadConfig] = None, + ) -> "DataFilesDict": + out = cls() + for key, patterns_for_key in patterns.items(): + out[key] = ( + patterns_for_key + if isinstance(patterns_for_key, DataFilesList) + else DataFilesList.from_local_or_remote( + patterns_for_key, + base_path=base_path, + allowed_extensions=allowed_extensions, + download_config=download_config, + ) + ) + return out + + @classmethod + def from_hf_repo( + cls, + patterns: dict[str, Union[list[str], DataFilesList]], + dataset_info: huggingface_hub.hf_api.DatasetInfo, + base_path: Optional[str] = None, + allowed_extensions: Optional[list[str]] = None, + download_config: Optional[DownloadConfig] = None, + ) -> "DataFilesDict": + out = cls() + for key, patterns_for_key in patterns.items(): + out[key] = ( + patterns_for_key + if isinstance(patterns_for_key, DataFilesList) + else DataFilesList.from_hf_repo( + patterns_for_key, + dataset_info=dataset_info, + base_path=base_path, + allowed_extensions=allowed_extensions, + download_config=download_config, + ) + ) + return out + + @classmethod + def from_patterns( + cls, + patterns: dict[str, Union[list[str], DataFilesList]], + base_path: Optional[str] = None, + allowed_extensions: Optional[list[str]] = None, + download_config: Optional[DownloadConfig] = None, + ) -> "DataFilesDict": + out = cls() + for key, patterns_for_key in patterns.items(): + out[key] = ( + patterns_for_key + if isinstance(patterns_for_key, DataFilesList) + else DataFilesList.from_patterns( + patterns_for_key, + base_path=base_path, + allowed_extensions=allowed_extensions, + download_config=download_config, + ) + ) + return out + + def filter( + self, *, extensions: Optional[list[str]] = None, file_names: Optional[list[str]] = None + ) -> "DataFilesDict": + out = type(self)() + for key, data_files_list in self.items(): + out[key] = data_files_list.filter(extensions=extensions, file_names=file_names) + return out + + +class DataFilesPatternsList(list[str]): + """ + List of data files patterns (absolute local paths or URLs). + For each pattern there should also be a list of allowed extensions + to keep, or a None ot keep all the files for the pattern. + """ + + def __init__( + self, + patterns: list[str], + allowed_extensions: list[Optional[list[str]]], + ): + super().__init__(patterns) + self.allowed_extensions = allowed_extensions + + def __add__(self, other): + return DataFilesList([*self, *other], self.allowed_extensions + other.allowed_extensions) + + @classmethod + def from_patterns( + cls, patterns: list[str], allowed_extensions: Optional[list[str]] = None + ) -> "DataFilesPatternsList": + return cls(patterns, [allowed_extensions] * len(patterns)) + + def resolve( + self, + base_path: str, + download_config: Optional[DownloadConfig] = None, + ) -> "DataFilesList": + base_path = base_path if base_path is not None else Path().resolve().as_posix() + data_files = [] + for pattern, allowed_extensions in zip(self, self.allowed_extensions): + try: + data_files.extend( + resolve_pattern( + pattern, + base_path=base_path, + allowed_extensions=allowed_extensions, + download_config=download_config, + ) + ) + except FileNotFoundError: + if not has_magic(pattern): + raise + origin_metadata = _get_origin_metadata(data_files, download_config=download_config) + return DataFilesList(data_files, origin_metadata) + + def filter_extensions(self, extensions: list[str]) -> "DataFilesPatternsList": + return DataFilesPatternsList( + self, [allowed_extensions + extensions for allowed_extensions in self.allowed_extensions] + ) + + +class DataFilesPatternsDict(dict[str, DataFilesPatternsList]): + """ + Dict of split_name -> list of data files patterns (absolute local paths or URLs). + """ + + @classmethod + def from_patterns( + cls, patterns: dict[str, list[str]], allowed_extensions: Optional[list[str]] = None + ) -> "DataFilesPatternsDict": + out = cls() + for key, patterns_for_key in patterns.items(): + out[key] = ( + patterns_for_key + if isinstance(patterns_for_key, DataFilesPatternsList) + else DataFilesPatternsList.from_patterns( + patterns_for_key, + allowed_extensions=allowed_extensions, + ) + ) + return out + + def resolve( + self, + base_path: str, + download_config: Optional[DownloadConfig] = None, + ) -> "DataFilesDict": + out = DataFilesDict() + for key, data_files_patterns_list in self.items(): + out[key] = data_files_patterns_list.resolve(base_path, download_config) + return out + + def filter_extensions(self, extensions: list[str]) -> "DataFilesPatternsDict": + out = type(self)() + for key, data_files_patterns_list in self.items(): + out[key] = data_files_patterns_list.filter_extensions(extensions) + return out diff --git a/lib/python3.12/site-packages/datasets/dataset_dict.py b/lib/python3.12/site-packages/datasets/dataset_dict.py new file mode 100644 index 0000000000000000000000000000000000000000..5e1946cf0f737983ae20b1625440a9edb38c0f4b --- /dev/null +++ b/lib/python3.12/site-packages/datasets/dataset_dict.py @@ -0,0 +1,2369 @@ +import contextlib +import copy +import fnmatch +import json +import math +import posixpath +import re +from collections.abc import Sequence +from functools import partial +from pathlib import Path +from typing import Callable, Optional, Union + +import fsspec +import numpy as np +from fsspec.core import url_to_fs +from huggingface_hub import ( + CommitInfo, + CommitOperationAdd, + CommitOperationDelete, + DatasetCard, + DatasetCardData, + HfApi, +) +from huggingface_hub.hf_api import RepoFile + +from . import config +from .arrow_dataset import ( + PUSH_TO_HUB_WITHOUT_METADATA_CONFIGS_SPLIT_PATTERN_SHARDED, + Dataset, +) +from .features import Features +from .features.features import FeatureType +from .info import DatasetInfo, DatasetInfosDict +from .naming import _split_re +from .splits import NamedSplit, Split, SplitDict, SplitInfo +from .table import Table +from .utils import logging +from .utils.doc_utils import is_documented_by +from .utils.metadata import MetadataConfigs +from .utils.py_utils import asdict, glob_pattern_to_regex, string_to_dict +from .utils.typing import PathLike + + +logger = logging.get_logger(__name__) + + +class bind(partial): + def __call__(self, *fn_args, **fn_kwargs): + return self.func(*fn_args, *self.args, **fn_kwargs) + + +class DatasetDict(dict): + """A dictionary (dict of str: datasets.Dataset) with dataset transforms methods (map, filter, etc.)""" + + def _check_values_type(self): + for dataset in self.values(): + if not isinstance(dataset, Dataset): + raise TypeError(f"Values in `DatasetDict` should be of type `Dataset` but got type '{type(dataset)}'") + + def _check_values_features(self): + items = list(self.items()) + for item_a, item_b in zip(items[:-1], items[1:]): + if item_a[1].features != item_b[1].features: + raise ValueError( + f"All datasets in `DatasetDict` should have the same features but features for '{item_a[0]}' and '{item_b[0]}' don't match: {item_a[1].features} != {item_b[1].features}" + ) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + # Here `del` is used to del the pyarrow tables. This properly closes the files used for memory mapped tables + for dataset in self.values(): + if hasattr(dataset, "_data"): + del dataset._data + if hasattr(dataset, "_indices"): + del dataset._indices + + def __getitem__(self, k) -> Dataset: + if isinstance(k, (str, NamedSplit)) or len(self) == 0: + return super().__getitem__(k) + else: + available_suggested_splits = [ + split for split in (Split.TRAIN, Split.TEST, Split.VALIDATION) if split in self + ] + suggested_split = available_suggested_splits[0] if available_suggested_splits else list(self)[0] + raise KeyError( + f"Invalid key: {k}. Please first select a split. For example: " + f"`my_dataset_dictionary['{suggested_split}'][{k}]`. " + f"Available splits: {sorted(self)}" + ) + + @property + def data(self) -> dict[str, Table]: + """The Apache Arrow tables backing each split. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes") + >>> ds.data + ``` + """ + self._check_values_type() + return {k: dataset.data for k, dataset in self.items()} + + @property + def cache_files(self) -> dict[str, dict]: + """The cache files containing the Apache Arrow table backing each split. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes") + >>> ds.cache_files + {'test': [{'filename': '/root/.cache/huggingface/datasets/rotten_tomatoes_movie_review/default/1.0.0/40d411e45a6ce3484deed7cc15b82a53dad9a72aafd9f86f8f227134bec5ca46/rotten_tomatoes_movie_review-test.arrow'}], + 'train': [{'filename': '/root/.cache/huggingface/datasets/rotten_tomatoes_movie_review/default/1.0.0/40d411e45a6ce3484deed7cc15b82a53dad9a72aafd9f86f8f227134bec5ca46/rotten_tomatoes_movie_review-train.arrow'}], + 'validation': [{'filename': '/root/.cache/huggingface/datasets/rotten_tomatoes_movie_review/default/1.0.0/40d411e45a6ce3484deed7cc15b82a53dad9a72aafd9f86f8f227134bec5ca46/rotten_tomatoes_movie_review-validation.arrow'}]} + ``` + """ + self._check_values_type() + return {k: dataset.cache_files for k, dataset in self.items()} + + @property + def num_columns(self) -> dict[str, int]: + """Number of columns in each split of the dataset. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes") + >>> ds.num_columns + {'test': 2, 'train': 2, 'validation': 2} + ``` + """ + self._check_values_type() + return {k: dataset.num_columns for k, dataset in self.items()} + + @property + def num_rows(self) -> dict[str, int]: + """Number of rows in each split of the dataset. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes") + >>> ds.num_rows + {'test': 1066, 'train': 8530, 'validation': 1066} + ``` + """ + self._check_values_type() + return {k: dataset.num_rows for k, dataset in self.items()} + + @property + def column_names(self) -> dict[str, list[str]]: + """Names of the columns in each split of the dataset. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes") + >>> ds.column_names + {'test': ['text', 'label'], + 'train': ['text', 'label'], + 'validation': ['text', 'label']} + ``` + """ + self._check_values_type() + return {k: dataset.column_names for k, dataset in self.items()} + + @property + def shape(self) -> dict[str, tuple[int]]: + """Shape of each split of the dataset (number of rows, number of columns). + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes") + >>> ds.shape + {'test': (1066, 2), 'train': (8530, 2), 'validation': (1066, 2)} + ``` + """ + self._check_values_type() + return {k: dataset.shape for k, dataset in self.items()} + + def flatten(self, max_depth=16) -> "DatasetDict": + """Flatten the Apache Arrow Table of each split (nested features are flatten). + Each column with a struct type is flattened into one column per struct field. + Other columns are left unchanged. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("rajpurkar/squad") + >>> ds["train"].features + {'answers': Sequence(feature={'text': Value(dtype='string', id=None), 'answer_start': Value(dtype='int32', id=None)}, length=-1, id=None), + 'context': Value(dtype='string', id=None), + 'id': Value(dtype='string', id=None), + 'question': Value(dtype='string', id=None), + 'title': Value(dtype='string', id=None)} + >>> ds.flatten() + DatasetDict({ + train: Dataset({ + features: ['id', 'title', 'context', 'question', 'answers.text', 'answers.answer_start'], + num_rows: 87599 + }) + validation: Dataset({ + features: ['id', 'title', 'context', 'question', 'answers.text', 'answers.answer_start'], + num_rows: 10570 + }) + }) + ``` + """ + self._check_values_type() + return DatasetDict({k: dataset.flatten(max_depth=max_depth) for k, dataset in self.items()}) + + def unique(self, column: str) -> dict[str, list]: + """Return a list of the unique elements in a column for each split. + + This is implemented in the low-level backend and as such, very fast. + + Args: + column (`str`): + column name (list all the column names with [`~datasets.DatasetDict.column_names`]) + + Returns: + Dict[`str`, `list`]: Dictionary of unique elements in the given column. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes") + >>> ds.unique("label") + {'test': [1, 0], 'train': [1, 0], 'validation': [1, 0]} + ``` + """ + self._check_values_type() + return {k: dataset.unique(column) for k, dataset in self.items()} + + def cleanup_cache_files(self) -> dict[str, int]: + """Clean up all cache files in the dataset cache directory, excepted the currently used cache file if there is one. + Be careful when running this command that no other process is currently using other cache files. + + Return: + `Dict` with the number of removed files for each split + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes") + >>> ds.cleanup_cache_files() + {'test': 0, 'train': 0, 'validation': 0} + ``` + """ + self._check_values_type() + return {k: dataset.cleanup_cache_files() for k, dataset in self.items()} + + def __repr__(self): + repr = "\n".join([f"{k}: {v}" for k, v in self.items()]) + repr = re.sub(r"^", " " * 4, repr, 0, re.M) + return f"DatasetDict({{\n{repr}\n}})" + + def cast(self, features: Features) -> "DatasetDict": + """ + Cast the dataset to a new set of features. + The transformation is applied to all the datasets of the dataset dictionary. + + Args: + features ([`Features`]): + New features to cast the dataset to. + The name and order of the fields in the features must match the current column names. + The type of the data must also be convertible from one type to the other. + For non-trivial conversion, e.g. `string` <-> `ClassLabel` you should use [`~DatasetDict.map`] to update the dataset. + + Example: + + ```py + >>> from datasets import load_dataset, ClassLabel, Value + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes") + >>> ds["train"].features + {'label': ClassLabel(names=['neg', 'pos'], id=None), + 'text': Value(dtype='string', id=None)} + >>> new_features = ds["train"].features.copy() + >>> new_features['label'] = ClassLabel(names=['bad', 'good']) + >>> new_features['text'] = Value('large_string') + >>> ds = ds.cast(new_features) + >>> ds["train"].features + {'label': ClassLabel(names=['bad', 'good'], id=None), + 'text': Value(dtype='large_string', id=None)} + ``` + """ + self._check_values_type() + return DatasetDict({k: dataset.cast(features=features) for k, dataset in self.items()}) + + def cast_column(self, column: str, feature) -> "DatasetDict": + """Cast column to feature for decoding. + + Args: + column (`str`): + Column name. + feature ([`Feature`]): + Target feature. + + Returns: + [`DatasetDict`] + + Example: + + ```py + >>> from datasets import load_dataset, ClassLabel + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes") + >>> ds["train"].features + {'label': ClassLabel(names=['neg', 'pos'], id=None), + 'text': Value(dtype='string', id=None)} + >>> ds = ds.cast_column('label', ClassLabel(names=['bad', 'good'])) + >>> ds["train"].features + {'label': ClassLabel(names=['bad', 'good'], id=None), + 'text': Value(dtype='string', id=None)} + ``` + """ + self._check_values_type() + return DatasetDict({k: dataset.cast_column(column=column, feature=feature) for k, dataset in self.items()}) + + def remove_columns(self, column_names: Union[str, list[str]]) -> "DatasetDict": + """ + Remove one or several column(s) from each split in the dataset + and the features associated to the column(s). + + The transformation is applied to all the splits of the dataset dictionary. + + You can also remove a column using [`~DatasetDict.map`] with `remove_columns` but the present method + doesn't copy the data of the remaining columns and is thus faster. + + Args: + column_names (`Union[str, list[str]]`): + Name of the column(s) to remove. + + Returns: + [`DatasetDict`]: A copy of the dataset object without the columns to remove. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes") + >>> ds = ds.remove_columns("label") + DatasetDict({ + train: Dataset({ + features: ['text'], + num_rows: 8530 + }) + validation: Dataset({ + features: ['text'], + num_rows: 1066 + }) + test: Dataset({ + features: ['text'], + num_rows: 1066 + }) + }) + ``` + """ + self._check_values_type() + return DatasetDict({k: dataset.remove_columns(column_names=column_names) for k, dataset in self.items()}) + + def rename_column(self, original_column_name: str, new_column_name: str) -> "DatasetDict": + """ + Rename a column in the dataset and move the features associated to the original column under the new column name. + The transformation is applied to all the datasets of the dataset dictionary. + + You can also rename a column using [`~DatasetDict.map`] with `remove_columns` but the present method: + - takes care of moving the original features under the new column name. + - doesn't copy the data to a new dataset and is thus much faster. + + Args: + original_column_name (`str`): + Name of the column to rename. + new_column_name (`str`): + New name for the column. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes") + >>> ds = ds.rename_column("label", "label_new") + DatasetDict({ + train: Dataset({ + features: ['text', 'label_new'], + num_rows: 8530 + }) + validation: Dataset({ + features: ['text', 'label_new'], + num_rows: 1066 + }) + test: Dataset({ + features: ['text', 'label_new'], + num_rows: 1066 + }) + }) + ``` + """ + self._check_values_type() + return DatasetDict( + { + k: dataset.rename_column( + original_column_name=original_column_name, + new_column_name=new_column_name, + ) + for k, dataset in self.items() + } + ) + + def rename_columns(self, column_mapping: dict[str, str]) -> "DatasetDict": + """ + Rename several columns in the dataset, and move the features associated to the original columns under + the new column names. + The transformation is applied to all the datasets of the dataset dictionary. + + Args: + column_mapping (`Dict[str, str]`): + A mapping of columns to rename to their new names. + + Returns: + [`DatasetDict`]: A copy of the dataset with renamed columns. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes") + >>> ds.rename_columns({'text': 'text_new', 'label': 'label_new'}) + DatasetDict({ + train: Dataset({ + features: ['text_new', 'label_new'], + num_rows: 8530 + }) + validation: Dataset({ + features: ['text_new', 'label_new'], + num_rows: 1066 + }) + test: Dataset({ + features: ['text_new', 'label_new'], + num_rows: 1066 + }) + }) + ``` + """ + self._check_values_type() + return DatasetDict({k: dataset.rename_columns(column_mapping=column_mapping) for k, dataset in self.items()}) + + def select_columns(self, column_names: Union[str, list[str]]) -> "DatasetDict": + """Select one or several column(s) from each split in the dataset and + the features associated to the column(s). + + The transformation is applied to all the splits of the dataset + dictionary. + + Args: + column_names (`Union[str, list[str]]`): + Name of the column(s) to keep. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes") + >>> ds.select_columns("text") + DatasetDict({ + train: Dataset({ + features: ['text'], + num_rows: 8530 + }) + validation: Dataset({ + features: ['text'], + num_rows: 1066 + }) + test: Dataset({ + features: ['text'], + num_rows: 1066 + }) + }) + ``` + """ + self._check_values_type() + return DatasetDict({k: dataset.select_columns(column_names=column_names) for k, dataset in self.items()}) + + def class_encode_column(self, column: str, include_nulls: bool = False) -> "DatasetDict": + """Casts the given column as [`~datasets.features.ClassLabel`] and updates the tables. + + Args: + column (`str`): + The name of the column to cast. + include_nulls (`bool`, defaults to `False`): + Whether to include null values in the class labels. If `True`, the null values will be encoded as the `"None"` class label. + + + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("boolq") + >>> ds["train"].features + {'answer': Value(dtype='bool', id=None), + 'passage': Value(dtype='string', id=None), + 'question': Value(dtype='string', id=None)} + >>> ds = ds.class_encode_column("answer") + >>> ds["train"].features + {'answer': ClassLabel(num_classes=2, names=['False', 'True'], id=None), + 'passage': Value(dtype='string', id=None), + 'question': Value(dtype='string', id=None)} + ``` + """ + self._check_values_type() + return DatasetDict( + {k: dataset.class_encode_column(column=column, include_nulls=include_nulls) for k, dataset in self.items()} + ) + + @contextlib.contextmanager + def formatted_as( + self, + type: Optional[str] = None, + columns: Optional[list] = None, + output_all_columns: bool = False, + **format_kwargs, + ): + """To be used in a `with` statement. Set `__getitem__` return format (type and columns). + The transformation is applied to all the datasets of the dataset dictionary. + + Args: + type (`str`, *optional*): + Either output type selected in `[None, 'numpy', 'torch', 'tensorflow', 'jax', 'arrow', 'pandas', 'polars']`. + `None` means `__getitem__` returns python objects (default). + columns (`list[str]`, *optional*): + Columns to format in the output. + `None` means `__getitem__` returns all columns (default). + output_all_columns (`bool`, defaults to False): + Keep un-formatted columns as well in the output (as python objects). + **format_kwargs (additional keyword arguments): + Keywords arguments passed to the convert function like `np.array`, `torch.tensor` or `tensorflow.ragged.constant`. + """ + self._check_values_type() + old_format_type = {k: dataset._format_type for k, dataset in self.items()} + old_format_kwargs = {k: dataset._format_kwargs for k, dataset in self.items()} + old_format_columns = {k: dataset._format_columns for k, dataset in self.items()} + old_output_all_columns = {k: dataset._output_all_columns for k, dataset in self.items()} + try: + self.set_format(type, columns, output_all_columns, **format_kwargs) + yield + finally: + for k, dataset in self.items(): + dataset.set_format( + old_format_type[k], + old_format_columns[k], + old_output_all_columns[k], + **old_format_kwargs[k], + ) + + def set_format( + self, + type: Optional[str] = None, + columns: Optional[list] = None, + output_all_columns: bool = False, + **format_kwargs, + ): + """Set `__getitem__` return format (type and columns). + The format is set for every dataset in the dataset dictionary. + + Args: + type (`str`, *optional*): + Either output type selected in `[None, 'numpy', 'torch', 'tensorflow', 'jax', 'arrow', 'pandas', 'polars']`. + `None` means `__getitem__` returns python objects (default). + columns (`list[str]`, *optional*): + Columns to format in the output. + `None` means `__getitem__` returns all columns (default). + output_all_columns (`bool`, defaults to False): + Keep un-formatted columns as well in the output (as python objects), + **format_kwargs (additional keyword arguments): + Keywords arguments passed to the convert function like `np.array`, `torch.tensor` or `tensorflow.ragged.constant`. + + It is possible to call `map` after calling `set_format`. Since `map` may add new columns, then the list of formatted columns + gets updated. In this case, if you apply `map` on a dataset to add a new column, then this column will be formatted: + + `new formatted columns = (all columns - previously unformatted columns)` + + Example: + + ```py + >>> from datasets import load_dataset + >>> from transformers import AutoTokenizer + >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-cased") + >>> ds = ds.map(lambda x: tokenizer(x["text"], truncation=True, padding=True), batched=True) + >>> ds.set_format(type="numpy", columns=['input_ids', 'token_type_ids', 'attention_mask', 'label']) + >>> ds["train"].format + {'columns': ['input_ids', 'token_type_ids', 'attention_mask', 'label'], + 'format_kwargs': {}, + 'output_all_columns': False, + 'type': 'numpy'} + ``` + """ + self._check_values_type() + for dataset in self.values(): + dataset.set_format( + type=type, + columns=columns, + output_all_columns=output_all_columns, + **format_kwargs, + ) + + def reset_format(self): + """Reset `__getitem__` return format to python objects and all columns. + The transformation is applied to all the datasets of the dataset dictionary. + + Same as `self.set_format()` + + Example: + + ```py + >>> from datasets import load_dataset + >>> from transformers import AutoTokenizer + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes") + >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-cased") + >>> ds = ds.map(lambda x: tokenizer(x["text"], truncation=True, padding=True), batched=True) + >>> ds.set_format(type="numpy", columns=['input_ids', 'token_type_ids', 'attention_mask', 'label']) + >>> ds["train"].format + {'columns': ['input_ids', 'token_type_ids', 'attention_mask', 'label'], + 'format_kwargs': {}, + 'output_all_columns': False, + 'type': 'numpy'} + >>> ds.reset_format() + >>> ds["train"].format + {'columns': ['text', 'label', 'input_ids', 'token_type_ids', 'attention_mask'], + 'format_kwargs': {}, + 'output_all_columns': False, + 'type': None} + ``` + """ + self._check_values_type() + for dataset in self.values(): + dataset.set_format() + + def set_transform( + self, + transform: Optional[Callable], + columns: Optional[list] = None, + output_all_columns: bool = False, + ): + """Set ``__getitem__`` return format using this transform. The transform is applied on-the-fly on batches when ``__getitem__`` is called. + The transform is set for every dataset in the dataset dictionary + As :func:`datasets.Dataset.set_format`, this can be reset using :func:`datasets.Dataset.reset_format` + + Args: + transform (`Callable`, optional): user-defined formatting transform, replaces the format defined by :func:`datasets.Dataset.set_format` + A formatting function is a callable that takes a batch (as a dict) as input and returns a batch. + This function is applied right before returning the objects in ``__getitem__``. + columns (`list[str]`, optional): columns to format in the output + If specified, then the input batch of the transform only contains those columns. + output_all_columns (`bool`, default to False): keep un-formatted columns as well in the output (as python objects) + If set to True, then the other un-formatted columns are kept with the output of the transform. + + """ + self._check_values_type() + for dataset in self.values(): + dataset.set_format( + "custom", + columns=columns, + output_all_columns=output_all_columns, + transform=transform, + ) + + def with_format( + self, + type: Optional[str] = None, + columns: Optional[list] = None, + output_all_columns: bool = False, + **format_kwargs, + ) -> "DatasetDict": + """Set `__getitem__` return format (type and columns). The data formatting is applied on-the-fly. + The format `type` (for example "numpy") is used to format batches when using `__getitem__`. + The format is set for every dataset in the dataset dictionary. + + It's also possible to use custom transforms for formatting using [`~datasets.Dataset.with_transform`]. + + Contrary to [`~datasets.DatasetDict.set_format`], `with_format` returns a new [`DatasetDict`] object with new [`Dataset`] objects. + + Args: + type (`str`, *optional*): + Either output type selected in `[None, 'numpy', 'torch', 'tensorflow', 'jax', 'arrow', 'pandas', 'polars']`. + `None` means `__getitem__` returns python objects (default). + columns (`list[str]`, *optional*): + Columns to format in the output. + `None` means `__getitem__` returns all columns (default). + output_all_columns (`bool`, defaults to `False`): + Keep un-formatted columns as well in the output (as python objects). + **format_kwargs (additional keyword arguments): + Keywords arguments passed to the convert function like `np.array`, `torch.tensor` or `tensorflow.ragged.constant`. + + Example: + + ```py + >>> from datasets import load_dataset + >>> from transformers import AutoTokenizer + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes") + >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-cased") + >>> ds = ds.map(lambda x: tokenizer(x['text'], truncation=True, padding=True), batched=True) + >>> ds["train"].format + {'columns': ['text', 'label', 'input_ids', 'token_type_ids', 'attention_mask'], + 'format_kwargs': {}, + 'output_all_columns': False, + 'type': None} + >>> ds = ds.with_format("torch") + >>> ds["train"].format + {'columns': ['text', 'label', 'input_ids', 'token_type_ids', 'attention_mask'], + 'format_kwargs': {}, + 'output_all_columns': False, + 'type': 'torch'} + >>> ds["train"][0] + {'text': 'compassionately explores the seemingly irreconcilable situation between conservative christian parents and their estranged gay and lesbian children .', + 'label': tensor(1), + 'input_ids': tensor([ 101, 18027, 16310, 16001, 1103, 9321, 178, 11604, 7235, 6617, + 1742, 2165, 2820, 1206, 6588, 22572, 12937, 1811, 2153, 1105, + 1147, 12890, 19587, 6463, 1105, 15026, 1482, 119, 102, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0]), + 'token_type_ids': tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]), + 'attention_mask': tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])} + ``` + """ + dataset = copy.deepcopy(self) + dataset.set_format( + type=type, + columns=columns, + output_all_columns=output_all_columns, + **format_kwargs, + ) + return dataset + + def with_transform( + self, + transform: Optional[Callable], + columns: Optional[list] = None, + output_all_columns: bool = False, + ) -> "DatasetDict": + """Set `__getitem__` return format using this transform. The transform is applied on-the-fly on batches when `__getitem__` is called. + The transform is set for every dataset in the dataset dictionary + + As [`~datasets.Dataset.set_format`], this can be reset using [`~datasets.Dataset.reset_format`]. + + Contrary to [`~datasets.DatasetDict.set_transform`], `with_transform` returns a new [`DatasetDict`] object with new [`Dataset`] objects. + + Args: + transform (`Callable`, *optional*): + User-defined formatting transform, replaces the format defined by [`~datasets.Dataset.set_format`]. + A formatting function is a callable that takes a batch (as a dict) as input and returns a batch. + This function is applied right before returning the objects in `__getitem__`. + columns (`list[str]`, *optional*): + Columns to format in the output. + If specified, then the input batch of the transform only contains those columns. + output_all_columns (`bool`, defaults to False): + Keep un-formatted columns as well in the output (as python objects). + If set to `True`, then the other un-formatted columns are kept with the output of the transform. + + Example: + + ```py + >>> from datasets import load_dataset + >>> from transformers import AutoTokenizer + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes") + >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-cased") + >>> def encode(example): + ... return tokenizer(example['text'], truncation=True, padding=True, return_tensors="pt") + >>> ds = ds.with_transform(encode) + >>> ds["train"][0] + {'attention_mask': tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1]), + 'input_ids': tensor([ 101, 1103, 2067, 1110, 17348, 1106, 1129, 1103, 6880, 1432, + 112, 188, 1207, 107, 14255, 1389, 107, 1105, 1115, 1119, + 112, 188, 1280, 1106, 1294, 170, 24194, 1256, 3407, 1190, + 170, 11791, 5253, 188, 1732, 7200, 10947, 12606, 2895, 117, + 179, 7766, 118, 172, 15554, 1181, 3498, 6961, 3263, 1137, + 188, 1566, 7912, 14516, 6997, 119, 102]), + 'token_type_ids': tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0])} + ``` + """ + dataset = copy.deepcopy(self) + dataset.set_transform(transform=transform, columns=columns, output_all_columns=output_all_columns) + return dataset + + def map( + self, + function: Optional[Callable] = None, + with_indices: bool = False, + with_rank: bool = False, + with_split: bool = False, + input_columns: Optional[Union[str, list[str]]] = None, + batched: bool = False, + batch_size: Optional[int] = 1000, + drop_last_batch: bool = False, + remove_columns: Optional[Union[str, list[str]]] = None, + keep_in_memory: bool = False, + load_from_cache_file: Optional[bool] = None, + cache_file_names: Optional[dict[str, Optional[str]]] = None, + writer_batch_size: Optional[int] = 1000, + features: Optional[Features] = None, + disable_nullable: bool = False, + fn_kwargs: Optional[dict] = None, + num_proc: Optional[int] = None, + desc: Optional[str] = None, + try_original_type: Optional[bool] = True, + ) -> "DatasetDict": + """ + Apply a function to all the examples in the table (individually or in batches) and update the table. + If your function returns a column that already exists, then it overwrites it. + The transformation is applied to all the datasets of the dataset dictionary. + + You can specify whether the function should be batched or not with the `batched` parameter: + + - If batched is `False`, then the function takes 1 example in and should return 1 example. + An example is a dictionary, e.g. `{"text": "Hello there !"}`. + - If batched is `True` and `batch_size` is 1, then the function takes a batch of 1 example as input and can return a batch with 1 or more examples. + A batch is a dictionary, e.g. a batch of 1 example is `{"text": ["Hello there !"]}`. + - If batched is `True` and `batch_size` is `n > 1`, then the function takes a batch of `n` examples as input and can return a batch with `n` examples, or with an arbitrary number of examples. + Note that the last batch may have less than `n` examples. + A batch is a dictionary, e.g. a batch of `n` examples is `{"text": ["Hello there !"] * n}`. + + If the function is asynchronous, then `map` will run your function in parallel, with up to one thousand simulatenous calls. + It is recommended to use a `asyncio.Semaphore` in your function if you want to set a maximum number of operations that can run at the same time. + + Args: + function (`callable`): with one of the following signature: + - `function(example: Dict[str, Any]) -> Dict[str, Any]` if `batched=False` and `with_indices=False` + - `function(example: Dict[str, Any], indices: int) -> Dict[str, Any]` if `batched=False` and `with_indices=True` + - `function(batch: Dict[str, list]) -> Dict[str, list]` if `batched=True` and `with_indices=False` + - `function(batch: Dict[str, list], indices: list[int]) -> Dict[str, list]` if `batched=True` and `with_indices=True` + + For advanced usage, the function can also return a `pyarrow.Table`. + If the function is asynchronous, then `map` will run your function in parallel. + Moreover if your function returns nothing (`None`), then `map` will run your function and return the dataset unchanged. + If no function is provided, default to identity function: `lambda x: x`. + with_indices (`bool`, defaults to `False`): + Provide example indices to `function`. Note that in this case the signature of `function` should be `def function(example, idx): ...`. + with_rank (`bool`, defaults to `False`): + Provide process rank to `function`. Note that in this case the + signature of `function` should be `def function(example[, idx], rank): ...`. + with_split (`bool`, defaults to `False`): + Provide process split to `function`. Note that in this case the + signature of `function` should be `def function(example[, idx], split): ...`. + input_columns (`[Union[str, list[str]]]`, *optional*, defaults to `None`): + The columns to be passed into `function` as + positional arguments. If `None`, a dict mapping to all formatted columns is passed as one argument. + batched (`bool`, defaults to `False`): + Provide batch of examples to `function`. + batch_size (`int`, *optional*, defaults to `1000`): + Number of examples per batch provided to `function` if `batched=True`, + `batch_size <= 0` or `batch_size == None` then provide the full dataset as a single batch to `function`. + drop_last_batch (`bool`, defaults to `False`): + Whether a last batch smaller than the batch_size should be + dropped instead of being processed by the function. + remove_columns (`[Union[str, list[str]]]`, *optional*, defaults to `None`): + Remove a selection of columns while doing the mapping. + Columns will be removed before updating the examples with the output of `function`, i.e. if `function` is adding + columns with names in `remove_columns`, these columns will be kept. + keep_in_memory (`bool`, defaults to `False`): + Keep the dataset in memory instead of writing it to a cache file. + load_from_cache_file (`Optional[bool]`, defaults to `True` if caching is enabled): + If a cache file storing the current computation from `function` + can be identified, use it instead of recomputing. + cache_file_names (`[Dict[str, str]]`, *optional*, defaults to `None`): + Provide the name of a path for the cache file. It is used to store the + results of the computation instead of the automatically generated cache file name. + You have to provide one `cache_file_name` per dataset in the dataset dictionary. + writer_batch_size (`int`, default `1000`): + Number of rows per write operation for the cache file writer. + This value is a good trade-off between memory usage during the processing, and processing speed. + Higher value makes the processing do fewer lookups, lower value consume less temporary memory while running `map`. + features (`[datasets.Features]`, *optional*, defaults to `None`): + Use a specific [`Features`] to store the cache file + instead of the automatically generated one. + disable_nullable (`bool`, defaults to `False`): + Disallow null values in the table. + fn_kwargs (`Dict`, *optional*, defaults to `None`): + Keyword arguments to be passed to `function` + num_proc (`int`, *optional*, defaults to `None`): + Number of processes for multiprocessing. By default it doesn't + use multiprocessing. + desc (`str`, *optional*, defaults to `None`): + Meaningful description to be displayed alongside with the progress bar while mapping examples. + try_original_type (`Optional[bool]`, defaults to `True`): + Try to keep the types of the original columns (e.g. int32 -> int32). + Set to False if you want to always infer new types. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes") + >>> def add_prefix(example): + ... example["text"] = "Review: " + example["text"] + ... return example + >>> ds = ds.map(add_prefix) + >>> ds["train"][0:3]["text"] + ['Review: the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .', + 'Review: the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson\'s expanded vision of j . r . r . tolkien\'s middle-earth .', + 'Review: effective but too-tepid biopic'] + + # process a batch of examples + >>> ds = ds.map(lambda example: tokenizer(example["text"]), batched=True) + # set number of processors + >>> ds = ds.map(add_prefix, num_proc=4) + ``` + """ + self._check_values_type() + if cache_file_names is None: + cache_file_names = dict.fromkeys(self) + + dataset_dict = {} + for split, dataset in self.items(): + if with_split: + function = bind(function, split) + + dataset_dict[split] = dataset.map( + function=function, + with_indices=with_indices, + with_rank=with_rank, + input_columns=input_columns, + batched=batched, + batch_size=batch_size, + drop_last_batch=drop_last_batch, + remove_columns=remove_columns, + keep_in_memory=keep_in_memory, + load_from_cache_file=load_from_cache_file, + cache_file_name=cache_file_names[split], + writer_batch_size=writer_batch_size, + features=features, + disable_nullable=disable_nullable, + fn_kwargs=fn_kwargs, + num_proc=num_proc, + desc=desc, + try_original_type=try_original_type, + ) + + if with_split: + function = function.func + + return DatasetDict(dataset_dict) + + def filter( + self, + function: Optional[Callable] = None, + with_indices: bool = False, + with_rank: bool = False, + input_columns: Optional[Union[str, list[str]]] = None, + batched: bool = False, + batch_size: Optional[int] = 1000, + keep_in_memory: bool = False, + load_from_cache_file: Optional[bool] = None, + cache_file_names: Optional[dict[str, Optional[str]]] = None, + writer_batch_size: Optional[int] = 1000, + fn_kwargs: Optional[dict] = None, + num_proc: Optional[int] = None, + desc: Optional[str] = None, + ) -> "DatasetDict": + """Apply a filter function to all the elements in the table in batches + and update the table so that the dataset only includes examples according to the filter function. + The transformation is applied to all the datasets of the dataset dictionary. + + Args: + function (`Callable`): Callable with one of the following signatures: + + - `function(example: Dict[str, Any]) -> bool` if `batched=False` and `with_indices=False` and `with_rank=False` + - `function(example: Dict[str, Any], *extra_args) -> bool` if `batched=False` and `with_indices=True` and/or `with_rank=True` (one extra arg for each) + - `function(batch: Dict[str, list]) -> list[bool]` if `batched=True` and `with_indices=False` and `with_rank=False` + - `function(batch: Dict[str, list], *extra_args) -> list[bool]` if `batched=True` and `with_indices=True` and/or `with_rank=True` (one extra arg for each) + + If no function is provided, defaults to an always `True` function: `lambda x: True`. + with_indices (`bool`, defaults to `False`): + Provide example indices to `function`. Note that in this case the + signature of `function` should be `def function(example, idx[, rank]): ...`. + with_rank (`bool`, defaults to `False`): + Provide process rank to `function`. Note that in this case the + signature of `function` should be `def function(example[, idx], rank): ...`. + input_columns (`[Union[str, list[str]]]`, *optional*, defaults to `None`): + The columns to be passed into `function` as + positional arguments. If `None`, a dict mapping to all formatted columns is passed as one argument. + batched (`bool`, defaults to `False`): + Provide batch of examples to `function`. + batch_size (`int`, *optional*, defaults to `1000`): + Number of examples per batch provided to `function` if `batched=True` + `batch_size <= 0` or `batch_size == None` then provide the full dataset as a single batch to `function`. + keep_in_memory (`bool`, defaults to `False`): + Keep the dataset in memory instead of writing it to a cache file. + load_from_cache_file (`Optional[bool]`, defaults to `True` if caching is enabled): + If a cache file storing the current computation from `function` + can be identified, use it instead of recomputing. + cache_file_names (`[Dict[str, str]]`, *optional*, defaults to `None`): + Provide the name of a path for the cache file. It is used to store the + results of the computation instead of the automatically generated cache file name. + You have to provide one `cache_file_name` per dataset in the dataset dictionary. + writer_batch_size (`int`, defaults to `1000`): + Number of rows per write operation for the cache file writer. + This value is a good trade-off between memory usage during the processing, and processing speed. + Higher value makes the processing do fewer lookups, lower value consume less temporary memory while running `map`. + fn_kwargs (`Dict`, *optional*, defaults to `None`): + Keyword arguments to be passed to `function` + num_proc (`int`, *optional*, defaults to `None`): + Number of processes for multiprocessing. By default it doesn't + use multiprocessing. + desc (`str`, *optional*, defaults to `None`): + Meaningful description to be displayed alongside with the progress bar while filtering examples. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes") + >>> ds.filter(lambda x: x["label"] == 1) + DatasetDict({ + train: Dataset({ + features: ['text', 'label'], + num_rows: 4265 + }) + validation: Dataset({ + features: ['text', 'label'], + num_rows: 533 + }) + test: Dataset({ + features: ['text', 'label'], + num_rows: 533 + }) + }) + ``` + """ + self._check_values_type() + if cache_file_names is None: + cache_file_names = dict.fromkeys(self) + return DatasetDict( + { + k: dataset.filter( + function=function, + with_indices=with_indices, + with_rank=with_rank, + input_columns=input_columns, + batched=batched, + batch_size=batch_size, + keep_in_memory=keep_in_memory, + load_from_cache_file=load_from_cache_file, + cache_file_name=cache_file_names[k], + writer_batch_size=writer_batch_size, + fn_kwargs=fn_kwargs, + num_proc=num_proc, + desc=desc, + ) + for k, dataset in self.items() + } + ) + + def flatten_indices( + self, + keep_in_memory: bool = False, + cache_file_names: Optional[dict[str, Optional[str]]] = None, + writer_batch_size: Optional[int] = 1000, + features: Optional[Features] = None, + disable_nullable: bool = False, + num_proc: Optional[int] = None, + new_fingerprint: Optional[str] = None, + ) -> "DatasetDict": + """Create and cache a new Dataset by flattening the indices mapping. + + Args: + keep_in_memory (`bool`, defaults to `False`): + Keep the dataset in memory instead of writing it to a cache file. + cache_file_names (`Dict[str, str]`, *optional*, default `None`): + Provide the name of a path for the cache file. It is used to store the + results of the computation instead of the automatically generated cache file name. + You have to provide one `cache_file_name` per dataset in the dataset dictionary. + writer_batch_size (`int`, defaults to `1000`): + Number of rows per write operation for the cache file writer. + This value is a good trade-off between memory usage during the processing, and processing speed. + Higher value makes the processing do fewer lookups, lower value consume less temporary memory while running `map`. + features (`Optional[datasets.Features]`, defaults to `None`): + Use a specific [`Features`] to store the cache file + instead of the automatically generated one. + disable_nullable (`bool`, defaults to `False`): + Allow null values in the table. + num_proc (`int`, optional, default `None`): + Max number of processes when generating cache. Already cached shards are loaded sequentially + new_fingerprint (`str`, *optional*, defaults to `None`): + The new fingerprint of the dataset after transform. + If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments + """ + self._check_values_type() + if cache_file_names is None: + cache_file_names = dict.fromkeys(self) + return DatasetDict( + { + k: dataset.flatten_indices( + keep_in_memory=keep_in_memory, + cache_file_name=cache_file_names[k], + writer_batch_size=writer_batch_size, + features=features, + disable_nullable=disable_nullable, + num_proc=num_proc, + new_fingerprint=new_fingerprint, + ) + for k, dataset in self.items() + } + ) + + def sort( + self, + column_names: Union[str, Sequence[str]], + reverse: Union[bool, Sequence[bool]] = False, + null_placement: str = "at_end", + keep_in_memory: bool = False, + load_from_cache_file: Optional[bool] = None, + indices_cache_file_names: Optional[dict[str, Optional[str]]] = None, + writer_batch_size: Optional[int] = 1000, + ) -> "DatasetDict": + """Create a new dataset sorted according to a single or multiple columns. + + Args: + column_names (`Union[str, Sequence[str]]`): + Column name(s) to sort by. + reverse (`Union[bool, Sequence[bool]]`, defaults to `False`): + If `True`, sort by descending order rather than ascending. If a single bool is provided, + the value is applied to the sorting of all column names. Otherwise a list of bools with the + same length and order as column_names must be provided. + null_placement (`str`, defaults to `at_end`): + Put `None` values at the beginning if `at_start` or `first` or at the end if `at_end` or `last` + keep_in_memory (`bool`, defaults to `False`): + Keep the sorted indices in memory instead of writing it to a cache file. + load_from_cache_file (`Optional[bool]`, defaults to `True` if caching is enabled): + If a cache file storing the sorted indices + can be identified, use it instead of recomputing. + indices_cache_file_names (`[Dict[str, str]]`, *optional*, defaults to `None`): + Provide the name of a path for the cache file. It is used to store the + indices mapping instead of the automatically generated cache file name. + You have to provide one `cache_file_name` per dataset in the dataset dictionary. + writer_batch_size (`int`, defaults to `1000`): + Number of rows per write operation for the cache file writer. + Higher value gives smaller cache files, lower value consume less temporary memory. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset('cornell-movie-review-data/rotten_tomatoes') + >>> ds['train']['label'][:10] + [1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + >>> sorted_ds = ds.sort('label') + >>> sorted_ds['train']['label'][:10] + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + >>> another_sorted_ds = ds.sort(['label', 'text'], reverse=[True, False]) + >>> another_sorted_ds['train']['label'][:10] + [1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + ``` + """ + self._check_values_type() + if indices_cache_file_names is None: + indices_cache_file_names = dict.fromkeys(self) + return DatasetDict( + { + k: dataset.sort( + column_names=column_names, + reverse=reverse, + null_placement=null_placement, + keep_in_memory=keep_in_memory, + load_from_cache_file=load_from_cache_file, + indices_cache_file_name=indices_cache_file_names[k], + writer_batch_size=writer_batch_size, + ) + for k, dataset in self.items() + } + ) + + def shuffle( + self, + seeds: Optional[Union[int, dict[str, Optional[int]]]] = None, + seed: Optional[int] = None, + generators: Optional[dict[str, np.random.Generator]] = None, + keep_in_memory: bool = False, + load_from_cache_file: Optional[bool] = None, + indices_cache_file_names: Optional[dict[str, Optional[str]]] = None, + writer_batch_size: Optional[int] = 1000, + ) -> "DatasetDict": + """Create a new Dataset where the rows are shuffled. + + The transformation is applied to all the datasets of the dataset dictionary. + + Currently shuffling uses numpy random generators. + You can either supply a NumPy BitGenerator to use, or a seed to initiate NumPy's default random generator (PCG64). + + Args: + seeds (`Dict[str, int]` or `int`, *optional*): + A seed to initialize the default BitGenerator if `generator=None`. + If `None`, then fresh, unpredictable entropy will be pulled from the OS. + If an `int` or `array_like[ints]` is passed, then it will be passed to SeedSequence to derive the initial BitGenerator state. + You can provide one `seed` per dataset in the dataset dictionary. + seed (`int`, *optional*): + A seed to initialize the default BitGenerator if `generator=None`. Alias for seeds (a `ValueError` is raised if both are provided). + generators (`Dict[str, *optional*, np.random.Generator]`): + Numpy random Generator to use to compute the permutation of the dataset rows. + If `generator=None` (default), uses `np.random.default_rng` (the default BitGenerator (PCG64) of NumPy). + You have to provide one `generator` per dataset in the dataset dictionary. + keep_in_memory (`bool`, defaults to `False`): + Keep the dataset in memory instead of writing it to a cache file. + load_from_cache_file (`Optional[bool]`, defaults to `True` if caching is enabled): + If a cache file storing the current computation from `function` + can be identified, use it instead of recomputing. + indices_cache_file_names (`Dict[str, str]`, *optional*): + Provide the name of a path for the cache file. It is used to store the + indices mappings instead of the automatically generated cache file name. + You have to provide one `cache_file_name` per dataset in the dataset dictionary. + writer_batch_size (`int`, defaults to `1000`): + Number of rows per write operation for the cache file writer. + This value is a good trade-off between memory usage during the processing, and processing speed. + Higher value makes the processing do fewer lookups, lower value consume less temporary memory while running `map`. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes") + >>> ds["train"]["label"][:10] + [1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + + # set a seed + >>> shuffled_ds = ds.shuffle(seed=42) + >>> shuffled_ds["train"]["label"][:10] + [0, 1, 0, 1, 0, 0, 0, 0, 0, 0] + ``` + """ + self._check_values_type() + if seed is not None and seeds is not None: + raise ValueError("Please specify seed or seeds, but not both") + seeds = seed if seed is not None else seeds + if seeds is None: + seeds = dict.fromkeys(self) + elif not isinstance(seeds, dict): + seeds = dict.fromkeys(self, seeds) + if generators is None: + generators = dict.fromkeys(self) + if indices_cache_file_names is None: + indices_cache_file_names = dict.fromkeys(self) + return DatasetDict( + { + k: dataset.shuffle( + seed=seeds[k], + generator=generators[k], + keep_in_memory=keep_in_memory, + load_from_cache_file=load_from_cache_file, + indices_cache_file_name=indices_cache_file_names[k], + writer_batch_size=writer_batch_size, + ) + for k, dataset in self.items() + } + ) + + def save_to_disk( + self, + dataset_dict_path: PathLike, + max_shard_size: Optional[Union[str, int]] = None, + num_shards: Optional[dict[str, int]] = None, + num_proc: Optional[int] = None, + storage_options: Optional[dict] = None, + ): + """ + Saves a dataset dict to a filesystem using `fsspec.spec.AbstractFileSystem`. + + For [`Image`], [`Audio`] and [`Video`] data: + + All the Image(), Audio() and Video() data are stored in the arrow files. + If you want to store paths or urls, please use the Value("string") type. + + Args: + dataset_dict_path (`path-like`): + Path (e.g. `dataset/train`) or remote URI (e.g. `s3://my-bucket/dataset/train`) + of the dataset dict directory where the dataset dict will be saved to. + max_shard_size (`int` or `str`, *optional*, defaults to `"500MB"`): + The maximum size of the dataset shards to be uploaded to the hub. If expressed as a string, needs to be digits followed by a unit + (like `"50MB"`). + num_shards (`Dict[str, int]`, *optional*): + Number of shards to write. By default the number of shards depends on `max_shard_size` and `num_proc`. + You need to provide the number of shards for each dataset in the dataset dictionary. + Use a dictionary to define a different num_shards for each split. + + + num_proc (`int`, *optional*, default `None`): + Number of processes when downloading and generating the dataset locally. + Multiprocessing is disabled by default. + + + storage_options (`dict`, *optional*): + Key/value pairs to be passed on to the file-system backend, if any. + + + + Example: + + ```python + >>> dataset_dict.save_to_disk("path/to/dataset/directory") + >>> dataset_dict.save_to_disk("path/to/dataset/directory", max_shard_size="1GB") + >>> dataset_dict.save_to_disk("path/to/dataset/directory", num_shards={"train": 1024, "test": 8}) + ``` + """ + fs: fsspec.AbstractFileSystem + fs, _ = url_to_fs(dataset_dict_path, **(storage_options or {})) + + if num_shards is None: + num_shards = dict.fromkeys(self) + elif not isinstance(num_shards, dict): + raise ValueError( + "Please provide one `num_shards` per dataset in the dataset dictionary, e.g. {{'train': 128, 'test': 4}}" + ) + + fs.makedirs(dataset_dict_path, exist_ok=True) + + with fs.open( + posixpath.join(dataset_dict_path, config.DATASETDICT_JSON_FILENAME), + "w", + encoding="utf-8", + ) as f: + json.dump({"splits": list(self)}, f) + for k, dataset in self.items(): + dataset.save_to_disk( + posixpath.join(dataset_dict_path, k), + num_shards=num_shards.get(k), + max_shard_size=max_shard_size, + num_proc=num_proc, + storage_options=storage_options, + ) + + @staticmethod + def load_from_disk( + dataset_dict_path: PathLike, + keep_in_memory: Optional[bool] = None, + storage_options: Optional[dict] = None, + ) -> "DatasetDict": + """ + Load a dataset that was previously saved using [`save_to_disk`] from a filesystem using `fsspec.spec.AbstractFileSystem`. + + Args: + dataset_dict_path (`path-like`): + Path (e.g. `"dataset/train"`) or remote URI (e.g. `"s3//my-bucket/dataset/train"`) + of the dataset dict directory where the dataset dict will be loaded from. + keep_in_memory (`bool`, defaults to `None`): + Whether to copy the dataset in-memory. If `None`, the + dataset will not be copied in-memory unless explicitly enabled by setting + `datasets.config.IN_MEMORY_MAX_SIZE` to nonzero. See more details in the + [improve performance](../cache#improve-performance) section. + storage_options (`dict`, *optional*): + Key/value pairs to be passed on to the file-system backend, if any. + + + + Returns: + [`DatasetDict`] + + Example: + + ```py + >>> ds = load_from_disk('path/to/dataset/directory') + ``` + """ + fs: fsspec.AbstractFileSystem + fs, dataset_dict_path = url_to_fs(dataset_dict_path, **(storage_options or {})) + + dataset_dict_json_path = posixpath.join(dataset_dict_path, config.DATASETDICT_JSON_FILENAME) + dataset_state_json_path = posixpath.join(dataset_dict_path, config.DATASET_STATE_JSON_FILENAME) + dataset_info_path = posixpath.join(dataset_dict_path, config.DATASET_INFO_FILENAME) + if not fs.isfile(dataset_dict_json_path): + if fs.isfile(dataset_info_path) and fs.isfile(dataset_state_json_path): + raise FileNotFoundError( + f"No such file: '{dataset_dict_json_path}'. Expected to load a `DatasetDict` object, but got a `Dataset`. Please use either `datasets.load_from_disk` or `Dataset.load_from_disk` instead." + ) + raise FileNotFoundError( + f"No such file: '{dataset_dict_json_path}'. Expected to load a `DatasetDict` object, but provided path is not a `DatasetDict`." + ) + + with fs.open(dataset_dict_json_path, "r", encoding="utf-8") as f: + splits = json.load(f)["splits"] + + dataset_dict = DatasetDict() + for k in splits: + dataset_dict_split_path = posixpath.join(fs.unstrip_protocol(dataset_dict_path), k) + dataset_dict[k] = Dataset.load_from_disk( + dataset_dict_split_path, + keep_in_memory=keep_in_memory, + storage_options=storage_options, + ) + return dataset_dict + + @staticmethod + def from_csv( + path_or_paths: dict[str, PathLike], + features: Optional[Features] = None, + cache_dir: str = None, + keep_in_memory: bool = False, + **kwargs, + ) -> "DatasetDict": + """Create [`DatasetDict`] from CSV file(s). + + Args: + path_or_paths (`dict` of path-like): + Path(s) of the CSV file(s). + features ([`Features`], *optional*): + Dataset features. + cache_dir (str, *optional*, defaults to `"~/.cache/huggingface/datasets"`): + Directory to cache data. + keep_in_memory (`bool`, defaults to `False`): + Whether to copy the data in-memory. + **kwargs (additional keyword arguments): + Keyword arguments to be passed to [`pandas.read_csv`]. + + Returns: + [`DatasetDict`] + + Example: + + ```py + >>> from datasets import DatasetDict + >>> ds = DatasetDict.from_csv({'train': 'path/to/dataset.csv'}) + ``` + """ + # Dynamic import to avoid circular dependency + from .io.csv import CsvDatasetReader + + return CsvDatasetReader( + path_or_paths, + features=features, + cache_dir=cache_dir, + keep_in_memory=keep_in_memory, + **kwargs, + ).read() + + @staticmethod + def from_json( + path_or_paths: dict[str, PathLike], + features: Optional[Features] = None, + cache_dir: str = None, + keep_in_memory: bool = False, + **kwargs, + ) -> "DatasetDict": + """Create [`DatasetDict`] from JSON Lines file(s). + + Args: + path_or_paths (`path-like` or list of `path-like`): + Path(s) of the JSON Lines file(s). + features ([`Features`], *optional*): + Dataset features. + cache_dir (str, *optional*, defaults to `"~/.cache/huggingface/datasets"`): + Directory to cache data. + keep_in_memory (`bool`, defaults to `False`): + Whether to copy the data in-memory. + **kwargs (additional keyword arguments): + Keyword arguments to be passed to [`JsonConfig`]. + + Returns: + [`DatasetDict`] + + Example: + + ```py + >>> from datasets import DatasetDict + >>> ds = DatasetDict.from_json({'train': 'path/to/dataset.json'}) + ``` + """ + # Dynamic import to avoid circular dependency + from .io.json import JsonDatasetReader + + return JsonDatasetReader( + path_or_paths, + features=features, + cache_dir=cache_dir, + keep_in_memory=keep_in_memory, + **kwargs, + ).read() + + @staticmethod + def from_parquet( + path_or_paths: dict[str, PathLike], + features: Optional[Features] = None, + cache_dir: str = None, + keep_in_memory: bool = False, + columns: Optional[list[str]] = None, + **kwargs, + ) -> "DatasetDict": + """Create [`DatasetDict`] from Parquet file(s). + + Args: + path_or_paths (`dict` of path-like): + Path(s) of the CSV file(s). + features ([`Features`], *optional*): + Dataset features. + cache_dir (`str`, *optional*, defaults to `"~/.cache/huggingface/datasets"`): + Directory to cache data. + keep_in_memory (`bool`, defaults to `False`): + Whether to copy the data in-memory. + columns (`list[str]`, *optional*): + If not `None`, only these columns will be read from the file. + A column name may be a prefix of a nested field, e.g. 'a' will select + 'a.b', 'a.c', and 'a.d.e'. + **kwargs (additional keyword arguments): + Keyword arguments to be passed to [`ParquetConfig`]. + + Returns: + [`DatasetDict`] + + Example: + + ```py + >>> from datasets import DatasetDict + >>> ds = DatasetDict.from_parquet({'train': 'path/to/dataset/parquet'}) + ``` + """ + # Dynamic import to avoid circular dependency + from .io.parquet import ParquetDatasetReader + + return ParquetDatasetReader( + path_or_paths, + features=features, + cache_dir=cache_dir, + keep_in_memory=keep_in_memory, + columns=columns, + **kwargs, + ).read() + + @staticmethod + def from_text( + path_or_paths: dict[str, PathLike], + features: Optional[Features] = None, + cache_dir: str = None, + keep_in_memory: bool = False, + **kwargs, + ) -> "DatasetDict": + """Create [`DatasetDict`] from text file(s). + + Args: + path_or_paths (`dict` of path-like): + Path(s) of the text file(s). + features ([`Features`], *optional*): + Dataset features. + cache_dir (`str`, *optional*, defaults to `"~/.cache/huggingface/datasets"`): + Directory to cache data. + keep_in_memory (`bool`, defaults to `False`): + Whether to copy the data in-memory. + **kwargs (additional keyword arguments): + Keyword arguments to be passed to [`TextConfig`]. + + Returns: + [`DatasetDict`] + + Example: + + ```py + >>> from datasets import DatasetDict + >>> ds = DatasetDict.from_text({'train': 'path/to/dataset.txt'}) + ``` + """ + # Dynamic import to avoid circular dependency + from .io.text import TextDatasetReader + + return TextDatasetReader( + path_or_paths, + features=features, + cache_dir=cache_dir, + keep_in_memory=keep_in_memory, + **kwargs, + ).read() + + @is_documented_by(Dataset.align_labels_with_mapping) + def align_labels_with_mapping(self, label2id: dict, label_column: str) -> "DatasetDict": + self._check_values_type() + return DatasetDict( + { + k: dataset.align_labels_with_mapping(label2id=label2id, label_column=label_column) + for k, dataset in self.items() + } + ) + + def push_to_hub( + self, + repo_id, + config_name: str = "default", + set_default: Optional[bool] = None, + data_dir: Optional[str] = None, + commit_message: Optional[str] = None, + commit_description: Optional[str] = None, + private: Optional[bool] = None, + token: Optional[str] = None, + revision: Optional[str] = None, + create_pr: Optional[bool] = False, + max_shard_size: Optional[Union[int, str]] = None, + num_shards: Optional[dict[str, int]] = None, + embed_external_files: bool = True, + ) -> CommitInfo: + """Pushes the [`DatasetDict`] to the hub as a Parquet dataset. + The [`DatasetDict`] is pushed using HTTP requests and does not need to have neither git or git-lfs installed. + + Each dataset split will be pushed independently. The pushed dataset will keep the original split names. + + The resulting Parquet files are self-contained by default: if your dataset contains [`Image`] or [`Audio`] + data, the Parquet files will store the bytes of your images or audio files. + You can disable this by setting `embed_external_files` to False. + + Args: + repo_id (`str`): + The ID of the repository to push to in the following format: `/` or + `/`. Also accepts ``, which will default to the namespace + of the logged-in user. + config_name (`str`): + Configuration name of a dataset. Defaults to "default". + set_default (`bool`, *optional*): + Whether to set this configuration as the default one. Otherwise, the default configuration is the one + named "default". + data_dir (`str`, *optional*): + Directory name that will contain the uploaded data files. Defaults to the `config_name` if different + from "default", else "data". + + + commit_message (`str`, *optional*): + Message to commit while pushing. Will default to `"Upload dataset"`. + commit_description (`str`, *optional*): + Description of the commit that will be created. + Additionally, description of the PR if a PR is created (`create_pr` is True). + + + private (`bool`, *optional*): + Whether to make the repo private. If `None` (default), the repo will be public unless the + organization's default is private. This value is ignored if the repo already exists. + token (`str`, *optional*): + An optional authentication token for the Hugging Face Hub. If no token is passed, will default + to the token saved locally when logging in with `huggingface-cli login`. Will raise an error + if no token is passed and the user is not logged-in. + revision (`str`, *optional*): + Branch to push the uploaded files to. Defaults to the `"main"` branch. + + + create_pr (`bool`, *optional*, defaults to `False`): + Whether to create a PR with the uploaded files or directly commit. + + + max_shard_size (`int` or `str`, *optional*, defaults to `"500MB"`): + The maximum size of the dataset shards to be uploaded to the hub. If expressed as a string, needs to be digits followed by a unit + (like `"500MB"` or `"1GB"`). + num_shards (`Dict[str, int]`, *optional*): + Number of shards to write. By default, the number of shards depends on `max_shard_size`. + Use a dictionary to define a different num_shards for each split. + + + embed_external_files (`bool`, defaults to `True`): + Whether to embed file bytes in the shards. + In particular, this will do the following before the push for the fields of type: + + - [`Audio`] and [`Image`] removes local path information and embed file content in the Parquet files. + + Return: + huggingface_hub.CommitInfo + + Example: + + ```python + >>> dataset_dict.push_to_hub("/") + >>> dataset_dict.push_to_hub("/", private=True) + >>> dataset_dict.push_to_hub("/", max_shard_size="1GB") + >>> dataset_dict.push_to_hub("/", num_shards={"train": 1024, "test": 8}) + ``` + + If you want to add a new configuration (or subset) to a dataset (e.g. if the dataset has multiple tasks/versions/languages): + + ```python + >>> english_dataset.push_to_hub("/", "en") + >>> french_dataset.push_to_hub("/", "fr") + >>> # later + >>> english_dataset = load_dataset("/", "en") + >>> french_dataset = load_dataset("/", "fr") + ``` + """ + if num_shards is None: + num_shards = dict.fromkeys(self) + elif not isinstance(num_shards, dict): + raise ValueError( + "Please provide one `num_shards` per dataset in the dataset dictionary, e.g. {{'train': 128, 'test': 4}}" + ) + + self._check_values_type() + self._check_values_features() + total_uploaded_size = 0 + total_dataset_nbytes = 0 + info_to_dump: DatasetInfo = next(iter(self.values())).info.copy() + info_to_dump.config_name = config_name + info_to_dump.splits = SplitDict() + + for split in self.keys(): + if not re.match(_split_re, split): + raise ValueError(f"Split name should match '{_split_re}' but got '{split}'.") + + api = HfApi(endpoint=config.HF_ENDPOINT, token=token) + + repo_url = api.create_repo( + repo_id, + token=token, + repo_type="dataset", + private=private, + exist_ok=True, + ) + repo_id = repo_url.repo_id + + if revision is not None and not revision.startswith("refs/pr/"): + # We do not call create_branch for a PR reference: 400 Bad Request + api.create_branch( + repo_id, + branch=revision, + token=token, + repo_type="dataset", + exist_ok=True, + ) + + if not data_dir: + data_dir = config_name if config_name != "default" else "data" # for backward compatibility + + additions = [] + for split in self.keys(): + logger.info(f"Pushing split {split} to the Hub.") + # The split=key needs to be removed before merging + split_additions, uploaded_size, dataset_nbytes = self[split]._push_parquet_shards_to_hub( + repo_id, + data_dir=data_dir, + split=split, + token=token, + revision=revision, + create_pr=create_pr, + max_shard_size=max_shard_size, + num_shards=num_shards.get(split), + embed_external_files=embed_external_files, + ) + additions += split_additions + total_uploaded_size += uploaded_size + total_dataset_nbytes += dataset_nbytes + info_to_dump.splits[split] = SplitInfo(str(split), num_bytes=dataset_nbytes, num_examples=len(self[split])) + info_to_dump.download_checksums = None + info_to_dump.download_size = total_uploaded_size + info_to_dump.dataset_size = total_dataset_nbytes + info_to_dump.size_in_bytes = total_uploaded_size + total_dataset_nbytes + + # Check if the repo already has a README.md and/or a dataset_infos.json to update them with the new split info (size and pattern) + # and delete old split shards (if they exist) + repo_with_dataset_card, repo_with_dataset_infos = False, False + repo_splits: list[str] = [] # use a list to keep the order of the splits + deletions: list[CommitOperationDelete] = [] + repo_files_to_add = [addition.path_in_repo for addition in additions] + for repo_file in api.list_repo_tree( + repo_id=repo_id, + revision=revision, + repo_type="dataset", + token=token, + recursive=True, + ): + if not isinstance(repo_file, RepoFile): + continue + if repo_file.rfilename == config.REPOCARD_FILENAME: + repo_with_dataset_card = True + elif repo_file.rfilename == config.DATASETDICT_INFOS_FILENAME: + repo_with_dataset_infos = True + elif ( + repo_file.rfilename.startswith(tuple(f"{data_dir}/{split}-" for split in self.keys())) + and repo_file.rfilename not in repo_files_to_add + ): + deletions.append(CommitOperationDelete(path_in_repo=repo_file.rfilename)) + elif fnmatch.fnmatch( + repo_file.rfilename, + PUSH_TO_HUB_WITHOUT_METADATA_CONFIGS_SPLIT_PATTERN_SHARDED.replace("{split}", "*"), + ): + pattern = glob_pattern_to_regex(PUSH_TO_HUB_WITHOUT_METADATA_CONFIGS_SPLIT_PATTERN_SHARDED) + split_pattern_fields = string_to_dict(repo_file.rfilename, pattern) + assert split_pattern_fields is not None + repo_split = split_pattern_fields["split"] + if repo_split not in repo_splits: + repo_splits.append(repo_split) + + # get the info from the README to update them + if repo_with_dataset_card: + dataset_card_path = api.hf_hub_download( + repo_id, + config.REPOCARD_FILENAME, + repo_type="dataset", + revision=revision, + ) + dataset_card = DatasetCard.load(Path(dataset_card_path)) + dataset_card_data = dataset_card.data + metadata_configs = MetadataConfigs.from_dataset_card_data(dataset_card_data) + # get the deprecated dataset_infos.json to update them + elif repo_with_dataset_infos: + dataset_card = None + dataset_card_data = DatasetCardData() + metadata_configs = MetadataConfigs() + else: + dataset_card = None + dataset_card_data = DatasetCardData() + metadata_configs = MetadataConfigs() + # create the metadata configs if it was uploaded with push_to_hub before metadata configs existed + if not metadata_configs and repo_splits: + default_metadata_configs_to_dump = { + "data_files": [{"split": split, "path": f"data/{split}-*"} for split in repo_splits] + } + MetadataConfigs({"default": default_metadata_configs_to_dump}).to_dataset_card_data(dataset_card_data) + metadata_config_to_dump = { + "data_files": [{"split": split, "path": f"{data_dir}/{split}-*"} for split in self.keys()], + } + if set_default and config_name != "default": + if metadata_configs: + default_config_name = metadata_configs.get_default_config_name() + if default_config_name == "default": + raise ValueError( + "There exists a configuration named 'default'. To set a different configuration as default, " + "rename the 'default' one first." + ) + else: + _ = metadata_configs[default_config_name].pop("default") + metadata_config_to_dump["default"] = True + # push to the deprecated dataset_infos.json + if repo_with_dataset_infos: + dataset_infos_path = api.hf_hub_download( + repo_id, + config.DATASETDICT_INFOS_FILENAME, + repo_type="dataset", + revision=revision, + ) + with open(dataset_infos_path, encoding="utf-8") as f: + dataset_infos: dict = json.load(f) + dataset_infos[config_name] = asdict(info_to_dump) + additions.append( + CommitOperationAdd( + path_in_repo=config.DATASETDICT_INFOS_FILENAME, + path_or_fileobj=json.dumps(dataset_infos, indent=4).encode("utf-8"), + ) + ) + # push to README + DatasetInfosDict({config_name: info_to_dump}).to_dataset_card_data(dataset_card_data) + MetadataConfigs({config_name: metadata_config_to_dump}).to_dataset_card_data(dataset_card_data) + dataset_card = DatasetCard(f"---\n{dataset_card_data}\n---\n") if dataset_card is None else dataset_card + additions.append( + CommitOperationAdd( + path_in_repo=config.REPOCARD_FILENAME, + path_or_fileobj=str(dataset_card).encode(), + ) + ) + + commit_message = commit_message if commit_message is not None else "Upload dataset" + if len(additions) <= config.UPLOADS_MAX_NUMBER_PER_COMMIT: + commit_info = api.create_commit( + repo_id, + operations=additions + deletions, + commit_message=commit_message, + commit_description=commit_description, + token=token, + repo_type="dataset", + revision=revision, + create_pr=create_pr, + ) + else: + logger.info( + f"Number of files to upload is larger than {config.UPLOADS_MAX_NUMBER_PER_COMMIT}. Splitting the push into multiple commits." + ) + num_commits = math.ceil(len(additions) / config.UPLOADS_MAX_NUMBER_PER_COMMIT) + for i in range(0, num_commits): + operations = additions[ + i * config.UPLOADS_MAX_NUMBER_PER_COMMIT : (i + 1) * config.UPLOADS_MAX_NUMBER_PER_COMMIT + ] + (deletions if i == 0 else []) + commit_info = api.create_commit( + repo_id, + operations=operations, + commit_message=commit_message + f" (part {i:05d}-of-{num_commits:05d})", + commit_description=commit_description, + token=token, + repo_type="dataset", + revision=revision, + create_pr=create_pr, + ) + logger.info( + f"Commit #{i + 1} completed" + + (f" (still {num_commits - i - 1} to go)" if num_commits - i - 1 else "") + + "." + ) + return commit_info + + +class IterableDatasetDict(dict): + def __repr__(self): + repr = "\n".join([f"{k}: {v}" for k, v in self.items()]) + repr = re.sub(r"^", " " * 4, repr, 0, re.M) + return f"IterableDatasetDict({{\n{repr}\n}})" + + def with_format( + self, + type: Optional[str] = None, + ) -> "IterableDatasetDict": + """ + Return a dataset with the specified format. + + Args: + + type (`str`, *optional*): + Either output type selected in `[None, 'numpy', 'torch', 'tensorflow', 'jax', 'arrow', 'pandas', 'polars']`. + `None` means it returns python objects (default). + + Example: + + ```py + >>> from datasets import load_dataset + >>> from transformers import AutoTokenizer + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation", streaming=True) + >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-cased") + >>> ds = ds.map(lambda x: tokenizer(x['text'], truncation=True, padding=True), batched=True) + >>> ds = ds.with_format("torch") + >>> next(iter(ds)) + {'text': 'compassionately explores the seemingly irreconcilable situation between conservative christian parents and their estranged gay and lesbian children .', + 'label': tensor(1), + 'input_ids': tensor([ 101, 18027, 16310, 16001, 1103, 9321, 178, 11604, 7235, 6617, + 1742, 2165, 2820, 1206, 6588, 22572, 12937, 1811, 2153, 1105, + 1147, 12890, 19587, 6463, 1105, 15026, 1482, 119, 102, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0]), + 'token_type_ids': tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]), + 'attention_mask': tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])} + ``` + """ + return IterableDatasetDict({k: dataset.with_format(type=type) for k, dataset in self.items()}) + + def map( + self, + function: Optional[Callable] = None, + with_indices: bool = False, + with_split: bool = False, + input_columns: Optional[Union[str, list[str]]] = None, + batched: bool = False, + batch_size: int = 1000, + drop_last_batch: bool = False, + remove_columns: Optional[Union[str, list[str]]] = None, + fn_kwargs: Optional[dict] = None, + ) -> "IterableDatasetDict": + """ + Apply a function to all the examples in the iterable dataset (individually or in batches) and update them. + If your function returns a column that already exists, then it overwrites it. + The function is applied on-the-fly on the examples when iterating over the dataset. + The transformation is applied to all the datasets of the dataset dictionary. + + You can specify whether the function should be batched or not with the `batched` parameter: + + - If batched is `False`, then the function takes 1 example in and should return 1 example. + An example is a dictionary, e.g. `{"text": "Hello there !"}`. + - If batched is `True` and `batch_size` is 1, then the function takes a batch of 1 example as input and can return a batch with 1 or more examples. + A batch is a dictionary, e.g. a batch of 1 example is `{"text": ["Hello there !"]}`. + - If batched is `True` and `batch_size` is `n` > 1, then the function takes a batch of `n` examples as input and can return a batch with `n` examples, or with an arbitrary number of examples. + Note that the last batch may have less than `n` examples. + A batch is a dictionary, e.g. a batch of `n` examples is `{"text": ["Hello there !"] * n}`. + + If the function is asynchronous, then `map` will run your function in parallel, with up to one thousand simulatenous calls. + It is recommended to use a `asyncio.Semaphore` in your function if you want to set a maximum number of operations that can run at the same time. + + Args: + function (`Callable`, *optional*, defaults to `None`): + Function applied on-the-fly on the examples when you iterate on the dataset. + It must have one of the following signatures: + + - `function(example: Dict[str, Any]) -> Dict[str, Any]` if `batched=False` and `with_indices=False` + - `function(example: Dict[str, Any], idx: int) -> Dict[str, Any]` if `batched=False` and `with_indices=True` + - `function(batch: Dict[str, list]) -> Dict[str, list]` if `batched=True` and `with_indices=False` + - `function(batch: Dict[str, list], indices: list[int]) -> Dict[str, list]` if `batched=True` and `with_indices=True` + + For advanced usage, the function can also return a `pyarrow.Table`. + If the function is asynchronous, then `map` will run your function in parallel. + Moreover if your function returns nothing (`None`), then `map` will run your function and return the dataset unchanged. + If no function is provided, default to identity function: `lambda x: x`. + with_indices (`bool`, defaults to `False`): + Provide example indices to `function`. Note that in this case the signature of `function` should be `def function(example, idx[, rank]): ...`. + input_columns (`[Union[str, list[str]]]`, *optional*, defaults to `None`): + The columns to be passed into `function` + as positional arguments. If `None`, a dict mapping to all formatted columns is passed as one argument. + batched (`bool`, defaults to `False`): + Provide batch of examples to `function`. + batch_size (`int`, *optional*, defaults to `1000`): + Number of examples per batch provided to `function` if `batched=True`. + drop_last_batch (`bool`, defaults to `False`): + Whether a last batch smaller than the `batch_size` should be + dropped instead of being processed by the function. + remove_columns (`[list[str]]`, *optional*, defaults to `None`): + Remove a selection of columns while doing the mapping. + Columns will be removed before updating the examples with the output of `function`, i.e. if `function` is adding + columns with names in `remove_columns`, these columns will be kept. + fn_kwargs (`Dict`, *optional*, defaults to `None`): + Keyword arguments to be passed to `function` + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", streaming=True) + >>> def add_prefix(example): + ... example["text"] = "Review: " + example["text"] + ... return example + >>> ds = ds.map(add_prefix) + >>> next(iter(ds["train"])) + {'label': 1, + 'text': 'Review: the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'} + ``` + """ + + dataset_dict = {} + for split, dataset in self.items(): + if with_split: + function = bind(function, split) + + dataset_dict[split] = dataset.map( + function=function, + with_indices=with_indices, + input_columns=input_columns, + batched=batched, + batch_size=batch_size, + drop_last_batch=drop_last_batch, + remove_columns=remove_columns, + fn_kwargs=fn_kwargs, + ) + + if with_split: + function = function.func + + return IterableDatasetDict(dataset_dict) + + def filter( + self, + function: Optional[Callable] = None, + with_indices=False, + input_columns: Optional[Union[str, list[str]]] = None, + batched: bool = False, + batch_size: Optional[int] = 1000, + fn_kwargs: Optional[dict] = None, + ) -> "IterableDatasetDict": + """Apply a filter function to all the elements so that the dataset only includes examples according to the filter function. + The filtering is done on-the-fly when iterating over the dataset. + The filtering is applied to all the datasets of the dataset dictionary. + + Args: + function (`Callable`): + Callable with one of the following signatures: + + - `function(example: Dict[str, Any]) -> bool` if `with_indices=False, batched=False` + - `function(example: Dict[str, Any], indices: int) -> bool` if `with_indices=True, batched=False` + - `function(example: Dict[str, list]) -> list[bool]` if `with_indices=False, batched=True` + - `function(example: Dict[str, list], indices: list[int]) -> list[bool]` if `with_indices=True, batched=True` + + If no function is provided, defaults to an always True function: `lambda x: True`. + with_indices (`bool`, defaults to `False`): + Provide example indices to `function`. Note that in this case the signature of `function` should be `def function(example, idx): ...`. + input_columns (`str` or `list[str]`, *optional*): + The columns to be passed into `function` as + positional arguments. If `None`, a dict mapping to all formatted columns is passed as one argument. + batched (`bool`, defaults to `False`): + Provide batch of examples to `function` + batch_size (`int`, *optional*, defaults to `1000`): + Number of examples per batch provided to `function` if `batched=True`. + fn_kwargs (`Dict`, *optional*, defaults to `None`): + Keyword arguments to be passed to `function` + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", streaming=True) + >>> ds = ds.filter(lambda x: x["label"] == 0) + >>> list(ds["train"].take(3)) + [{'label': 0, 'text': 'Review: simplistic , silly and tedious .'}, + {'label': 0, + 'text': "Review: it's so laddish and juvenile , only teenage boys could possibly find it funny ."}, + {'label': 0, + 'text': 'Review: exploitative and largely devoid of the depth or sophistication that would make watching such a graphic treatment of the crimes bearable .'}] + ``` + """ + return IterableDatasetDict( + { + k: dataset.filter( + function=function, + with_indices=with_indices, + input_columns=input_columns, + batched=batched, + batch_size=batch_size, + fn_kwargs=fn_kwargs, + ) + for k, dataset in self.items() + } + ) + + def shuffle( + self, + seed=None, + generator: Optional[np.random.Generator] = None, + buffer_size: int = 1000, + ) -> "IterableDatasetDict": + """ + Randomly shuffles the elements of this dataset. + The shuffling is applied to all the datasets of the dataset dictionary. + + This dataset fills a buffer with buffer_size elements, then randomly samples elements from this buffer, + replacing the selected elements with new elements. For perfect shuffling, a buffer size greater than or + equal to the full size of the dataset is required. + + For instance, if your dataset contains 10,000 elements but `buffer_size` is set to 1000, then `shuffle` will + initially select a random element from only the first 1000 elements in the buffer. Once an element is + selected, its space in the buffer is replaced by the next (i.e. 1,001-st) element, + maintaining the 1000 element buffer. + + If the dataset is made of several shards, it also does `shuffle` the order of the shards. + However if the order has been fixed by using [`~datasets.IterableDataset.skip`] or [`~datasets.IterableDataset.take`] + then the order of the shards is kept unchanged. + + Args: + seed (`int`, *optional*, defaults to `None`): + Random seed that will be used to shuffle the dataset. + It is used to sample from the shuffle buffer and also to shuffle the data shards. + generator (`numpy.random.Generator`, *optional*): + Numpy random Generator to use to compute the permutation of the dataset rows. + If `generator=None` (default), uses `np.random.default_rng` (the default BitGenerator (PCG64) of NumPy). + buffer_size (`int`, defaults to `1000`): + Size of the buffer. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", streaming=True) + >>> list(ds["train"].take(3)) + [{'label': 1, + 'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'}, + {'label': 1, + 'text': 'the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson\'s expanded vision of j . r . r . tolkien\'s middle-earth .'}, + {'label': 1, 'text': 'effective but too-tepid biopic'}] + >>> ds = ds.shuffle(seed=42) + >>> list(ds["train"].take(3)) + [{'label': 1, + 'text': "a sports movie with action that's exciting on the field and a story you care about off it ."}, + {'label': 1, + 'text': 'at its best , the good girl is a refreshingly adult take on adultery . . .'}, + {'label': 1, + 'text': "sam jones became a very lucky filmmaker the day wilco got dropped from their record label , proving that one man's ruin may be another's fortune ."}] + ``` + """ + return IterableDatasetDict( + { + k: dataset.shuffle(seed=seed, generator=generator, buffer_size=buffer_size) + for k, dataset in self.items() + } + ) + + def rename_column(self, original_column_name: str, new_column_name: str) -> "IterableDatasetDict": + """ + Rename a column in the dataset, and move the features associated to the original column under the new column + name. + The renaming is applied to all the datasets of the dataset dictionary. + + Args: + original_column_name (`str`): + Name of the column to rename. + new_column_name (`str`): + New name for the column. + + Returns: + [`IterableDatasetDict`]: A copy of the dataset with a renamed column. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", streaming=True) + >>> ds = ds.rename_column("text", "movie_review") + >>> next(iter(ds["train"])) + {'label': 1, + 'movie_review': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'} + ``` + """ + return IterableDatasetDict( + { + k: dataset.rename_column( + original_column_name=original_column_name, + new_column_name=new_column_name, + ) + for k, dataset in self.items() + } + ) + + def rename_columns(self, column_mapping: dict[str, str]) -> "IterableDatasetDict": + """ + Rename several columns in the dataset, and move the features associated to the original columns under + the new column names. + The renaming is applied to all the datasets of the dataset dictionary. + + Args: + column_mapping (`Dict[str, str]`): + A mapping of columns to rename to their new names. + + Returns: + [`IterableDatasetDict`]: A copy of the dataset with renamed columns + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", streaming=True) + >>> ds = ds.rename_columns({"text": "movie_review", "label": "rating"}) + >>> next(iter(ds["train"])) + {'movie_review': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .', + 'rating': 1} + ``` + """ + return IterableDatasetDict( + {k: dataset.rename_columns(column_mapping=column_mapping) for k, dataset in self.items()} + ) + + def remove_columns(self, column_names: Union[str, list[str]]) -> "IterableDatasetDict": + """ + Remove one or several column(s) in the dataset and the features associated to them. + The removal is done on-the-fly on the examples when iterating over the dataset. + The removal is applied to all the datasets of the dataset dictionary. + + + Args: + column_names (`Union[str, list[str]]`): + Name of the column(s) to remove. + + Returns: + [`IterableDatasetDict`]: A copy of the dataset object without the columns to remove. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", streaming=True) + >>> ds = ds.remove_columns("label") + >>> next(iter(ds["train"])) + {'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'} + ``` + """ + return IterableDatasetDict({k: dataset.remove_columns(column_names) for k, dataset in self.items()}) + + def select_columns(self, column_names: Union[str, list[str]]) -> "IterableDatasetDict": + """Select one or several column(s) in the dataset and the features + associated to them. The selection is done on-the-fly on the examples + when iterating over the dataset. The selection is applied to all the + datasets of the dataset dictionary. + + + Args: + column_names (`Union[str, list[str]]`): + Name of the column(s) to keep. + + Returns: + [`IterableDatasetDict`]: A copy of the dataset object with only selected columns. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", streaming=True) + >>> ds = ds.select("text") + >>> next(iter(ds["train"])) + {'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'} + ``` + """ + return IterableDatasetDict({k: dataset.select_columns(column_names) for k, dataset in self.items()}) + + def cast_column(self, column: str, feature: FeatureType) -> "IterableDatasetDict": + """Cast column to feature for decoding. + The type casting is applied to all the datasets of the dataset dictionary. + + Args: + column (`str`): + Column name. + feature ([`Feature`]): + Target feature. + + Returns: + [`IterableDatasetDict`] + + Example: + + ```py + >>> from datasets import load_dataset, ClassLabel + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", streaming=True) + >>> ds["train"].features + {'label': ClassLabel(names=['neg', 'pos'], id=None), + 'text': Value(dtype='string', id=None)} + >>> ds = ds.cast_column('label', ClassLabel(names=['bad', 'good'])) + >>> ds["train"].features + {'label': ClassLabel(names=['bad', 'good'], id=None), + 'text': Value(dtype='string', id=None)} + ``` + """ + return IterableDatasetDict( + {k: dataset.cast_column(column=column, feature=feature) for k, dataset in self.items()} + ) + + def cast( + self, + features: Features, + ) -> "IterableDatasetDict": + """ + Cast the dataset to a new set of features. + The type casting is applied to all the datasets of the dataset dictionary. + + Args: + features (`Features`): + New features to cast the dataset to. + The name of the fields in the features must match the current column names. + The type of the data must also be convertible from one type to the other. + For non-trivial conversion, e.g. `string` <-> `ClassLabel` you should use [`map`] to update the Dataset. + + Returns: + [`IterableDatasetDict`]: A copy of the dataset with casted features. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", streaming=True) + >>> ds["train"].features + {'label': ClassLabel(names=['neg', 'pos'], id=None), + 'text': Value(dtype='string', id=None)} + >>> new_features = ds["train"].features.copy() + >>> new_features['label'] = ClassLabel(names=['bad', 'good']) + >>> new_features['text'] = Value('large_string') + >>> ds = ds.cast(new_features) + >>> ds["train"].features + {'label': ClassLabel(names=['bad', 'good'], id=None), + 'text': Value(dtype='large_string', id=None)} + ``` + """ + return IterableDatasetDict({k: dataset.cast(features=features) for k, dataset in self.items()}) diff --git a/lib/python3.12/site-packages/datasets/distributed.py b/lib/python3.12/site-packages/datasets/distributed.py new file mode 100644 index 0000000000000000000000000000000000000000..4697948f342a81d489084b068758327156a89b8b --- /dev/null +++ b/lib/python3.12/site-packages/datasets/distributed.py @@ -0,0 +1,39 @@ +from typing import TypeVar + +from .arrow_dataset import Dataset, _split_by_node_map_style_dataset +from .iterable_dataset import IterableDataset, _split_by_node_iterable_dataset + + +DatasetType = TypeVar("DatasetType", Dataset, IterableDataset) + + +def split_dataset_by_node(dataset: DatasetType, rank: int, world_size: int) -> DatasetType: + """ + Split a dataset for the node at rank `rank` in a pool of nodes of size `world_size`. + + For map-style datasets: + + Each node is assigned a chunk of data, e.g. rank 0 is given the first chunk of the dataset. + To maximize data loading throughput, chunks are made of contiguous data on disk if possible. + + For iterable datasets: + + If the dataset has a number of shards that is a factor of `world_size` (i.e. if `dataset.num_shards % world_size == 0`), + then the shards are evenly assigned across the nodes, which is the most optimized. + Otherwise, each node keeps 1 example out of `world_size`, skipping the other examples. + + Args: + dataset ([`Dataset`] or [`IterableDataset`]): + The dataset to split by node. + rank (`int`): + Rank of the current node. + world_size (`int`): + Total number of nodes. + + Returns: + [`Dataset`] or [`IterableDataset`]: The dataset to be used on the node at rank `rank`. + """ + if isinstance(dataset, Dataset): + return _split_by_node_map_style_dataset(dataset, rank=rank, world_size=world_size) + else: + return _split_by_node_iterable_dataset(dataset, rank=rank, world_size=world_size) diff --git a/lib/python3.12/site-packages/datasets/exceptions.py b/lib/python3.12/site-packages/datasets/exceptions.py new file mode 100644 index 0000000000000000000000000000000000000000..7c1724fb572dc98a2b65ad07f0a1a08c5d2a32e5 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/exceptions.py @@ -0,0 +1,119 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2023 The HuggingFace Authors. +from typing import Any, Optional, Union + +from huggingface_hub import HfFileSystem + +from . import config +from .table import CastError +from .utils.track import TrackedIterableFromGenerator, tracked_list, tracked_str + + +class DatasetsError(Exception): + """Base class for exceptions in this library.""" + + +class DefunctDatasetError(DatasetsError): + """The dataset has been defunct.""" + + +class FileNotFoundDatasetsError(DatasetsError, FileNotFoundError): + """FileNotFoundError raised by this library.""" + + +class DataFilesNotFoundError(FileNotFoundDatasetsError): + """No (supported) data files found.""" + + +class DatasetNotFoundError(FileNotFoundDatasetsError): + """Dataset not found. + + Raised when trying to access: + - a missing dataset, or + - a private/gated dataset and the user is not authenticated. + """ + + +class DatasetBuildError(DatasetsError): + pass + + +class ManualDownloadError(DatasetBuildError): + pass + + +class FileFormatError(DatasetBuildError): + pass + + +class DatasetGenerationError(DatasetBuildError): + pass + + +class DatasetGenerationCastError(DatasetGenerationError): + @classmethod + def from_cast_error( + cls, + cast_error: CastError, + builder_name: str, + gen_kwargs: dict[str, Any], + token: Optional[Union[bool, str]], + ) -> "DatasetGenerationCastError": + explanation_message = ( + f"\n\nAll the data files must have the same columns, but at some point {cast_error.details()}" + ) + formatted_tracked_gen_kwargs: list[str] = [] + for gen_kwarg in gen_kwargs.values(): + if not isinstance(gen_kwarg, (tracked_str, tracked_list, TrackedIterableFromGenerator)): + continue + while ( + isinstance(gen_kwarg, (tracked_list, TrackedIterableFromGenerator)) and gen_kwarg.last_item is not None + ): + gen_kwarg = gen_kwarg.last_item + if isinstance(gen_kwarg, tracked_str): + gen_kwarg = gen_kwarg.get_origin() + if isinstance(gen_kwarg, str) and gen_kwarg.startswith("hf://"): + resolved_path = HfFileSystem(endpoint=config.HF_ENDPOINT, token=token).resolve_path(gen_kwarg) + gen_kwarg = "hf://" + resolved_path.unresolve() + if "@" + resolved_path.revision in gen_kwarg: + gen_kwarg = ( + gen_kwarg.replace("@" + resolved_path.revision, "", 1) + + f" (at revision {resolved_path.revision})" + ) + formatted_tracked_gen_kwargs.append(str(gen_kwarg)) + if formatted_tracked_gen_kwargs: + explanation_message += f"\n\nThis happened while the {builder_name} dataset builder was generating data using\n\n{', '.join(formatted_tracked_gen_kwargs)}" + help_message = "\n\nPlease either edit the data files to have matching columns, or separate them into different configurations (see docs at https://hf.co/docs/hub/datasets-manual-configuration#multiple-configurations)" + return cls("An error occurred while generating the dataset" + explanation_message + help_message) + + +class ChecksumVerificationError(DatasetsError): + """Error raised during checksums verifications of downloaded files.""" + + +class UnexpectedDownloadedFileError(ChecksumVerificationError): + """Some downloaded files were not expected.""" + + +class ExpectedMoreDownloadedFilesError(ChecksumVerificationError): + """Some files were supposed to be downloaded but were not.""" + + +class NonMatchingChecksumError(ChecksumVerificationError): + """The downloaded file checksum don't match the expected checksum.""" + + +class SplitsVerificationError(DatasetsError): + """Error raised during splits verifications.""" + + +class UnexpectedSplitsError(SplitsVerificationError): + """The expected splits of the downloaded file is missing.""" + + +class ExpectedMoreSplitsError(SplitsVerificationError): + """Some recorded splits are missing.""" + + +class NonMatchingSplitsSizesError(SplitsVerificationError): + """The splits sizes don't match the expected splits sizes.""" diff --git a/lib/python3.12/site-packages/datasets/fingerprint.py b/lib/python3.12/site-packages/datasets/fingerprint.py new file mode 100644 index 0000000000000000000000000000000000000000..93724bb68490b31d633f7a106cd2ee9e1f122dff --- /dev/null +++ b/lib/python3.12/site-packages/datasets/fingerprint.py @@ -0,0 +1,454 @@ +import inspect +import os +import random +import shutil +import tempfile +import weakref +from functools import wraps +from pathlib import Path +from typing import TYPE_CHECKING, Any, Callable, Optional, Union + +import numpy as np +import xxhash + +from . import config +from .naming import INVALID_WINDOWS_CHARACTERS_IN_PATH +from .utils._dill import dumps +from .utils.logging import get_logger + + +if TYPE_CHECKING: + from .arrow_dataset import Dataset + + +logger = get_logger(__name__) + + +# Fingerprinting allows to have one deterministic fingerprint per dataset state. +# A dataset fingerprint is updated after each transform. +# Re-running the same transforms on a dataset in a different session results in the same fingerprint. +# This is possible thanks to a custom hashing function that works with most python objects. + +# Fingerprinting is the main mechanism that enables caching. +# The caching mechanism allows to reload an existing cache file if it's already been computed. + + +################# +# Caching +################# + +_CACHING_ENABLED = True +_TEMP_DIR_FOR_TEMP_CACHE_FILES: Optional["_TempCacheDir"] = None +_DATASETS_WITH_TABLE_IN_TEMP_DIR: Optional[weakref.WeakSet] = None + + +class _TempCacheDir: + """ + A temporary directory for storing cached Arrow files with a cleanup that frees references to the Arrow files + before deleting the directory itself to avoid permission errors on Windows. + """ + + def __init__(self): + self.name = tempfile.mkdtemp(prefix=config.TEMP_CACHE_DIR_PREFIX) + self._finalizer = weakref.finalize(self, self._cleanup) + + def _cleanup(self): + for dset in get_datasets_with_cache_file_in_temp_dir(): + dset.__del__() + if os.path.exists(self.name): + try: + shutil.rmtree(self.name) + except Exception as e: + raise OSError( + f"An error occured while trying to delete temporary cache directory {self.name}. Please delete it manually." + ) from e + + def cleanup(self): + if self._finalizer.detach(): + self._cleanup() + + +def maybe_register_dataset_for_temp_dir_deletion(dataset): + """ + This function registers the datasets that have cache files in _TEMP_DIR_FOR_TEMP_CACHE_FILES in order + to properly delete them before deleting the temporary directory. + The temporary directory _TEMP_DIR_FOR_TEMP_CACHE_FILES is used when caching is disabled. + """ + if _TEMP_DIR_FOR_TEMP_CACHE_FILES is None: + return + + global _DATASETS_WITH_TABLE_IN_TEMP_DIR + if _DATASETS_WITH_TABLE_IN_TEMP_DIR is None: + _DATASETS_WITH_TABLE_IN_TEMP_DIR = weakref.WeakSet() + if any( + Path(_TEMP_DIR_FOR_TEMP_CACHE_FILES.name) in Path(cache_file["filename"]).parents + for cache_file in dataset.cache_files + ): + _DATASETS_WITH_TABLE_IN_TEMP_DIR.add(dataset) + + +def get_datasets_with_cache_file_in_temp_dir(): + return list(_DATASETS_WITH_TABLE_IN_TEMP_DIR) if _DATASETS_WITH_TABLE_IN_TEMP_DIR is not None else [] + + +def enable_caching(): + """ + When applying transforms on a dataset, the data are stored in cache files. + The caching mechanism allows to reload an existing cache file if it's already been computed. + + Reloading a dataset is possible since the cache files are named using the dataset fingerprint, which is updated + after each transform. + + If disabled, the library will no longer reload cached datasets files when applying transforms to the datasets. + More precisely, if the caching is disabled: + - cache files are always recreated + - cache files are written to a temporary directory that is deleted when session closes + - cache files are named using a random hash instead of the dataset fingerprint + - use [`~datasets.Dataset.save_to_disk`] to save a transformed dataset or it will be deleted when session closes + - caching doesn't affect [`~datasets.load_dataset`]. If you want to regenerate a dataset from scratch you should use + the `download_mode` parameter in [`~datasets.load_dataset`]. + """ + global _CACHING_ENABLED + _CACHING_ENABLED = True + + +def disable_caching(): + """ + When applying transforms on a dataset, the data are stored in cache files. + The caching mechanism allows to reload an existing cache file if it's already been computed. + + Reloading a dataset is possible since the cache files are named using the dataset fingerprint, which is updated + after each transform. + + If disabled, the library will no longer reload cached datasets files when applying transforms to the datasets. + More precisely, if the caching is disabled: + - cache files are always recreated + - cache files are written to a temporary directory that is deleted when session closes + - cache files are named using a random hash instead of the dataset fingerprint + - use [`~datasets.Dataset.save_to_disk`] to save a transformed dataset or it will be deleted when session closes + - caching doesn't affect [`~datasets.load_dataset`]. If you want to regenerate a dataset from scratch you should use + the `download_mode` parameter in [`~datasets.load_dataset`]. + """ + global _CACHING_ENABLED + _CACHING_ENABLED = False + + +def is_caching_enabled() -> bool: + """ + When applying transforms on a dataset, the data are stored in cache files. + The caching mechanism allows to reload an existing cache file if it's already been computed. + + Reloading a dataset is possible since the cache files are named using the dataset fingerprint, which is updated + after each transform. + + If disabled, the library will no longer reload cached datasets files when applying transforms to the datasets. + More precisely, if the caching is disabled: + - cache files are always recreated + - cache files are written to a temporary directory that is deleted when session closes + - cache files are named using a random hash instead of the dataset fingerprint + - use [`~datasets.Dataset.save_to_disk`]] to save a transformed dataset or it will be deleted when session closes + - caching doesn't affect [`~datasets.load_dataset`]. If you want to regenerate a dataset from scratch you should use + the `download_mode` parameter in [`~datasets.load_dataset`]. + """ + global _CACHING_ENABLED + return bool(_CACHING_ENABLED) + + +def get_temporary_cache_files_directory() -> str: + """Return a directory that is deleted when session closes.""" + global _TEMP_DIR_FOR_TEMP_CACHE_FILES + if _TEMP_DIR_FOR_TEMP_CACHE_FILES is None: + _TEMP_DIR_FOR_TEMP_CACHE_FILES = _TempCacheDir() + return _TEMP_DIR_FOR_TEMP_CACHE_FILES.name + + +################# +# Hashing +################# + + +class Hasher: + """Hasher that accepts python objects as inputs.""" + + dispatch: dict = {} + + def __init__(self): + self.m = xxhash.xxh64() + + @classmethod + def hash_bytes(cls, value: Union[bytes, list[bytes]]) -> str: + value = [value] if isinstance(value, bytes) else value + m = xxhash.xxh64() + for x in value: + m.update(x) + return m.hexdigest() + + @classmethod + def hash(cls, value: Any) -> str: + return cls.hash_bytes(dumps(value)) + + def update(self, value: Any) -> None: + header_for_update = f"=={type(value)}==" + value_for_update = self.hash(value) + self.m.update(header_for_update.encode("utf8")) + self.m.update(value_for_update.encode("utf-8")) + + def hexdigest(self) -> str: + return self.m.hexdigest() + + +################# +# Fingerprinting +################# + +fingerprint_rng = random.Random() +# we show a warning only once when fingerprinting fails to avoid spam +fingerprint_warnings: dict[str, bool] = {} + + +def generate_fingerprint(dataset: "Dataset") -> str: + state = dataset.__dict__ + hasher = Hasher() + for key in sorted(state): + if key == "_fingerprint": + continue + hasher.update(key) + hasher.update(state[key]) + # hash data files last modification timestamps as well + for cache_file in dataset.cache_files: + hasher.update(os.path.getmtime(cache_file["filename"])) + return hasher.hexdigest() + + +def generate_random_fingerprint(nbits: int = 64) -> str: + return f"{fingerprint_rng.getrandbits(nbits):0{nbits // 4}x}" + + +def update_fingerprint(fingerprint, transform, transform_args): + global fingerprint_warnings + hasher = Hasher() + hasher.update(fingerprint) + try: + hasher.update(transform) + except: # noqa various errors might raise here from pickle or dill + if _CACHING_ENABLED: + if not fingerprint_warnings.get("update_fingerprint_transform_hash_failed", False): + logger.warning( + f"Transform {transform} couldn't be hashed properly, a random hash was used instead. " + "Make sure your transforms and parameters are serializable with pickle or dill for the dataset fingerprinting and caching to work. " + "If you reuse this transform, the caching mechanism will consider it to be different from the previous calls and recompute everything. " + "This warning is only showed once. Subsequent hashing failures won't be showed." + ) + fingerprint_warnings["update_fingerprint_transform_hash_failed"] = True + else: + logger.info(f"Transform {transform} couldn't be hashed properly, a random hash was used instead.") + else: + logger.info( + f"Transform {transform} couldn't be hashed properly, a random hash was used instead. This doesn't affect caching since it's disabled." + ) + + return generate_random_fingerprint() + for key in sorted(transform_args): + hasher.update(key) + try: + hasher.update(transform_args[key]) + except: # noqa various errors might raise here from pickle or dill + if _CACHING_ENABLED: + if not fingerprint_warnings.get("update_fingerprint_transform_hash_failed", False): + logger.warning( + f"Parameter '{key}'={transform_args[key]} of the transform {transform} couldn't be hashed properly, a random hash was used instead. " + "Make sure your transforms and parameters are serializable with pickle or dill for the dataset fingerprinting and caching to work. " + "If you reuse this transform, the caching mechanism will consider it to be different from the previous calls and recompute everything. " + "This warning is only showed once. Subsequent hashing failures won't be showed." + ) + fingerprint_warnings["update_fingerprint_transform_hash_failed"] = True + else: + logger.info( + f"Parameter '{key}'={transform_args[key]} of the transform {transform} couldn't be hashed properly, a random hash was used instead." + ) + else: + logger.info( + f"Parameter '{key}'={transform_args[key]} of the transform {transform} couldn't be hashed properly, a random hash was used instead. This doesn't affect caching since it's disabled." + ) + return generate_random_fingerprint() + return hasher.hexdigest() + + +def validate_fingerprint(fingerprint: str, max_length=64): + """ + Make sure the fingerprint is a non-empty string that is not longer that max_length=64 by default, + so that the fingerprint can be used to name cache files without issues. + """ + if not isinstance(fingerprint, str) or not fingerprint: + raise ValueError(f"Invalid fingerprint '{fingerprint}': it should be a non-empty string.") + for invalid_char in INVALID_WINDOWS_CHARACTERS_IN_PATH: + if invalid_char in fingerprint: + raise ValueError( + f"Invalid fingerprint. Bad characters from black list '{INVALID_WINDOWS_CHARACTERS_IN_PATH}' found in '{fingerprint}'. " + f"They could create issues when creating cache files." + ) + if len(fingerprint) > max_length: + raise ValueError( + f"Invalid fingerprint. Maximum lenth is {max_length} but '{fingerprint}' has length {len(fingerprint)}." + "It could create issues when creating cache files." + ) + + +def format_transform_for_fingerprint(func: Callable, version: Optional[str] = None) -> str: + """ + Format a transform to the format that will be used to update the fingerprint. + """ + transform = f"{func.__module__}.{func.__qualname__}" + if version is not None: + transform += f"@{version}" + return transform + + +def format_kwargs_for_fingerprint( + func: Callable, + args: tuple, + kwargs: dict[str, Any], + use_kwargs: Optional[list[str]] = None, + ignore_kwargs: Optional[list[str]] = None, + randomized_function: bool = False, +) -> dict[str, Any]: + """ + Format the kwargs of a transform to the format that will be used to update the fingerprint. + """ + kwargs_for_fingerprint = kwargs.copy() + if args: + params = [p.name for p in inspect.signature(func).parameters.values() if p != p.VAR_KEYWORD] + args = args[1:] # assume the first argument is the dataset + params = params[1:] + kwargs_for_fingerprint.update(zip(params, args)) + else: + del kwargs_for_fingerprint[ + next(iter(inspect.signature(func).parameters)) + ] # assume the first key is the dataset + + # keep the right kwargs to be hashed to generate the fingerprint + + if use_kwargs: + kwargs_for_fingerprint = {k: v for k, v in kwargs_for_fingerprint.items() if k in use_kwargs} + if ignore_kwargs: + kwargs_for_fingerprint = {k: v for k, v in kwargs_for_fingerprint.items() if k not in ignore_kwargs} + if randomized_function: # randomized functions have `seed` and `generator` parameters + if kwargs_for_fingerprint.get("seed") is None and kwargs_for_fingerprint.get("generator") is None: + _, seed, pos, *_ = np.random.get_state() + seed = seed[pos] if pos < 624 else seed[0] + kwargs_for_fingerprint["generator"] = np.random.default_rng(seed) + + # remove kwargs that are the default values + + default_values = { + p.name: p.default for p in inspect.signature(func).parameters.values() if p.default != inspect._empty + } + for default_varname, default_value in default_values.items(): + if default_varname in kwargs_for_fingerprint and kwargs_for_fingerprint[default_varname] == default_value: + kwargs_for_fingerprint.pop(default_varname) + return kwargs_for_fingerprint + + +def fingerprint_transform( + inplace: bool, + use_kwargs: Optional[list[str]] = None, + ignore_kwargs: Optional[list[str]] = None, + fingerprint_names: Optional[list[str]] = None, + randomized_function: bool = False, + version: Optional[str] = None, +): + """ + Wrapper for dataset transforms to update the dataset fingerprint using ``update_fingerprint`` + Args: + inplace (:obj:`bool`): If inplace is True, the fingerprint of the dataset is updated inplace. + Otherwise, a parameter "new_fingerprint" is passed to the wrapped method that should take care of + setting the fingerprint of the returned Dataset. + use_kwargs (:obj:`List[str]`, optional): optional white list of argument names to take into account + to update the fingerprint to the wrapped method that should take care of + setting the fingerprint of the returned Dataset. By default all the arguments are used. + ignore_kwargs (:obj:`List[str]`, optional): optional black list of argument names to take into account + to update the fingerprint. Note that ignore_kwargs prevails on use_kwargs. + fingerprint_names (:obj:`List[str]`, optional, defaults to ["new_fingerprint"]): + If the dataset transforms is not inplace and returns a DatasetDict, then it can require + several fingerprints (one per dataset in the DatasetDict). By specifying fingerprint_names, + one fingerprint named after each element of fingerprint_names is going to be passed. + randomized_function (:obj:`bool`, defaults to False): If the dataset transform is random and has + optional parameters "seed" and "generator", then you can set randomized_function to True. + This way, even if users set "seed" and "generator" to None, then the fingerprint is + going to be randomly generated depending on numpy's current state. In this case, the + generator is set to np.random.default_rng(np.random.get_state()[1][0]). + version (:obj:`str`, optional): version of the transform. The version is taken into account when + computing the fingerprint. If a datase transform changes (or at least if the output data + that are cached changes), then one should increase the version. If the version stays the + same, then old cached data could be reused that are not compatible with the new transform. + It should be in the format "MAJOR.MINOR.PATCH". + """ + + if use_kwargs is not None and not isinstance(use_kwargs, list): + raise ValueError(f"use_kwargs is supposed to be a list, not {type(use_kwargs)}") + + if ignore_kwargs is not None and not isinstance(ignore_kwargs, list): + raise ValueError(f"ignore_kwargs is supposed to be a list, not {type(use_kwargs)}") + + if inplace and fingerprint_names: + raise ValueError("fingerprint_names are only used when inplace is False") + + fingerprint_names = fingerprint_names if fingerprint_names is not None else ["new_fingerprint"] + + def _fingerprint(func): + if not inplace and not all(name in func.__code__.co_varnames for name in fingerprint_names): + raise ValueError(f"function {func} is missing parameters {fingerprint_names} in signature") + + if randomized_function: # randomized function have seed and generator parameters + if "seed" not in func.__code__.co_varnames: + raise ValueError(f"'seed' must be in {func}'s signature") + if "generator" not in func.__code__.co_varnames: + raise ValueError(f"'generator' must be in {func}'s signature") + # this call has to be outside the wrapper or since __qualname__ changes in multiprocessing + transform = format_transform_for_fingerprint(func, version=version) + + @wraps(func) + def wrapper(*args, **kwargs): + kwargs_for_fingerprint = format_kwargs_for_fingerprint( + func, + args, + kwargs, + use_kwargs=use_kwargs, + ignore_kwargs=ignore_kwargs, + randomized_function=randomized_function, + ) + + if args: + dataset: Dataset = args[0] + args = args[1:] + else: + dataset: Dataset = kwargs.pop(next(iter(inspect.signature(func).parameters))) + + # compute new_fingerprint and add it to the args of not in-place transforms + if inplace: + new_fingerprint = update_fingerprint(dataset._fingerprint, transform, kwargs_for_fingerprint) + else: + for fingerprint_name in fingerprint_names: # transforms like `train_test_split` have several hashes + if kwargs.get(fingerprint_name) is None: + kwargs_for_fingerprint["fingerprint_name"] = fingerprint_name + kwargs[fingerprint_name] = update_fingerprint( + dataset._fingerprint, transform, kwargs_for_fingerprint + ) + else: + validate_fingerprint(kwargs[fingerprint_name]) + + # Call actual function + + out = func(dataset, *args, **kwargs) + + # Update fingerprint of in-place transforms + update in-place history of transforms + + if inplace: # update after calling func so that the fingerprint doesn't change if the function fails + dataset._fingerprint = new_fingerprint + + return out + + wrapper._decorator_name_ = "fingerprint" + return wrapper + + return _fingerprint diff --git a/lib/python3.12/site-packages/datasets/hub.py b/lib/python3.12/site-packages/datasets/hub.py new file mode 100644 index 0000000000000000000000000000000000000000..06d4f0ddb95ab6765167798f1d980a168c246bd8 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/hub.py @@ -0,0 +1,230 @@ +import time +from itertools import chain +from typing import Optional, Union + +from huggingface_hub import ( + CommitInfo, + CommitOperationAdd, + CommitOperationDelete, + DatasetCard, + DatasetCardData, + HfApi, + HfFileSystem, +) +from huggingface_hub.utils import HfHubHTTPError + +import datasets.config +from datasets.info import DatasetInfosDict +from datasets.inspect import get_dataset_config_names, get_dataset_default_config_name +from datasets.load import load_dataset, load_dataset_builder +from datasets.utils.metadata import MetadataConfigs + + +def convert_to_parquet( + repo_id: str, + revision: Optional[str] = None, + token: Optional[Union[bool, str]] = None, + trust_remote_code: Optional[bool] = None, +) -> CommitInfo: + """Convert Hub [script-based dataset](dataset_script) to Parquet [data-only dataset](repository_structure), so that + the dataset viewer will be supported. + + This function: + - makes a copy of the script on the "main" branch into a dedicated branch called "script" (if it does not already exist) + - creates a pull request to the Hub dataset to convert it to Parquet files (and deletes the script from the main branch) + + If in the future you need to recreate the Parquet files from the "script" branch, pass the `revision="script"` argument. + + Note that you should pass the `trust_remote_code=True` argument only if you trust the remote code to be executed locally on your machine. + + Args: + repo_id (`str`): ID of the source Hub dataset repository, in the following format: `/` or + `/`. + revision (`str`, *optional*): Branch of the source Hub dataset repository. Defaults to the `"main"` branch. + token (`bool` or `str`, *optional*): Authentication token for the Hugging Face Hub. + trust_remote_code (`bool`, defaults to `False`): Whether you trust the remote code of the Hub script-based + dataset to be executed locally on your machine. This option should only be set to `True` for repositories + where you have read the code and which you trust. + + + + `trust_remote_code` defaults to `False` if not specified. + + + + Returns: + `huggingface_hub.CommitInfo` + """ + print(f"{repo_id}") + configs = get_dataset_config_names(repo_id, token=token, revision=revision, trust_remote_code=trust_remote_code) + print(f"{configs = }") + default_config = get_dataset_default_config_name( + repo_id, token=token, revision=revision, trust_remote_code=trust_remote_code + ) + print(f"{default_config = }") + if default_config: + config = default_config + configs.remove(default_config) + else: + config = configs.pop(0) + print(f"{config = }") + dataset = load_dataset(repo_id, config, revision=revision, trust_remote_code=trust_remote_code) + commit_info = dataset.push_to_hub( + repo_id, + config_name=config, + commit_message="Convert dataset to Parquet", + commit_description="Convert dataset to Parquet.", + create_pr=True, + token=token, + set_default=default_config is not None, + ) + time.sleep(5) + pr_revision, pr_url = commit_info.pr_revision, commit_info.pr_url + for config in configs: + print(f"{config = }") + dataset = load_dataset(repo_id, config, revision=revision, trust_remote_code=trust_remote_code) + dataset.push_to_hub( + repo_id, + config_name=config, + commit_message=f"Add '{config}' config data files", + revision=pr_revision, + token=token, + ) + time.sleep(5) + _delete_files(repo_id, revision=pr_revision, token=token) + if not revision: + api = HfApi(endpoint=datasets.config.HF_ENDPOINT, token=token) + try: + api.create_branch(repo_id, branch="script", repo_type="dataset", token=token, exist_ok=True) + except HfHubHTTPError: + pass + print(f"You can find your PR to convert the dataset to Parquet at: {pr_url}") + return commit_info + + +def delete_from_hub( + repo_id: str, + config_name: str, + revision: Optional[str] = None, + token: Optional[Union[bool, str]] = None, +) -> CommitInfo: + """Delete a dataset configuration from a [data-only dataset](repository_structure) on the Hub. + + Args: + repo_id (`str`): ID of the Hub dataset repository, in the following format: `/` or + `/`. + config_name (`str`): Name of the dataset configuration. + revision (`str`, *optional*): Branch to delete the configuration from. Defaults to the `"main"` branch. + token (`bool` or `str`, *optional*): Authentication token for the Hugging Face Hub. + + Returns: + `huggingface_hub.CommitInfo` + """ + operations = [] + # data_files + fs = HfFileSystem(endpoint=datasets.config.HF_ENDPOINT, token=token) + builder = load_dataset_builder(repo_id, config_name, revision=revision, token=token, trust_remote_code=False) + for data_file in chain(*builder.config.data_files.values()): + data_file_resolved_path = fs.resolve_path(data_file) + if data_file_resolved_path.repo_id == repo_id: + operations.append(CommitOperationDelete(path_in_repo=data_file_resolved_path.path_in_repo)) + # README.md + dataset_card = DatasetCard.load(repo_id) + # config_names + if dataset_card.data.get("config_names", None) and config_name in dataset_card.data["config_names"]: + dataset_card.data["config_names"].remove(config_name) + # metadata_configs + metadata_configs = MetadataConfigs.from_dataset_card_data(dataset_card.data) + if metadata_configs: + _ = metadata_configs.pop(config_name, None) + dataset_card_data = DatasetCardData() + metadata_configs.to_dataset_card_data(dataset_card_data) + if datasets.config.METADATA_CONFIGS_FIELD in dataset_card_data: + dataset_card.data[datasets.config.METADATA_CONFIGS_FIELD] = dataset_card_data[ + datasets.config.METADATA_CONFIGS_FIELD + ] + else: + _ = dataset_card.data.pop(datasets.config.METADATA_CONFIGS_FIELD, None) + # dataset_info + dataset_infos: DatasetInfosDict = DatasetInfosDict.from_dataset_card_data(dataset_card.data) + if dataset_infos: + _ = dataset_infos.pop(config_name, None) + dataset_card_data = DatasetCardData() + dataset_infos.to_dataset_card_data(dataset_card_data) + if "dataset_info" in dataset_card_data: + dataset_card.data["dataset_info"] = dataset_card_data["dataset_info"] + else: + _ = dataset_card.data.pop("dataset_info", None) + # Commit + operations.append( + CommitOperationAdd(path_in_repo=datasets.config.REPOCARD_FILENAME, path_or_fileobj=str(dataset_card).encode()) + ) + api = HfApi(endpoint=datasets.config.HF_ENDPOINT, token=token) + commit_info = api.create_commit( + repo_id, + operations=operations, + commit_message=f"Delete '{config_name}' config", + commit_description=f"Delete '{config_name}' config.", + token=token, + repo_type="dataset", + revision=revision, + create_pr=True, + ) + print(f"You can find your PR to delete the dataset config at: {commit_info.pr_url}") + return commit_info + + +def _delete_files(dataset_id, revision=None, token=None): + dataset_name = dataset_id.split("/")[-1] + hf_api = HfApi(endpoint=datasets.config.HF_ENDPOINT, token=token) + repo_files = hf_api.list_repo_files( + dataset_id, + repo_type="dataset", + ) + if repo_files: + legacy_json_file = [] + python_files = [] + data_files = [] + for filename in repo_files: + if filename in {".gitattributes", "README.md"}: + continue + elif filename == f"{dataset_name}.py": + hf_api.delete_file( + filename, + dataset_id, + repo_type="dataset", + revision=revision, + commit_message="Delete loading script", + ) + elif filename == "dataset_infos.json": + legacy_json_file.append(filename) + elif filename.endswith(".py"): + python_files.append(filename) + else: + data_files.append(filename) + if legacy_json_file: + hf_api.delete_file( + "dataset_infos.json", + dataset_id, + repo_type="dataset", + revision=revision, + commit_message="Delete legacy dataset_infos.json", + ) + if python_files: + for filename in python_files: + hf_api.delete_file( + filename, + dataset_id, + repo_type="dataset", + revision=revision, + commit_message="Delete loading script auxiliary file", + ) + if data_files: + for filename in data_files: + hf_api.delete_file( + filename, + dataset_id, + repo_type="dataset", + revision=revision, + commit_message="Delete data file", + ) diff --git a/lib/python3.12/site-packages/datasets/info.py b/lib/python3.12/site-packages/datasets/info.py new file mode 100644 index 0000000000000000000000000000000000000000..1217c58bb396b539e96217b106b93caef14ee643 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/info.py @@ -0,0 +1,430 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +"""DatasetInfo record information we know about a dataset. + +This includes things that we know about the dataset statically, i.e.: + - description + - canonical location + - does it have validation and tests splits + - size + - etc. + +This also includes the things that can and should be computed once we've +processed the dataset as well: + - number of examples (in each split) + - etc. +""" + +import copy +import dataclasses +import json +import os +import posixpath +from dataclasses import dataclass +from pathlib import Path +from typing import ClassVar, Optional, Union + +import fsspec +from fsspec.core import url_to_fs +from huggingface_hub import DatasetCard, DatasetCardData + +from . import config +from .features import Features +from .splits import SplitDict +from .utils import Version +from .utils.logging import get_logger +from .utils.py_utils import asdict, unique_values + + +logger = get_logger(__name__) + + +@dataclass +class SupervisedKeysData: + input: str = "" + output: str = "" + + +@dataclass +class DownloadChecksumsEntryData: + key: str = "" + value: str = "" + + +class MissingCachedSizesConfigError(Exception): + """The expected cached sizes of the download file are missing.""" + + +class NonMatchingCachedSizesError(Exception): + """The prepared split doesn't have expected sizes.""" + + +@dataclass +class PostProcessedInfo: + features: Optional[Features] = None + resources_checksums: Optional[dict] = None + + def __post_init__(self): + # Convert back to the correct classes when we reload from dict + if self.features is not None and not isinstance(self.features, Features): + self.features = Features.from_dict(self.features) + + @classmethod + def from_dict(cls, post_processed_info_dict: dict) -> "PostProcessedInfo": + field_names = {f.name for f in dataclasses.fields(cls)} + return cls(**{k: v for k, v in post_processed_info_dict.items() if k in field_names}) + + +@dataclass +class DatasetInfo: + """Information about a dataset. + + `DatasetInfo` documents datasets, including its name, version, and features. + See the constructor arguments and properties for a full list. + + Not all fields are known on construction and may be updated later. + + Attributes: + description (`str`): + A description of the dataset. + citation (`str`): + A BibTeX citation of the dataset. + homepage (`str`): + A URL to the official homepage for the dataset. + license (`str`): + The dataset's license. It can be the name of the license or a paragraph containing the terms of the license. + features ([`Features`], *optional*): + The features used to specify the dataset's column types. + post_processed (`PostProcessedInfo`, *optional*): + Information regarding the resources of a possible post-processing of a dataset. For example, it can contain the information of an index. + supervised_keys (`SupervisedKeysData`, *optional*): + Specifies the input feature and the label for supervised learning if applicable for the dataset (legacy from TFDS). + builder_name (`str`, *optional*): + The name of the `GeneratorBasedBuilder` subclass used to create the dataset. Usually matched to the corresponding script name. It is also the snake_case version of the dataset builder class name. + config_name (`str`, *optional*): + The name of the configuration derived from [`BuilderConfig`]. + version (`str` or [`Version`], *optional*): + The version of the dataset. + splits (`dict`, *optional*): + The mapping between split name and metadata. + download_checksums (`dict`, *optional*): + The mapping between the URL to download the dataset's checksums and corresponding metadata. + download_size (`int`, *optional*): + The size of the files to download to generate the dataset, in bytes. + post_processing_size (`int`, *optional*): + Size of the dataset in bytes after post-processing, if any. + dataset_size (`int`, *optional*): + The combined size in bytes of the Arrow tables for all splits. + size_in_bytes (`int`, *optional*): + The combined size in bytes of all files associated with the dataset (downloaded files + Arrow files). + **config_kwargs (additional keyword arguments): + Keyword arguments to be passed to the [`BuilderConfig`] and used in the [`DatasetBuilder`]. + """ + + # Set in the dataset scripts + description: str = dataclasses.field(default_factory=str) + citation: str = dataclasses.field(default_factory=str) + homepage: str = dataclasses.field(default_factory=str) + license: str = dataclasses.field(default_factory=str) + features: Optional[Features] = None + post_processed: Optional[PostProcessedInfo] = None + supervised_keys: Optional[SupervisedKeysData] = None + + # Set later by the builder + builder_name: Optional[str] = None + dataset_name: Optional[str] = None # for packaged builders, to be different from builder_name + config_name: Optional[str] = None + version: Optional[Union[str, Version]] = None + # Set later by `download_and_prepare` + splits: Optional[dict] = None + download_checksums: Optional[dict] = None + download_size: Optional[int] = None + post_processing_size: Optional[int] = None + dataset_size: Optional[int] = None + size_in_bytes: Optional[int] = None + + _INCLUDED_INFO_IN_YAML: ClassVar[list[str]] = [ + "config_name", + "download_size", + "dataset_size", + "features", + "splits", + ] + + def __post_init__(self): + # Convert back to the correct classes when we reload from dict + if self.features is not None and not isinstance(self.features, Features): + self.features = Features.from_dict(self.features) + if self.post_processed is not None and not isinstance(self.post_processed, PostProcessedInfo): + self.post_processed = PostProcessedInfo.from_dict(self.post_processed) + if self.version is not None and not isinstance(self.version, Version): + if isinstance(self.version, str): + self.version = Version(self.version) + else: + self.version = Version.from_dict(self.version) + if self.splits is not None and not isinstance(self.splits, SplitDict): + self.splits = SplitDict.from_split_dict(self.splits) + if self.supervised_keys is not None and not isinstance(self.supervised_keys, SupervisedKeysData): + if isinstance(self.supervised_keys, (tuple, list)): + self.supervised_keys = SupervisedKeysData(*self.supervised_keys) + else: + self.supervised_keys = SupervisedKeysData(**self.supervised_keys) + + def write_to_directory(self, dataset_info_dir, pretty_print=False, storage_options: Optional[dict] = None): + """Write `DatasetInfo` and license (if present) as JSON files to `dataset_info_dir`. + + Args: + dataset_info_dir (`str`): + Destination directory. + pretty_print (`bool`, defaults to `False`): + If `True`, the JSON will be pretty-printed with the indent level of 4. + storage_options (`dict`, *optional*): + Key/value pairs to be passed on to the file-system backend, if any. + + + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> ds.info.write_to_directory("/path/to/directory/") + ``` + """ + fs: fsspec.AbstractFileSystem + fs, *_ = url_to_fs(dataset_info_dir, **(storage_options or {})) + with fs.open(posixpath.join(dataset_info_dir, config.DATASET_INFO_FILENAME), "wb") as f: + self._dump_info(f, pretty_print=pretty_print) + if self.license: + with fs.open(posixpath.join(dataset_info_dir, config.LICENSE_FILENAME), "wb") as f: + self._dump_license(f) + + def _dump_info(self, file, pretty_print=False): + """Dump info in `file` file-like object open in bytes mode (to support remote files)""" + file.write(json.dumps(asdict(self), indent=4 if pretty_print else None).encode("utf-8")) + + def _dump_license(self, file): + """Dump license in `file` file-like object open in bytes mode (to support remote files)""" + file.write(self.license.encode("utf-8")) + + @classmethod + def from_merge(cls, dataset_infos: list["DatasetInfo"]): + dataset_infos = [dset_info.copy() for dset_info in dataset_infos if dset_info is not None] + + if len(dataset_infos) > 0 and all(dataset_infos[0] == dset_info for dset_info in dataset_infos): + # if all dataset_infos are equal we don't need to merge. Just return the first. + return dataset_infos[0] + + description = "\n\n".join(unique_values(info.description for info in dataset_infos)).strip() + citation = "\n\n".join(unique_values(info.citation for info in dataset_infos)).strip() + homepage = "\n\n".join(unique_values(info.homepage for info in dataset_infos)).strip() + license = "\n\n".join(unique_values(info.license for info in dataset_infos)).strip() + features = None + supervised_keys = None + + return cls( + description=description, + citation=citation, + homepage=homepage, + license=license, + features=features, + supervised_keys=supervised_keys, + ) + + @classmethod + def from_directory(cls, dataset_info_dir: str, storage_options: Optional[dict] = None) -> "DatasetInfo": + """Create [`DatasetInfo`] from the JSON file in `dataset_info_dir`. + + This function updates all the dynamically generated fields (num_examples, + hash, time of creation,...) of the [`DatasetInfo`]. + + This will overwrite all previous metadata. + + Args: + dataset_info_dir (`str`): + The directory containing the metadata file. This + should be the root directory of a specific dataset version. + storage_options (`dict`, *optional*): + Key/value pairs to be passed on to the file-system backend, if any. + + + + Example: + + ```py + >>> from datasets import DatasetInfo + >>> ds_info = DatasetInfo.from_directory("/path/to/directory/") + ``` + """ + fs: fsspec.AbstractFileSystem + fs, *_ = url_to_fs(dataset_info_dir, **(storage_options or {})) + logger.info(f"Loading Dataset info from {dataset_info_dir}") + if not dataset_info_dir: + raise ValueError("Calling DatasetInfo.from_directory() with undefined dataset_info_dir.") + with fs.open(posixpath.join(dataset_info_dir, config.DATASET_INFO_FILENAME), "r", encoding="utf-8") as f: + dataset_info_dict = json.load(f) + return cls.from_dict(dataset_info_dict) + + @classmethod + def from_dict(cls, dataset_info_dict: dict) -> "DatasetInfo": + field_names = {f.name for f in dataclasses.fields(cls)} + return cls(**{k: v for k, v in dataset_info_dict.items() if k in field_names}) + + def update(self, other_dataset_info: "DatasetInfo", ignore_none=True): + self_dict = self.__dict__ + self_dict.update( + **{ + k: copy.deepcopy(v) + for k, v in other_dataset_info.__dict__.items() + if (v is not None or not ignore_none) + } + ) + + def copy(self) -> "DatasetInfo": + return self.__class__(**{k: copy.deepcopy(v) for k, v in self.__dict__.items()}) + + def _to_yaml_dict(self) -> dict: + yaml_dict = {} + dataset_info_dict = asdict(self) + for key in dataset_info_dict: + if key in self._INCLUDED_INFO_IN_YAML: + value = getattr(self, key) + if hasattr(value, "_to_yaml_list"): # Features, SplitDict + yaml_dict[key] = value._to_yaml_list() + elif hasattr(value, "_to_yaml_string"): # Version + yaml_dict[key] = value._to_yaml_string() + else: + yaml_dict[key] = value + return yaml_dict + + @classmethod + def _from_yaml_dict(cls, yaml_data: dict) -> "DatasetInfo": + yaml_data = copy.deepcopy(yaml_data) + if yaml_data.get("features") is not None: + yaml_data["features"] = Features._from_yaml_list(yaml_data["features"]) + if yaml_data.get("splits") is not None: + yaml_data["splits"] = SplitDict._from_yaml_list(yaml_data["splits"]) + field_names = {f.name for f in dataclasses.fields(cls)} + return cls(**{k: v for k, v in yaml_data.items() if k in field_names}) + + +class DatasetInfosDict(dict[str, DatasetInfo]): + def write_to_directory(self, dataset_infos_dir, overwrite=False, pretty_print=False) -> None: + total_dataset_infos = {} + dataset_infos_path = os.path.join(dataset_infos_dir, config.DATASETDICT_INFOS_FILENAME) + dataset_readme_path = os.path.join(dataset_infos_dir, config.REPOCARD_FILENAME) + if not overwrite: + total_dataset_infos = self.from_directory(dataset_infos_dir) + total_dataset_infos.update(self) + if os.path.exists(dataset_infos_path): + # for backward compatibility, let's update the JSON file if it exists + with open(dataset_infos_path, "w", encoding="utf-8") as f: + dataset_infos_dict = { + config_name: asdict(dset_info) for config_name, dset_info in total_dataset_infos.items() + } + json.dump(dataset_infos_dict, f, indent=4 if pretty_print else None) + # Dump the infos in the YAML part of the README.md file + if os.path.exists(dataset_readme_path): + dataset_card = DatasetCard.load(dataset_readme_path) + dataset_card_data = dataset_card.data + else: + dataset_card = None + dataset_card_data = DatasetCardData() + if total_dataset_infos: + total_dataset_infos.to_dataset_card_data(dataset_card_data) + dataset_card = ( + DatasetCard("---\n" + str(dataset_card_data) + "\n---\n") if dataset_card is None else dataset_card + ) + dataset_card.save(Path(dataset_readme_path)) + + @classmethod + def from_directory(cls, dataset_infos_dir) -> "DatasetInfosDict": + logger.info(f"Loading Dataset Infos from {dataset_infos_dir}") + # Load the info from the YAML part of README.md + if os.path.exists(os.path.join(dataset_infos_dir, config.REPOCARD_FILENAME)): + dataset_card_data = DatasetCard.load(Path(dataset_infos_dir) / config.REPOCARD_FILENAME).data + if "dataset_info" in dataset_card_data: + return cls.from_dataset_card_data(dataset_card_data) + if os.path.exists(os.path.join(dataset_infos_dir, config.DATASETDICT_INFOS_FILENAME)): + # this is just to have backward compatibility with dataset_infos.json files + with open(os.path.join(dataset_infos_dir, config.DATASETDICT_INFOS_FILENAME), encoding="utf-8") as f: + return cls( + { + config_name: DatasetInfo.from_dict(dataset_info_dict) + for config_name, dataset_info_dict in json.load(f).items() + } + ) + else: + return cls() + + @classmethod + def from_dataset_card_data(cls, dataset_card_data: DatasetCardData) -> "DatasetInfosDict": + if isinstance(dataset_card_data.get("dataset_info"), (list, dict)): + if isinstance(dataset_card_data["dataset_info"], list): + return cls( + { + dataset_info_yaml_dict.get("config_name", "default"): DatasetInfo._from_yaml_dict( + dataset_info_yaml_dict + ) + for dataset_info_yaml_dict in dataset_card_data["dataset_info"] + } + ) + else: + dataset_info = DatasetInfo._from_yaml_dict(dataset_card_data["dataset_info"]) + dataset_info.config_name = dataset_card_data["dataset_info"].get("config_name", "default") + return cls({dataset_info.config_name: dataset_info}) + else: + return cls() + + def to_dataset_card_data(self, dataset_card_data: DatasetCardData) -> None: + if self: + # first get existing metadata info + if "dataset_info" in dataset_card_data and isinstance(dataset_card_data["dataset_info"], dict): + dataset_metadata_infos = { + dataset_card_data["dataset_info"].get("config_name", "default"): dataset_card_data["dataset_info"] + } + elif "dataset_info" in dataset_card_data and isinstance(dataset_card_data["dataset_info"], list): + dataset_metadata_infos = { + config_metadata["config_name"]: config_metadata + for config_metadata in dataset_card_data["dataset_info"] + } + else: + dataset_metadata_infos = {} + # update/rewrite existing metadata info with the one to dump + total_dataset_infos = { + **dataset_metadata_infos, + **{config_name: dset_info._to_yaml_dict() for config_name, dset_info in self.items()}, + } + # the config_name from the dataset_infos_dict takes over the config_name of the DatasetInfo + for config_name, dset_info_yaml_dict in total_dataset_infos.items(): + dset_info_yaml_dict["config_name"] = config_name + if len(total_dataset_infos) == 1: + # use a struct instead of a list of configurations, since there's only one + dataset_card_data["dataset_info"] = next(iter(total_dataset_infos.values())) + config_name = dataset_card_data["dataset_info"].pop("config_name", None) + if config_name != "default": + # if config_name is not "default" preserve it and put at the first position + dataset_card_data["dataset_info"] = { + "config_name": config_name, + **dataset_card_data["dataset_info"], + } + else: + dataset_card_data["dataset_info"] = [] + for config_name, dataset_info_yaml_dict in sorted(total_dataset_infos.items()): + # add the config_name field in first position + dataset_info_yaml_dict.pop("config_name", None) + dataset_info_yaml_dict = {"config_name": config_name, **dataset_info_yaml_dict} + dataset_card_data["dataset_info"].append(dataset_info_yaml_dict) diff --git a/lib/python3.12/site-packages/datasets/inspect.py b/lib/python3.12/site-packages/datasets/inspect.py new file mode 100644 index 0000000000000000000000000000000000000000..0eff9d2c002f741a3fd2f8d11f4079061a1be702 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/inspect.py @@ -0,0 +1,363 @@ +# Copyright 2020 The HuggingFace Datasets Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +"""List and inspect datasets.""" + +import os +from collections.abc import Mapping, Sequence +from typing import Optional, Union + +from .download.download_config import DownloadConfig +from .download.download_manager import DownloadMode +from .download.streaming_download_manager import StreamingDownloadManager +from .info import DatasetInfo +from .load import ( + dataset_module_factory, + get_dataset_builder_class, + load_dataset_builder, +) +from .utils.logging import get_logger +from .utils.version import Version + + +logger = get_logger(__name__) + + +class SplitsNotFoundError(ValueError): + pass + + +def get_dataset_infos( + path: str, + data_files: Optional[Union[dict, list, str]] = None, + download_config: Optional[DownloadConfig] = None, + download_mode: Optional[Union[DownloadMode, str]] = None, + revision: Optional[Union[str, Version]] = None, + token: Optional[Union[bool, str]] = None, + **config_kwargs, +): + """Get the meta information about a dataset, returned as a dict mapping config name to DatasetInfoDict. + + Args: + path (`str`): path to the dataset processing script with the dataset builder. Can be either: + + - a local path to processing script or the directory containing the script (if the script has the same name as the directory), + e.g. `'./dataset/squad'` or `'./dataset/squad/squad.py'` + - a dataset identifier on the Hugging Face Hub (list all available datasets and ids with [`huggingface_hub.list_datasets`]), + e.g. `'rajpurkar/squad'`, `'nyu-mll/glue'` or``'openai/webtext'` + revision (`Union[str, datasets.Version]`, *optional*): + If specified, the dataset module will be loaded from the datasets repository at this version. + By default: + - it is set to the local version of the lib. + - it will also try to load it from the main branch if it's not available at the local version of the lib. + Specifying a version that is different from your local version of the lib might cause compatibility issues. + download_config ([`DownloadConfig`], *optional*): + Specific download configuration parameters. + download_mode ([`DownloadMode`] or `str`, defaults to `REUSE_DATASET_IF_EXISTS`): + Download/generate mode. + data_files (`Union[Dict, List, str]`, *optional*): + Defining the data_files of the dataset configuration. + token (`str` or `bool`, *optional*): + Optional string or boolean to use as Bearer token for remote files on the Datasets Hub. + If `True`, or not specified, will get token from `"~/.huggingface"`. + **config_kwargs (additional keyword arguments): + Optional attributes for builder class which will override the attributes if supplied. + + Example: + + ```py + >>> from datasets import get_dataset_infos + >>> get_dataset_infos('cornell-movie-review-data/rotten_tomatoes') + {'default': DatasetInfo(description="Movie Review Dataset.\nThis is a dataset of containing 5,331 positive and 5,331 negative processed\nsentences from Rotten Tomatoes movie reviews...), ...} + ``` + """ + config_names = get_dataset_config_names( + path=path, + revision=revision, + download_config=download_config, + download_mode=download_mode, + data_files=data_files, + token=token, + ) + return { + config_name: get_dataset_config_info( + path=path, + config_name=config_name, + data_files=data_files, + download_config=download_config, + download_mode=download_mode, + revision=revision, + token=token, + **config_kwargs, + ) + for config_name in config_names + } + + +def get_dataset_config_names( + path: str, + revision: Optional[Union[str, Version]] = None, + download_config: Optional[DownloadConfig] = None, + download_mode: Optional[Union[DownloadMode, str]] = None, + dynamic_modules_path: Optional[str] = None, + data_files: Optional[Union[dict, list, str]] = None, + **download_kwargs, +): + """Get the list of available config names for a particular dataset. + + Args: + path (`str`): path to the dataset processing script with the dataset builder. Can be either: + + - a local path to processing script or the directory containing the script (if the script has the same name as the directory), + e.g. `'./dataset/squad'` or `'./dataset/squad/squad.py'` + - a dataset identifier on the Hugging Face Hub (list all available datasets and ids with [`huggingface_hub.list_datasets`]), + e.g. `'rajpurkar/squad'`, `'nyu-mll/glue'` or `'openai/webtext'` + revision (`Union[str, datasets.Version]`, *optional*): + If specified, the dataset module will be loaded from the datasets repository at this version. + By default: + - it is set to the local version of the lib. + - it will also try to load it from the main branch if it's not available at the local version of the lib. + Specifying a version that is different from your local version of the lib might cause compatibility issues. + download_config ([`DownloadConfig`], *optional*): + Specific download configuration parameters. + download_mode ([`DownloadMode`] or `str`, defaults to `REUSE_DATASET_IF_EXISTS`): + Download/generate mode. + dynamic_modules_path (`str`, defaults to `~/.cache/huggingface/modules/datasets_modules`): + Optional path to the directory in which the dynamic modules are saved. It must have been initialized with `init_dynamic_modules`. + By default the datasets are stored inside the `datasets_modules` module. + data_files (`Union[Dict, List, str]`, *optional*): + Defining the data_files of the dataset configuration. + **download_kwargs (additional keyword arguments): + Optional attributes for [`DownloadConfig`] which will override the attributes in `download_config` if supplied, + for example `token`. + + Example: + + ```py + >>> from datasets import get_dataset_config_names + >>> get_dataset_config_names("nyu-mll/glue") + ['cola', + 'sst2', + 'mrpc', + 'qqp', + 'stsb', + 'mnli', + 'mnli_mismatched', + 'mnli_matched', + 'qnli', + 'rte', + 'wnli', + 'ax'] + ``` + """ + dataset_module = dataset_module_factory( + path, + revision=revision, + download_config=download_config, + download_mode=download_mode, + dynamic_modules_path=dynamic_modules_path, + data_files=data_files, + **download_kwargs, + ) + builder_cls = get_dataset_builder_class(dataset_module, dataset_name=os.path.basename(path)) + return list(builder_cls.builder_configs.keys()) or [ + dataset_module.builder_kwargs.get("config_name", builder_cls.DEFAULT_CONFIG_NAME or "default") + ] + + +def get_dataset_default_config_name( + path: str, + revision: Optional[Union[str, Version]] = None, + download_config: Optional[DownloadConfig] = None, + download_mode: Optional[Union[DownloadMode, str]] = None, + dynamic_modules_path: Optional[str] = None, + data_files: Optional[Union[dict, list, str]] = None, + **download_kwargs, +) -> Optional[str]: + """Get the default config name for a particular dataset. + Can return None only if the dataset has multiple configurations and no default configuration. + + Args: + path (`str`): path to the dataset processing script with the dataset builder. Can be either: + + - a local path to processing script or the directory containing the script (if the script has the same name as the directory), + e.g. `'./dataset/squad'` or `'./dataset/squad/squad.py'` + - a dataset identifier on the Hugging Face Hub (list all available datasets and ids with [`huggingface_hub.list_datasets`]), + e.g. `'rajpurkar/squad'`, `'nyu-mll/glue'` or `'openai/webtext'` + revision (`Union[str, datasets.Version]`, *optional*): + If specified, the dataset module will be loaded from the datasets repository at this version. + By default: + - it is set to the local version of the lib. + - it will also try to load it from the main branch if it's not available at the local version of the lib. + Specifying a version that is different from your local version of the lib might cause compatibility issues. + download_config ([`DownloadConfig`], *optional*): + Specific download configuration parameters. + download_mode ([`DownloadMode`] or `str`, defaults to `REUSE_DATASET_IF_EXISTS`): + Download/generate mode. + dynamic_modules_path (`str`, defaults to `~/.cache/huggingface/modules/datasets_modules`): + Optional path to the directory in which the dynamic modules are saved. It must have been initialized with `init_dynamic_modules`. + By default the datasets are stored inside the `datasets_modules` module. + data_files (`Union[Dict, List, str]`, *optional*): + Defining the data_files of the dataset configuration. + **download_kwargs (additional keyword arguments): + Optional attributes for [`DownloadConfig`] which will override the attributes in `download_config` if supplied, + for example `token`. + + Returns: + Optional[str]: the default config name if there is one + + Example: + + ```py + >>> from datasets import get_dataset_default_config_name + >>> get_dataset_default_config_name("openbookqa") + 'main' + ``` + """ + dataset_module = dataset_module_factory( + path, + revision=revision, + download_config=download_config, + download_mode=download_mode, + dynamic_modules_path=dynamic_modules_path, + data_files=data_files, + **download_kwargs, + ) + builder_cls = get_dataset_builder_class(dataset_module, dataset_name=os.path.basename(path)) + builder_configs = list(builder_cls.builder_configs.keys()) + if builder_configs: + default_config_name = builder_configs[0] if len(builder_configs) == 1 else None + else: + default_config_name = "default" + return builder_cls.DEFAULT_CONFIG_NAME or default_config_name + + +def get_dataset_config_info( + path: str, + config_name: Optional[str] = None, + data_files: Optional[Union[str, Sequence[str], Mapping[str, Union[str, Sequence[str]]]]] = None, + download_config: Optional[DownloadConfig] = None, + download_mode: Optional[Union[DownloadMode, str]] = None, + revision: Optional[Union[str, Version]] = None, + token: Optional[Union[bool, str]] = None, + **config_kwargs, +) -> DatasetInfo: + """Get the meta information (DatasetInfo) about a dataset for a particular config + + Args: + path (``str``): path to the dataset processing script with the dataset builder. Can be either: + + - a local path to processing script or the directory containing the script (if the script has the same name as the directory), + e.g. ``'./dataset/squad'`` or ``'./dataset/squad/squad.py'`` + - a dataset identifier on the Hugging Face Hub (list all available datasets and ids with [`huggingface_hub.list_datasets`]), + e.g. ``'rajpurkar/squad'``, ``'nyu-mll/glue'`` or ``'openai/webtext'`` + config_name (:obj:`str`, optional): Defining the name of the dataset configuration. + data_files (:obj:`str` or :obj:`Sequence` or :obj:`Mapping`, optional): Path(s) to source data file(s). + download_config (:class:`~download.DownloadConfig`, optional): Specific download configuration parameters. + download_mode (:class:`DownloadMode` or :obj:`str`, default ``REUSE_DATASET_IF_EXISTS``): Download/generate mode. + revision (:class:`~utils.Version` or :obj:`str`, optional): Version of the dataset script to load. + As datasets have their own git repository on the Datasets Hub, the default version "main" corresponds to their "main" branch. + You can specify a different version than the default "main" by using a commit SHA or a git tag of the dataset repository. + token (``str`` or :obj:`bool`, optional): Optional string or boolean to use as Bearer token for remote files on the Datasets Hub. + If True, or not specified, will get token from `"~/.huggingface"`. + **config_kwargs (additional keyword arguments): optional attributes for builder class which will override the attributes if supplied. + + """ + builder = load_dataset_builder( + path, + name=config_name, + data_files=data_files, + download_config=download_config, + download_mode=download_mode, + revision=revision, + token=token, + **config_kwargs, + ) + info = builder.info + if info.splits is None: + download_config = download_config.copy() if download_config else DownloadConfig() + if token is not None: + download_config.token = token + builder._check_manual_download( + StreamingDownloadManager(base_path=builder.base_path, download_config=download_config) + ) + try: + info.splits = { + split_generator.name: {"name": split_generator.name, "dataset_name": path} + for split_generator in builder._split_generators( + StreamingDownloadManager(base_path=builder.base_path, download_config=download_config) + ) + } + except Exception as err: + raise SplitsNotFoundError("The split names could not be parsed from the dataset config.") from err + return info + + +def get_dataset_split_names( + path: str, + config_name: Optional[str] = None, + data_files: Optional[Union[str, Sequence[str], Mapping[str, Union[str, Sequence[str]]]]] = None, + download_config: Optional[DownloadConfig] = None, + download_mode: Optional[Union[DownloadMode, str]] = None, + revision: Optional[Union[str, Version]] = None, + token: Optional[Union[bool, str]] = None, + **config_kwargs, +): + """Get the list of available splits for a particular config and dataset. + + Args: + path (`str`): path to the dataset processing script with the dataset builder. Can be either: + + - a local path to processing script or the directory containing the script (if the script has the same name as the directory), + e.g. `'./dataset/squad'` or `'./dataset/squad/squad.py'` + - a dataset identifier on the Hugging Face Hub (list all available datasets and ids with [`huggingface_hub.list_datasets`]), + e.g. `'rajpurkar/squad'`, `'nyu-mll/glue'` or `'openai/webtext'` + config_name (`str`, *optional*): + Defining the name of the dataset configuration. + data_files (`str` or `Sequence` or `Mapping`, *optional*): + Path(s) to source data file(s). + download_config ([`DownloadConfig`], *optional*): + Specific download configuration parameters. + download_mode ([`DownloadMode`] or `str`, defaults to `REUSE_DATASET_IF_EXISTS`): + Download/generate mode. + revision ([`Version`] or `str`, *optional*): + Version of the dataset script to load. + As datasets have their own git repository on the Datasets Hub, the default version "main" corresponds to their "main" branch. + You can specify a different version than the default "main" by using a commit SHA or a git tag of the dataset repository. + token (`str` or `bool`, *optional*): + Optional string or boolean to use as Bearer token for remote files on the Datasets Hub. + If `True`, or not specified, will get token from `"~/.huggingface"`. + **config_kwargs (additional keyword arguments): + Optional attributes for builder class which will override the attributes if supplied. + + Example: + + ```py + >>> from datasets import get_dataset_split_names + >>> get_dataset_split_names('cornell-movie-review-data/rotten_tomatoes') + ['train', 'validation', 'test'] + ``` + """ + info = get_dataset_config_info( + path, + config_name=config_name, + data_files=data_files, + download_config=download_config, + download_mode=download_mode, + revision=revision, + token=token, + **config_kwargs, + ) + return list(info.splits.keys()) diff --git a/lib/python3.12/site-packages/datasets/iterable_dataset.py b/lib/python3.12/site-packages/datasets/iterable_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..80e4b6b729292c3b9bdf48d6d4242dbb1e79f52f --- /dev/null +++ b/lib/python3.12/site-packages/datasets/iterable_dataset.py @@ -0,0 +1,3547 @@ +import asyncio +import copy +import inspect +import itertools +import multiprocessing.pool +import sys +from collections import Counter +from collections.abc import Iterable, Iterator +from copy import deepcopy +from dataclasses import dataclass +from functools import partial +from itertools import cycle, islice +from typing import TYPE_CHECKING, Any, Callable, Optional, Union + +import fsspec.asyn +import numpy as np +import pandas as pd +import pyarrow as pa + +from . import config +from .arrow_dataset import Dataset, DatasetInfoMixin +from .features import Features +from .features.features import ( + FeatureType, + Value, + _align_features, + _check_if_features_can_be_aligned, + _visit, + cast_to_python_objects, +) +from .formatting import ( + ArrowFormatter, + PythonFormatter, + TableFormatter, + TensorFormatter, + get_format_type_from_alias, + get_formatter, +) +from .info import DatasetInfo +from .splits import NamedSplit, Split +from .table import cast_table_to_features, read_schema_from_file, table_cast +from .utils.logging import get_logger +from .utils.py_utils import Literal +from .utils.sharding import _merge_gen_kwargs, _number_of_shards_in_gen_kwargs, _shuffle_gen_kwargs, _split_gen_kwargs + + +if TYPE_CHECKING: + import torch + +logger = get_logger(__name__) + +Key = Union[int, str] + + +def identity_func(x): + return x + + +def _rename_columns_fn(example: dict, column_mapping: dict[str, str]): + if any(col not in example for col in column_mapping): + raise ValueError( + f"Error when renaming {list(column_mapping)} to {list(column_mapping.values())}: columns {set(column_mapping) - set(example)} are not in the dataset." + ) + if any(col in example for col in column_mapping.values()): + raise ValueError( + f"Error when renaming {list(column_mapping)} to {list(column_mapping.values())}: columns {set(example) - set(column_mapping.values())} are already in the dataset." + ) + return { + new_column_name: example[original_column_name] + for original_column_name, new_column_name in column_mapping.items() + } + + +def add_column_fn(example: dict, idx: int, name: str, column: list[dict]): + if name in example: + raise ValueError(f"Error when adding {name}: column {name} is already in the dataset.") + return {name: column[idx]} + + +def _infer_features_from_batch(batch: dict[str, list], try_features: Optional[Features] = None) -> Features: + pa_table = pa.Table.from_pydict(batch) + if try_features is not None: + try: + pa_table = table_cast(pa_table, pa.schema(try_features.type)) + except (TypeError, pa.ArrowInvalid, pa.ArrowNotImplementedError): + pass + return Features.from_arrow_schema(pa_table.schema) + + +def _examples_to_batch(examples: list[dict[str, Any]]) -> dict[str, list]: + # we order the columns by order of appearance + # to do so, we use a dict as an ordered set + cols = {col: None for example in examples for col in example} + # when an example is missing a column, we set the value to None with .get() + arrays = [[example.get(col) for example in examples] for col in cols] + return dict(zip(cols, arrays)) + + +def _batch_to_examples(batch: dict[str, list]) -> Iterator[dict[str, Any]]: + """Convert a batch (dict of examples) to examples list""" + n_examples = 0 if len(batch) == 0 else len(batch[next(iter(batch))]) + for i in range(n_examples): + yield {col: array[i] for col, array in batch.items()} + + +def _convert_to_arrow( + iterable: Iterable[tuple[Key, dict]], + batch_size: int, + drop_last_batch: bool = False, +) -> Iterator[tuple[Key, pa.Table]]: + """Convert and group examples in Arrow tables of size `batch_size`. + + Args: + iterable (`Iterable[Tuple[Key, dict]]`): + An examples iterable containing tuples (example_key, example) of type (int/str, dict) + batch_size (`Optional[int]`): + Size of each sub-table to yield. If None or <= 0, yields the full table. + drop_last_batch (`bool`, defaults to `False`): + Drop the last batch if it is smaller than `batch_size`. + """ + if batch_size is None or batch_size <= 0: + yield ( + "all", + pa.Table.from_pylist(cast_to_python_objects([example for _, example in iterable], only_1d_for_numpy=True)), + ) + return + iterator = iter(iterable) + for key, example in iterator: + iterator_batch = islice(iterator, batch_size - 1) + key_examples_list = [(key, example)] + list(iterator_batch) + if len(key_examples_list) < batch_size and drop_last_batch: + return + keys, examples = zip(*key_examples_list) + new_key = "_".join(str(key) for key in keys) + yield new_key, pa.Table.from_pylist(cast_to_python_objects(examples, only_1d_for_numpy=True)) + + +class _BaseExamplesIterable: + """Base class for the examples iterable used by an IterableDataset""" + + def __init__(self) -> None: + self._state_dict: Optional[Union[list, dict]] = None + + def __iter__(self) -> Iterator[tuple[Key, dict]]: + """An examples iterable should yield tuples (example_key, example) of type (int/str, dict)""" + raise NotImplementedError(f"{type(self)} doesn't implement __iter__ yet") + + @property + def iter_arrow(self) -> Optional[Callable[[], Iterator[tuple[Key, pa.Table]]]]: + return None + + @property + def is_typed(self) -> bool: + return False + + @property + def features(self) -> Optional[Features]: + return None + + def shuffle_data_sources(self, generator: np.random.Generator) -> "_BaseExamplesIterable": + """ + Either shuffle the shards/sources of the dataset, or propagate the shuffling to the underlying iterable. + If the order of the shards must stay fixed (when using .skip or .take for example), then this method returns self. + """ + raise NotImplementedError(f"{type(self)} doesn't implement shuffle_data_sources yet") + + def shard_data_sources(self, num_shards: int, index: int, contiguous=True) -> "_BaseExamplesIterable": + """Either keep only the requested shard, or propagate the request to the underlying iterable.""" + raise NotImplementedError(f"{type(self)} doesn't implement shard_data_sources yet") + + def split_shard_indices_by_worker(self, num_shards: int, index: int, contiguous=True) -> list[int]: + if contiguous: + div = self.num_shards // num_shards + mod = self.num_shards % num_shards + start = div * index + min(index, mod) + end = start + div + (1 if index < mod else 0) + return list(range(start, end)) + else: + return list(range(index, self.num_shards, num_shards)) + + @property + def num_shards(self) -> int: + raise NotImplementedError(f"{type(self)} doesn't implement num_shards yet") + + def _init_state_dict(self) -> dict: + raise NotImplementedError(f"{type(self)} doesn't implement _init_state_dict yet") + + def load_state_dict(self, state_dict: dict) -> dict: + def _inner_load_state_dict(state, new_state): + if new_state is not None and isinstance(state, dict): + for key in new_state: + state[key] = _inner_load_state_dict(state[key], new_state[key]) + return state + elif new_state is not None and isinstance(state, list): + for i in range(len(state)): + state[i] = _inner_load_state_dict(state[i], new_state[i]) + return state + return new_state + + return _inner_load_state_dict(self._state_dict, state_dict) + + def state_dict(self) -> dict: + if self._state_dict: + return copy.deepcopy(self._state_dict) + raise RuntimeError("State dict is not initialized, please call ex_iterable._init_state_dict() first.") + + +class ExamplesIterable(_BaseExamplesIterable): + def __init__(self, generate_examples_fn: Callable[..., tuple[Key, dict]], kwargs: dict): + super().__init__() + self.generate_examples_fn = generate_examples_fn + self.kwargs = kwargs + + def _init_state_dict(self) -> dict: + self._state_dict = {"shard_idx": 0, "shard_example_idx": 0, "type": self.__class__.__name__} + return self._state_dict + + def __iter__(self): + shard_idx_start = self._state_dict["shard_idx"] if self._state_dict else 0 + for gen_kwags in islice(_split_gen_kwargs(self.kwargs, max_num_jobs=self.num_shards), shard_idx_start, None): + shard_example_idx_start = self._state_dict["shard_example_idx"] if self._state_dict else 0 + for key_example in islice(self.generate_examples_fn(**gen_kwags), shard_example_idx_start, None): + if self._state_dict: + self._state_dict["shard_example_idx"] += 1 + yield key_example + if self._state_dict: + self._state_dict["shard_idx"] += 1 + self._state_dict["shard_example_idx"] = 0 + + def shuffle_data_sources(self, generator: np.random.Generator) -> "ExamplesIterable": + return ShuffledDataSourcesExamplesIterable(self.generate_examples_fn, self.kwargs, generator) + + def shard_data_sources(self, num_shards: int, index: int, contiguous=True) -> "ExamplesIterable": + """Keep only the requested shard.""" + gen_kwargs_list = _split_gen_kwargs(self.kwargs, max_num_jobs=self.num_shards) + shard_indices = self.split_shard_indices_by_worker(num_shards, index, contiguous=contiguous) + requested_gen_kwargs = _merge_gen_kwargs([gen_kwargs_list[i] for i in shard_indices]) + return ExamplesIterable(self.generate_examples_fn, requested_gen_kwargs) + + @property + def num_shards(self) -> int: + return _number_of_shards_in_gen_kwargs(self.kwargs) + + +class ShuffledDataSourcesExamplesIterable(ExamplesIterable): + def __init__( + self, generate_examples_fn: Callable[..., tuple[Key, dict]], kwargs: dict, generator: np.random.Generator + ): + super().__init__(generate_examples_fn, kwargs) + self.generator = deepcopy(generator) + + def _init_state_dict(self) -> dict: + self._state_dict = {"shard_idx": 0, "shard_example_idx": 0, "type": self.__class__.__name__} + return self._state_dict + + def __iter__(self): + """Shuffle the kwargs order to shuffle shards""" + rng = deepcopy(self.generator) + kwargs_with_shuffled_shards = _shuffle_gen_kwargs(rng, self.kwargs) + shard_idx_start = self._state_dict["shard_idx"] if self._state_dict else 0 + for gen_kwags in islice( + _split_gen_kwargs(kwargs_with_shuffled_shards, max_num_jobs=self.num_shards), shard_idx_start, None + ): + shard_example_idx_start = self._state_dict["shard_example_idx"] if self._state_dict else 0 + for key_example in islice(self.generate_examples_fn(**gen_kwags), shard_example_idx_start, None): + if self._state_dict: + self._state_dict["shard_example_idx"] += 1 + yield key_example + if self._state_dict: + self._state_dict["shard_idx"] += 1 + self._state_dict["shard_example_idx"] = 0 + + def shard_data_sources(self, num_shards: int, index: int, contiguous=True) -> "ExamplesIterable": + """Keep only the requested shard.""" + rng = deepcopy(self.generator) + kwargs_with_shuffled_shards = _shuffle_gen_kwargs(rng, self.kwargs) + return ExamplesIterable(self.generate_examples_fn, kwargs_with_shuffled_shards).shard_data_sources( + num_shards, index, contiguous=contiguous + ) + + +class ArrowExamplesIterable(_BaseExamplesIterable): + def __init__(self, generate_tables_fn: Callable[..., tuple[Key, pa.Table]], kwargs: dict): + super().__init__() + self.generate_tables_fn = generate_tables_fn + self.kwargs = kwargs + + @property + def iter_arrow(self): + return self._iter_arrow + + def _init_state_dict(self) -> dict: + self._state_dict = {"shard_idx": 0, "shard_example_idx": 0, "type": self.__class__.__name__} + return self._state_dict + + def __iter__(self): + formatter = PythonFormatter() + shard_idx_start = self._state_dict["shard_idx"] if self._state_dict else 0 + for gen_kwags in islice(_split_gen_kwargs(self.kwargs, max_num_jobs=self.num_shards), shard_idx_start, None): + shard_example_idx_start = self._state_dict["shard_example_idx"] if self._state_dict else 0 + shard_example_idx = 0 + for key, pa_table in self.generate_tables_fn(**gen_kwags): + if shard_example_idx + len(pa_table) <= shard_example_idx_start: + shard_example_idx += len(pa_table) + continue + for pa_subtable in pa_table.to_reader(max_chunksize=config.ARROW_READER_BATCH_SIZE_IN_DATASET_ITER): + formatted_batch = formatter.format_batch(pa_subtable) + for example in _batch_to_examples(formatted_batch): + if shard_example_idx >= shard_example_idx_start: + if self._state_dict: + self._state_dict["shard_example_idx"] += 1 + yield key, example + shard_example_idx += 1 + if self._state_dict: + self._state_dict["shard_idx"] += 1 + self._state_dict["shard_example_idx"] = 0 + + def _iter_arrow(self): + shard_idx_start = self._state_dict["shard_idx"] if self._state_dict else 0 + for gen_kwags in islice(_split_gen_kwargs(self.kwargs, max_num_jobs=self.num_shards), shard_idx_start, None): + shard_example_idx_start = self._state_dict["shard_example_idx"] if self._state_dict else 0 + shard_example_idx = 0 + for key, pa_table in self.generate_tables_fn(**gen_kwags): + shard_example_idx += len(pa_table) + if shard_example_idx <= shard_example_idx_start: + continue + if self._state_dict: + self._state_dict["shard_example_idx"] += len(pa_table) + yield key, pa_table + if self._state_dict: + self._state_dict["shard_idx"] += 1 + self._state_dict["shard_example_idx"] = 0 + + def shuffle_data_sources(self, generator: np.random.Generator) -> "ArrowExamplesIterable": + return ShuffledDataSourcesArrowExamplesIterable(self.generate_tables_fn, self.kwargs, generator) + + def shard_data_sources(self, num_shards: int, index: int, contiguous=True) -> "ArrowExamplesIterable": + """Keep only the requested shard.""" + gen_kwargs_list = _split_gen_kwargs(self.kwargs, max_num_jobs=self.num_shards) + shard_indices = self.split_shard_indices_by_worker(num_shards, index, contiguous=contiguous) + requested_gen_kwargs = _merge_gen_kwargs([gen_kwargs_list[i] for i in shard_indices]) + return ArrowExamplesIterable(self.generate_tables_fn, requested_gen_kwargs) + + @property + def num_shards(self) -> int: + return _number_of_shards_in_gen_kwargs(self.kwargs) + + +class ShuffledDataSourcesArrowExamplesIterable(ArrowExamplesIterable): + def __init__( + self, + generate_tables_fn: Callable[..., tuple[Key, pa.Table]], + kwargs: dict, + generator: np.random.Generator, + ): + super().__init__(generate_tables_fn, kwargs) + self.generator = deepcopy(generator) + + def _init_state_dict(self) -> dict: + self._state_dict = {"shard_idx": 0, "shard_example_idx": 0, "type": self.__class__.__name__} + return self._state_dict + + def __iter__(self): + """Shuffle the kwargs order to shuffle shards""" + rng = deepcopy(self.generator) + kwargs_with_shuffled_shards = _shuffle_gen_kwargs(rng, self.kwargs) + formatter = PythonFormatter() + shard_idx_start = self._state_dict["shard_idx"] if self._state_dict else 0 + for gen_kwags in islice( + _split_gen_kwargs(kwargs_with_shuffled_shards, max_num_jobs=self.num_shards), shard_idx_start, None + ): + shard_example_idx_start = self._state_dict["shard_example_idx"] if self._state_dict else 0 + shard_example_idx = 0 + for key, pa_table in self.generate_tables_fn(**gen_kwags): + if shard_example_idx + len(pa_table) <= shard_example_idx_start: + shard_example_idx += len(pa_table) + continue + for pa_subtable in pa_table.to_reader(max_chunksize=config.ARROW_READER_BATCH_SIZE_IN_DATASET_ITER): + formatted_batch = formatter.format_batch(pa_subtable) + for example in _batch_to_examples(formatted_batch): + if shard_example_idx >= shard_example_idx_start: + if self._state_dict: + self._state_dict["shard_example_idx"] += 1 + yield key, example + shard_example_idx += 1 + if self._state_dict: + self._state_dict["shard_idx"] += 1 + self._state_dict["shard_example_idx"] = 0 + + def _iter_arrow(self): + rng = deepcopy(self.generator) + kwargs_with_shuffled_shards = _shuffle_gen_kwargs(rng, self.kwargs) + shard_idx_start = self._state_dict["shard_idx"] if self._state_dict else 0 + for gen_kwags in islice( + _split_gen_kwargs(kwargs_with_shuffled_shards, max_num_jobs=self.num_shards), shard_idx_start, None + ): + shard_example_idx_start = self._state_dict["shard_example_idx"] if self._state_dict else 0 + shard_example_idx = 0 + for key, pa_table in self.generate_tables_fn(**gen_kwags): + shard_example_idx += len(pa_table) + if shard_example_idx <= shard_example_idx_start: + continue + if self._state_dict: + self._state_dict["shard_example_idx"] += len(pa_table) + yield key, pa_table + if self._state_dict: + self._state_dict["shard_idx"] += 1 + self._state_dict["shard_example_idx"] = 0 + + def shard_data_sources(self, num_shards: int, index: int, contiguous=True) -> "ArrowExamplesIterable": + """Keep only the requested shard.""" + rng = deepcopy(self.generator) + kwargs_with_shuffled_shards = _shuffle_gen_kwargs(rng, self.kwargs) + return ArrowExamplesIterable(self.generate_tables_fn, kwargs_with_shuffled_shards).shard_data_sources( + num_shards, index, contiguous=contiguous + ) + + +class RebatchedArrowExamplesIterable(_BaseExamplesIterable): + def __init__(self, ex_iterable: _BaseExamplesIterable, batch_size: Optional[int], drop_last_batch: bool = False): + super().__init__() + self.ex_iterable = ex_iterable + self.batch_size = batch_size + self.drop_last_batch = drop_last_batch + + @property + def iter_arrow(self): + return self._iter_arrow + + @property + def is_typed(self): + return self.ex_iterable.is_typed + + @property + def features(self): + return self.ex_iterable.features + + def _init_state_dict(self) -> dict: + self._state_dict = { + "examples_iterable": self.ex_iterable._init_state_dict(), + "previous_state": None, + "batch_idx": 0, + "num_chunks_since_previous_state": 0, + "cropped_chunk_length": 0, + "type": self.__class__.__name__, + } + return self._state_dict + + def __iter__(self): + yield from self.ex_iterable + + def _iter_arrow(self) -> Iterator[tuple[Key, pa.Table]]: + """Iterate over sub-tables of size `batch_size`.""" + if self._state_dict and self._state_dict["previous_state"]: + self.ex_iterable.load_state_dict(self._state_dict["previous_state"]) + if self.ex_iterable.iter_arrow: + iterator = self.ex_iterable.iter_arrow() + else: + iterator = _convert_to_arrow(self.ex_iterable, batch_size=1) + if self.batch_size is None or self.batch_size <= 0: + if self._state_dict and self._state_dict["batch_idx"] > 0: + return + all_pa_table = pa.concat_tables([pa_table for _, pa_table in iterator]) + if self._state_dict: + self._state_dict["batch_idx"] = 1 + yield "all", all_pa_table + return + keys_buffer = [] + chunks_buffer = [] + chunks_buffer_size = 0 + num_chunks_to_skip = self._state_dict["num_chunks_since_previous_state"] if self._state_dict else 0 + chunk_length_to_crop = self._state_dict["cropped_chunk_length"] if self._state_dict else 0 + if self._state_dict: + previous_state = self.ex_iterable.state_dict() + self._state_dict["previous_state"] = previous_state + for key, pa_table in iterator: + for num_chunks_since_previous_state, chunk in enumerate(pa_table.to_reader(max_chunksize=self.batch_size)): + if num_chunks_to_skip > 1: + num_chunks_to_skip -= 1 + continue + elif num_chunks_to_skip == 1 and chunk_length_to_crop == 0: + num_chunks_to_skip -= 1 + continue + elif num_chunks_to_skip == 1 and chunk_length_to_crop > 0: + chunk = chunk.slice(chunk_length_to_crop, len(chunk) - chunk_length_to_crop) + num_chunks_to_skip = 0 + chunk_length_to_crop = 0 + if len(chunk) == 0: + continue + + if chunks_buffer_size + len(chunk) < self.batch_size: + keys_buffer.append(key) + chunks_buffer.append(chunk) + chunks_buffer_size += len(chunk) + continue + elif chunks_buffer_size + len(chunk) == self.batch_size: + keys_buffer.append(key) + chunks_buffer.append(chunk) + new_key = "_".join(str(_key) for _key in keys_buffer) + if self._state_dict: + self._state_dict["batch_idx"] += 1 + self._state_dict["num_chunks_since_previous_state"] += len(chunks_buffer) + self._state_dict["cropped_chunk_length"] = 0 + yield new_key, pa.Table.from_batches(chunks_buffer) + keys_buffer = [] + chunks_buffer = [] + chunks_buffer_size = 0 + if self._state_dict: + self._state_dict["previous_state"] = previous_state + self._state_dict["num_chunks_since_previous_state"] = num_chunks_since_previous_state + 1 + else: + cropped_chunk_length = self.batch_size - chunks_buffer_size + keys_buffer.append(f"{key}[:{cropped_chunk_length}]") + chunks_buffer.append(chunk.slice(0, cropped_chunk_length)) + new_key = "_".join(str(_key) for _key in keys_buffer) + if self._state_dict: + self._state_dict["batch_idx"] += 1 + self._state_dict["num_chunks_since_previous_state"] += len(chunks_buffer) + self._state_dict["cropped_chunk_length"] = cropped_chunk_length + yield new_key, pa.Table.from_batches(chunks_buffer) + keys_buffer = [f"{key}[{cropped_chunk_length}:]"] + chunks_buffer = [chunk.slice(cropped_chunk_length, len(chunk) - cropped_chunk_length)] + chunks_buffer_size = len(chunk) - cropped_chunk_length + if self._state_dict: + self._state_dict["previous_state"] = previous_state + self._state_dict["num_chunks_since_previous_state"] = num_chunks_since_previous_state + if self._state_dict: + previous_state = self.ex_iterable.state_dict() + if not self.drop_last_batch and chunks_buffer: + new_key = "_".join(str(_key) for _key in keys_buffer) + if self._state_dict: + self._state_dict["previous_state"] = previous_state + self._state_dict["batch_idx"] += 1 + self._state_dict["num_chunks_since_previous_state"] = 0 + self._state_dict["cropped_chunk_length"] = 0 + yield new_key, pa.Table.from_batches(chunks_buffer) + + def shuffle_data_sources(self, generator: np.random.Generator) -> "RebatchedArrowExamplesIterable": + return RebatchedArrowExamplesIterable( + self.ex_iterable.shuffle_data_sources(generator), self.batch_size, self.drop_last_batch + ) + + def shard_data_sources(self, num_shards: int, index: int, contiguous=True) -> "RebatchedArrowExamplesIterable": + return RebatchedArrowExamplesIterable( + self.ex_iterable.shard_data_sources(num_shards, index, contiguous=contiguous), + self.batch_size, + self.drop_last_batch, + ) + + @property + def num_shards(self) -> int: + return self.ex_iterable.num_shards + + +class SelectColumnsIterable(_BaseExamplesIterable): + def __init__(self, ex_iterable: _BaseExamplesIterable, column_names: list[str]): + super().__init__() + self.ex_iterable = ex_iterable + self.column_names = column_names + + @property + def iter_arrow(self): + if self.ex_iterable.iter_arrow: + return self._iter_arrow + + @property + def is_typed(self): + return self.ex_iterable.is_typed + + @property + def features(self): + return self.ex_iterable.features + + def _init_state_dict(self) -> dict: + self._state_dict = self.ex_iterable._init_state_dict() + return self._state_dict + + def __iter__(self): + for idx, row in self.ex_iterable: + yield idx, {c: row[c] for c in self.column_names} + + def _iter_arrow(self) -> Iterator[tuple[Key, pa.Table]]: + for idx, pa_table in self.ex_iterable.iter_arrow(): + if len(pa_table) > 0: # empty tables have no schema + yield idx, pa_table.select(self.column_names) + + def shuffle_data_sources(self, generator: np.random.Generator) -> "SelectColumnsIterable": + return SelectColumnsIterable(self.ex_iterable.shuffle_data_sources(generator), self.column_names) + + def shard_data_sources(self, num_shards: int, index: int, contiguous=True) -> "SelectColumnsIterable": + return SelectColumnsIterable( + self.ex_iterable.shard_data_sources(num_shards, index, contiguous=contiguous), self.column_names + ) + + @property + def num_shards(self) -> int: + return self.ex_iterable.num_shards + + +class StepExamplesIterable(_BaseExamplesIterable): + def __init__(self, ex_iterable: _BaseExamplesIterable, step: int, offset: int): + super().__init__() + self.ex_iterable = ex_iterable + self.step = step + self.offset = offset + # TODO(QL): implement iter_arrow + + @property + def is_typed(self): + return self.ex_iterable.is_typed + + @property + def features(self): + return self.ex_iterable.features + + def _init_state_dict(self) -> dict: + self._state_dict = self.ex_iterable._init_state_dict() + return self._state_dict + + def __iter__(self): + ex_iterator = iter(self.ex_iterable) + while True: + batch = list(islice(ex_iterator, self.step)) + if len(batch) > self.offset: + yield batch[self.offset] + else: + break + + def shuffle_data_sources(self, generator: np.random.Generator) -> "StepExamplesIterable": + return StepExamplesIterable( + self.ex_iterable.shuffle_data_sources(generator), step=self.step, offset=self.offset + ) + + def shard_data_sources(self, num_shards: int, index: int, contiguous=True) -> "StepExamplesIterable": + return StepExamplesIterable( + self.ex_iterable.shard_data_sources(num_shards, index, contiguous=contiguous), + step=self.step, + offset=self.offset, + ) + + @property + def num_shards(self) -> int: + return self.ex_iterable.num_shards + + +class CyclingMultiSourcesExamplesIterable(_BaseExamplesIterable): + def __init__( + self, + ex_iterables: list[_BaseExamplesIterable], + stopping_strategy: Literal["first_exhausted", "all_exhausted"] = "first_exhausted", + ): + super().__init__() + self.ex_iterables = ex_iterables + self.stopping_strategy = stopping_strategy + + # if undersampling ("first_exhausted"), we stop as soon as one dataset is exhausted + # if oversampling ("all_exhausted"), we stop as soons as every dataset is exhausted, i.e as soon as every samples of every dataset has been visited at least once + self.bool_strategy_func = np.all if (stopping_strategy == "all_exhausted") else np.any + # TODO(QL): implement iter_arrow + + @property + def is_typed(self): + return self.ex_iterables[0].is_typed + + @property + def features(self): + return self.ex_iterables[0].features + + def _get_indices_iterator(self): + # this is an infinite iterator to keep track of which iterator we want to pick examples from + ex_iterable_idx = self._state_dict["ex_iterable_idx"] if self._state_dict else 0 + for next_ex_iterable_idx in islice(cycle(range(len(self.ex_iterables))), ex_iterable_idx + 1, None): + if self._state_dict: + self._state_dict["ex_iterable_idx"] = next_ex_iterable_idx + yield ex_iterable_idx + ex_iterable_idx = next_ex_iterable_idx + + def _init_state_dict(self) -> dict: + self._state_dict = { + "ex_iterable_idx": 0, + "ex_iterables": [ex_iterable._init_state_dict() for ex_iterable in self.ex_iterables], + "previous_states": [None] * len(self.ex_iterables), + "is_exhausted": [False] * len(self.ex_iterables), + "type": self.__class__.__name__, + } + return self._state_dict + + def __iter__(self): + # we use this to buffer one example of each iterator to know if an iterator is exhausted + nexts = [None] * len(self.ex_iterables) + # because of that, we need to rewind 1 example when reloading the state dict + if self._state_dict: + for i in range(len(self.ex_iterables)): + if self._state_dict["previous_states"][i] is not None: + self.ex_iterables[i].load_state_dict(self._state_dict["previous_states"][i]) + iterators = [iter(ex_iterable) for ex_iterable in self.ex_iterables] + + indices_iterator = self._get_indices_iterator() + + is_exhausted = ( + np.array(self._state_dict["is_exhausted"]) if self._state_dict else np.full(len(self.ex_iterables), False) + ) + for i in indices_iterator: + # if the stopping criteria is met, break the main for loop + if self.bool_strategy_func(is_exhausted): + break + # let's pick one example from the iterator at index i + if nexts[i] is None: + nexts[i] = next(iterators[i], False) + result = nexts[i] + if self._state_dict: + self._state_dict["previous_states"][i] = deepcopy(self._state_dict["ex_iterables"][i]) + nexts[i] = next(iterators[i], False) + + # the iterator is exhausted + if nexts[i] is False: + is_exhausted[i] = True + if self._state_dict: + self._state_dict["is_exhausted"][i] = True + # we reset it in case the stopping crtieria isn't met yet + nexts[i] = None + if self._state_dict: + self._state_dict["ex_iterables"][i] = self.ex_iterables[i]._init_state_dict() + self._state_dict["previous_states"][i] = None + iterators[i] = iter(self.ex_iterables[i]) + + if result is not False: + yield result + + def shuffle_data_sources(self, generator: np.random.Generator) -> "CyclingMultiSourcesExamplesIterable": + """Shuffle each underlying examples iterable.""" + ex_iterables = [ex_iterable.shuffle_data_sources(generator) for ex_iterable in self.ex_iterables] + return CyclingMultiSourcesExamplesIterable(ex_iterables, self.stopping_strategy) + + @property + def num_shards(self) -> int: + return min(ex_iterable.num_shards for ex_iterable in self.ex_iterables) + + def shard_data_sources( + self, num_shards: int, index: int, contiguous=True + ) -> "CyclingMultiSourcesExamplesIterable": + """Either keep only the requested shard, or propagate the request to the underlying iterable.""" + return CyclingMultiSourcesExamplesIterable( + [iterable.shard_data_sources(num_shards, index, contiguous=contiguous) for iterable in self.ex_iterables], + stopping_strategy=self.stopping_strategy, + ) + + +class VerticallyConcatenatedMultiSourcesExamplesIterable(_BaseExamplesIterable): + """ + VerticallyConcatenatedMultiSourcesExamplesIterable simply chains the input iterables. + It doesn't require the examples iterables to always yield the same columns. + Instead, this is handled by the `IterableDataset` class or `FormattedExamplesIterable`. + + For information, `IterableDataset` merges the features of all the datasets to concatenate into one. + We use `IterableDataset._resolve_features` to obtain the features of all the datasets to concatenate. + + Then for each example, `IterableDataset` and `FormattedExamplesIterable` automatically fill missing columns with None. + This is done with `_apply_feature_types_on_example`. + """ + + def __init__(self, ex_iterables: list[_BaseExamplesIterable]): + super().__init__() + self.ex_iterables = ex_iterables + + @property + def is_typed(self): + return self.ex_iterables[0].is_typed + + @property + def features(self): + return self.ex_iterables[0].features + + @property + def iter_arrow(self): + if all(ex_iterable.iter_arrow is not None for ex_iterable in self.ex_iterables): + return self._iter_arrow + + def _init_state_dict(self) -> dict: + self._state_dict = { + "ex_iterable_idx": 0, + "ex_iterables": [ex_iterable._init_state_dict() for ex_iterable in self.ex_iterables], + "type": self.__class__.__name__, + } + return self._state_dict + + def __iter__(self): + ex_iterable_idx_start = self._state_dict["ex_iterable_idx"] if self._state_dict else 0 + for ex_iterable in islice(self.ex_iterables, ex_iterable_idx_start, None): + yield from ex_iterable + if self._state_dict: + self._state_dict["ex_iterable_idx"] += 1 + + def _iter_arrow(self): + ex_iterable_idx_start = self._state_dict["ex_iterable_idx"] if self._state_dict else 0 + for ex_iterable in islice(self.ex_iterables, ex_iterable_idx_start, None): + yield from ex_iterable.iter_arrow() + if self._state_dict: + self._state_dict["ex_iterable_idx"] += 1 + + def shuffle_data_sources( + self, generator: np.random.Generator + ) -> "VerticallyConcatenatedMultiSourcesExamplesIterable": + """Shuffle the list of examples iterable, as well as each underlying examples iterable.""" + rng = deepcopy(generator) + ex_iterables = list(self.ex_iterables) + rng.shuffle(ex_iterables) + ex_iterables = [ex_iterable.shuffle_data_sources(generator) for ex_iterable in ex_iterables] + return VerticallyConcatenatedMultiSourcesExamplesIterable(ex_iterables) + + @property + def num_shards(self) -> int: + return min(ex_iterable.num_shards for ex_iterable in self.ex_iterables) + + def shard_data_sources( + self, num_shards: int, index: int, contiguous=True + ) -> "VerticallyConcatenatedMultiSourcesExamplesIterable": + """Either keep only the requested shard, or propagate the request to the underlying iterable.""" + return VerticallyConcatenatedMultiSourcesExamplesIterable( + [iterable.shard_data_sources(num_shards, index, contiguous=contiguous) for iterable in self.ex_iterables] + ) + + +def _check_column_names(column_names: list[str]): + """Check the column names to make sure they don't contain duplicates.""" + counter = Counter(column_names) + if not all(count == 1 for count in counter.values()): + duplicated_columns = [col for col in counter if counter[col] > 1] + raise ValueError( + f"The examples iterables can't have duplicated columns but columns {duplicated_columns} are duplicated." + ) + + +class HorizontallyConcatenatedMultiSourcesExamplesIterable(_BaseExamplesIterable): + """ + HorizontallyConcatenatedMultiSourcesExamplesIterable merges examples together for the input list of iterables. + It also checks that there are no duplicate columns (otherwise we don't know which one to keep). + This check is done once when yielding the first example. + + However it doesn't fill missing columns with None. + Instead, this is handled by the `IterableDataset` class or `FormattedExamplesIterable`. + + For information, `IterableDataset` merges the features of all the datasets to concatenate into one. + We use `IterableDataset._resolve_features` to obtain the features of all the datasets to concatenate. + + Then for each example, `IterableDataset` and `FormattedExamplesIterable` automatically fill missing columns with None. + This is done with `_apply_feature_types_on_example`. + """ + + def __init__(self, ex_iterables: list[_BaseExamplesIterable]): + super().__init__() + self.ex_iterables = ex_iterables + # TODO(QL): implement iter_arrow + + @property + def is_typed(self): + return self.ex_iterables[0].is_typed + + @property + def features(self): + return self.ex_iterables[0].features + + def _init_state_dict(self) -> dict: + self._state_dict = { + "ex_iterables": [ex_iterable._init_state_dict() for ex_iterable in self.ex_iterables], + "type": self.__class__.__name__, + } + return self._state_dict + + def __iter__(self): + ex_iterators = [iter(ex_iterable) for ex_iterable in self.ex_iterables] + for i in itertools.count(): + keys = [] + examples = [] + for ex_iterator in list(ex_iterators): + try: + key, example = next(ex_iterator) + keys.append(key) + examples.append(example) + except StopIteration: + ex_iterators.remove(ex_iterator) + if ex_iterators: + if i == 0: + _check_column_names([column_name for example in examples for column_name in example]) + new_example = {} + for example in examples: + new_example.update(example) + new_key = "_".join(str(key) for key in keys) + yield new_key, new_example + else: + break + + def shuffle_data_sources( + self, generator: np.random.Generator + ) -> "HorizontallyConcatenatedMultiSourcesExamplesIterable": + """Doesn't shuffle the wrapped examples iterable since it would break the alignment between them.""" + return self + + @property + def num_shards(self) -> int: + return 1 + + def shard_data_sources( + self, num_shards: int, index: int, contiguous=True + ) -> "HorizontallyConcatenatedMultiSourcesExamplesIterable": + """Either keep only the requested shard, or propagate the request to the underlying iterable.""" + return HorizontallyConcatenatedMultiSourcesExamplesIterable( + [iterable.shard_data_sources(num_shards, index, contiguous=contiguous) for iterable in self.ex_iterables] + ) + + +class RandomlyCyclingMultiSourcesExamplesIterable(CyclingMultiSourcesExamplesIterable): + def __init__( + self, + ex_iterables: list[_BaseExamplesIterable], + generator: np.random.Generator, + probabilities: Optional[list[float]] = None, + stopping_strategy: Literal["first_exhausted", "all_exhausted"] = "first_exhausted", + ): + super().__init__(ex_iterables, stopping_strategy) + self.generator = deepcopy(generator) + self.probabilities = probabilities + # TODO(QL): implement iter_arrow + + @property + def is_typed(self): + return self.ex_iterables[0].is_typed + + @property + def features(self): + return self.ex_iterables[0].features + + def _get_indices_iterator(self): + rng = deepcopy(self.generator) + num_sources = len(self.ex_iterables) + random_batch_size = 1000 + # this is an infinite iterator that randomly samples the index of the source to pick examples from + index_offset = self._state_dict["bit_generator_index_offset"] if self._state_dict else 0 + if self._state_dict: + rng.bit_generator.state = self._state_dict["bit_generator_state"] + if self.probabilities is None: + while True: + for i in islice(rng.integers(0, num_sources, size=random_batch_size), index_offset, None): + index_offset = (index_offset + 1) % random_batch_size + if self._state_dict: + self._state_dict["bit_generator_index_offset"] = index_offset + if index_offset == 0: + self._state_dict["bit_generator_state"] = rng.bit_generator.state + yield int(i) + else: + while True: + for i in islice( + rng.choice(num_sources, size=random_batch_size, p=self.probabilities), index_offset, None + ): + index_offset = (index_offset + 1) % random_batch_size + if self._state_dict: + self._state_dict["bit_generator_index_offset"] = index_offset + if index_offset == 0: + self._state_dict["bit_generator_state"] = rng.bit_generator.state + yield int(i) + + def _init_state_dict(self) -> dict: + self._state_dict = { + "bit_generator_state": self.generator.bit_generator.state, + "bit_generator_index_offset": 0, + "ex_iterables": [ex_iterable._init_state_dict() for ex_iterable in self.ex_iterables], + "previous_states": [None] * len(self.ex_iterables), + "is_exhausted": [False] * len(self.ex_iterables), + "type": self.__class__.__name__, + } + return self._state_dict + + def shuffle_data_sources(self, generator: np.random.Generator) -> "RandomlyCyclingMultiSourcesExamplesIterable": + """Shuffle the data sources of each wrapped examples iterable.""" + ex_iterables = [ex_iterable.shuffle_data_sources(generator) for ex_iterable in self.ex_iterables] + return RandomlyCyclingMultiSourcesExamplesIterable( + ex_iterables, + generator=generator, + probabilities=self.probabilities, + stopping_strategy=self.stopping_strategy, + ) + + def shard_data_sources( + self, num_shards: int, index: int, contiguous=True + ) -> "RandomlyCyclingMultiSourcesExamplesIterable": + """Either keep only the requested shard, or propagate the request to the underlying iterable.""" + return RandomlyCyclingMultiSourcesExamplesIterable( + [iterable.shard_data_sources(num_shards, index, contiguous=contiguous) for iterable in self.ex_iterables], + self.generator, + self.probabilities, + self.stopping_strategy, + ) + + +def _table_output_to_arrow(output) -> pa.Table: + if isinstance(output, pa.Table): + return output + if isinstance(output, (pd.DataFrame, pd.Series)): + return pa.Table.from_pandas(output) + if config.POLARS_AVAILABLE and "polars" in sys.modules: + import polars as pl + + if isinstance(output, (pl.DataFrame, pl.Series)): + return output.to_arrow() + return output + + +class MappedExamplesIterable(_BaseExamplesIterable): + def __init__( + self, + ex_iterable: _BaseExamplesIterable, + function: Callable, + with_indices: bool = False, + input_columns: Optional[list[str]] = None, + batched: bool = False, + batch_size: Optional[int] = 1000, + drop_last_batch: bool = False, + remove_columns: Optional[list[str]] = None, + fn_kwargs: Optional[dict] = None, + formatting: Optional["FormattingConfig"] = None, + features: Optional[Features] = None, + max_num_running_async_map_functions_in_parallel: Optional[int] = None, + ): + super().__init__() + self.ex_iterable = ex_iterable + self.function = function + self.batched = batched + self.batch_size = batch_size + self.drop_last_batch = drop_last_batch + self.remove_columns = remove_columns + self.with_indices = with_indices + self.input_columns = input_columns + self.fn_kwargs = fn_kwargs or {} + self.formatting = formatting # required for iter_arrow + self._features = features + self.max_num_running_async_map_functions_in_parallel = ( + max_num_running_async_map_functions_in_parallel or config.MAX_NUM_RUNNING_ASYNC_MAP_FUNCTIONS_IN_PARALLEL + ) + # sanity checks + if formatting and formatting.is_table: + # batch_size should match for iter_arrow + if not isinstance(ex_iterable, RebatchedArrowExamplesIterable): + raise ValueError( + f"The {formatting.format_type.capitalize()}-formatted {type(self).__name__} has underlying iterable" + f"that is a {type(ex_iterable).__name__} instead of a RebatchedArrowExamplesIterable." + ) + elif ex_iterable.batch_size != (batch_size if batched else 1): + raise ValueError( + f"The {formatting.format_type.capitalize()}-formatted {type(self).__name__} has batch_size={batch_size if batched else 1} which is" + f"different from {ex_iterable.batch_size=} from its underlying iterable." + ) + # to enable graceful ends + self._owned_loops_and_tasks: list[tuple[asyncio.AbstractEventLoop, list[asyncio.Task]]] = [] + + @property + def iter_arrow(self): + if self.formatting and self.formatting.is_table: + return self._iter_arrow + + @property + def is_typed(self): + return self.features is not None # user has extracted features + + @property + def features(self): + return self._features + + def _init_state_dict(self) -> dict: + self._state_dict = { + "examples_iterable": self.ex_iterable._init_state_dict(), + "previous_state": None, + "num_examples_since_previous_state": 0, + "previous_state_example_idx": 0, + "type": self.__class__.__name__, + } + return self._state_dict + + def __iter__(self): + if self.formatting and self.formatting.is_table: + formatter = PythonFormatter() + for key, pa_table in self._iter_arrow(max_chunksize=1): + yield key, formatter.format_row(pa_table) + else: + yield from self._iter() + + def _iter(self): + current_idx = self._state_dict["previous_state_example_idx"] if self._state_dict else 0 + if self._state_dict and self._state_dict["previous_state"]: + self.ex_iterable.load_state_dict(self._state_dict["previous_state"]) + num_examples_to_skip = self._state_dict["num_examples_since_previous_state"] + else: + num_examples_to_skip = 0 + iterator = iter(self.ex_iterable) + + # We use the same logic as in Dataset.map, but with less features/formatting + # since they're handled by FormattedExamplesIterable + + if self.formatting: + formatter = get_formatter(self.formatting.format_type) + format_dict = formatter.recursive_tensorize if isinstance(formatter, TensorFormatter) else None + else: + format_dict = None + + def iter_batched_inputs(): + nonlocal current_idx + for key, example in iterator: + # If `batched`, first build the batch, if `batch_size` is None or <=0, then the batch is the whole dataset + iterator_batch = ( + iterator + if self.batch_size is None or self.batch_size <= 0 + else islice(iterator, self.batch_size - 1) + ) + key_examples_list = [(key, example)] + list(iterator_batch) + keys, examples = zip(*key_examples_list) + # the new key is the concatenation of the examples keys from the batch + key = "_".join(str(key) for key in keys) + if ( + self.drop_last_batch + and self.batch_size is not None + and self.batch_size > 0 + and len(examples) < self.batch_size + ): # ignore last batch + return + batch = _examples_to_batch(examples) + # we need to format here in case we need to stack tensors together + batch = format_dict(batch) if format_dict else batch + indices = [current_idx + i for i in range(len(key_examples_list))] + current_idx += len(indices) + yield indices, (key, batch) + + def iter_inputs(): + nonlocal current_idx + for key, example in iterator: + # If not batched, we can apply the transform and yield the example directly + # first copy the example, since we might drop some keys + example = dict(example) + # no need to do formatting here + current_idx += 1 + yield current_idx - 1, (key, example) + + def validate_function_output(processed_inputs): + if self.batched and processed_inputs: + first_col = next(iter(processed_inputs)) + bad_cols = [ + col for col in processed_inputs if len(processed_inputs[col]) != len(processed_inputs[first_col]) + ] + if bad_cols: + raise ValueError( + f"Column lengths mismatch: columns {bad_cols} have length {[len(processed_inputs[col]) for col in bad_cols]} " + f"while {first_col} has length {len(processed_inputs[first_col])}." + ) + + def prepare_inputs(key_example, indices): + key, example = key_example + fn_args = [example] if self.input_columns is None else [example[col] for col in self.input_columns] + additional_args = () + if self.with_indices: + fn_args += (indices,) + inputs = dict(example) + return inputs, fn_args, additional_args, self.fn_kwargs + + def prepare_outputs(key_example, inputs, processed_inputs): + validate_function_output(processed_inputs) + # this logic mimics the one in Dataset.map + if self.remove_columns: + for c in self.remove_columns: + if c in inputs: + del inputs[c] + if processed_inputs is key_example[1] and c in processed_inputs: + del processed_inputs[c] + transformed_inputs = {**inputs, **processed_inputs} + # no need to do features decoding here + return transformed_inputs + + def apply_function(key_example, indices): + """Utility to apply the function on a selection of columns.""" + inputs, fn_args, additional_args, fn_kwargs = prepare_inputs(key_example, indices) + processed_inputs = self.function(*fn_args, *additional_args, **fn_kwargs) + return prepare_outputs(key_example, inputs, processed_inputs) + + async def async_apply_function(key_example, indices): + """Utility to apply the function on a selection of columns. Same code but async""" + inputs, fn_args, additional_args, fn_kwargs = prepare_inputs(key_example, indices) + processed_inputs = await self.function(*fn_args, *additional_args, **fn_kwargs) + return prepare_outputs(key_example, inputs, processed_inputs) + + tasks: list[asyncio.Task] = [] + if inspect.iscoroutinefunction(self.function): + try: + loop = asyncio.get_running_loop() + except RuntimeError: + loop = asyncio.new_event_loop() + self._owned_loops_and_tasks.append((loop, tasks)) + else: + loop = None + + def iter_outputs(): + nonlocal tasks, loop + inputs_iterator = iter_batched_inputs() if self.batched else iter_inputs() + if inspect.iscoroutinefunction(self.function): + if self._state_dict: + previous_state = self.ex_iterable.state_dict() + self._state_dict["previous_state"] = previous_state + previous_state_task = None + previous_state_example_idx = self._state_dict["previous_state_example_idx"] + indices: Union[list[int], list[list[int]]] = [] + for i, key_example in inputs_iterator: + indices.append(i) + tasks.append(loop.create_task(async_apply_function(key_example, i))) + # keep the total active tasks under a certain number + if len(tasks) >= self.max_num_running_async_map_functions_in_parallel: + done, pending = loop.run_until_complete( + asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED) + ) + while tasks and len(pending) >= self.max_num_running_async_map_functions_in_parallel: + done, pending = loop.run_until_complete( + asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED) + ) + if len(tasks) >= 10 * self.max_num_running_async_map_functions_in_parallel: + loop.run_until_complete(tasks[0]) + # yield finished tasks + while tasks and tasks[0].done(): + i, task = indices.pop(0), tasks.pop(0) + yield i, task.result() + if self._state_dict and task is previous_state_task: + self._state_dict["previous_state"] = previous_state + self._state_dict["num_examples_since_previous_state"] = 0 + self._state_dict["previous_state_example_idx"] = previous_state_example_idx + previous_state, previous_state_task = None, None + # checkpoint + if self._state_dict and previous_state_task is None and tasks: + previous_state = self.ex_iterable.state_dict() + previous_state_task = tasks[-1] + previous_state_example_idx = current_idx + while tasks: + yield indices[0], loop.run_until_complete(tasks[0]) + indices.pop(0), tasks.pop(0) + else: + if self._state_dict: + if self.batched: + self._state_dict["previous_state"] = self.ex_iterable.state_dict() + self._state_dict["num_examples_since_previous_state"] = 0 + self._state_dict["previous_state_example_idx"] = current_idx + for i, key_example in inputs_iterator: + if self._state_dict: + if not self.batched: + self._state_dict["previous_state_example_idx"] = current_idx + yield i, apply_function(key_example, i) + if self._state_dict: + if self.batched: + self._state_dict["previous_state"] = self.ex_iterable.state_dict() + self._state_dict["num_examples_since_previous_state"] = 0 + self._state_dict["previous_state_example_idx"] = current_idx + + try: + outputs = iter_outputs() + if self.batched: + outputs = ( + (key, transformed_example) + for key, transformed_batch in outputs + for transformed_example in _batch_to_examples(transformed_batch) + ) + for key, transformed_example in outputs: + if self._state_dict and self._state_dict["previous_state"] is not None: + self._state_dict["num_examples_since_previous_state"] += 1 + if num_examples_to_skip > 0: + num_examples_to_skip -= 1 + continue + yield key, transformed_example + except (Exception, KeyboardInterrupt): + if loop: + logger.debug(f"Canceling {len(tasks)} async tasks.") + for task in tasks: + task.cancel(msg="KeyboardInterrupt") + try: + loop.run_until_complete(asyncio.gather(*tasks)) + except (asyncio.CancelledError, ValueError): + logger.debug("Tasks canceled.") + raise + + def _iter_arrow(self, max_chunksize: Optional[int] = None) -> Iterator[tuple[Key, pa.Table]]: + formatter: TableFormatter = get_formatter(self.formatting.format_type) if self.formatting else ArrowFormatter() + if self.ex_iterable.iter_arrow: + iterator = self.ex_iterable.iter_arrow() + else: + iterator = _convert_to_arrow( + self.ex_iterable, + batch_size=self.batch_size if self.batched else 1, + drop_last_batch=self.drop_last_batch, + ) + if self._state_dict and self._state_dict["previous_state"]: + self.ex_iterable.load_state_dict(self._state_dict["previous_state"]) + num_examples_to_skip = self._state_dict["num_examples_since_previous_state"] + else: + num_examples_to_skip = 0 + if self._state_dict and max_chunksize is not None: + self._state_dict["previous_state"] = self.ex_iterable.state_dict() + self._state_dict["num_examples_since_previous_state"] = 0 + current_idx = self._state_dict["previous_state_example_idx"] if self._state_dict else 0 + for key, pa_table in iterator: + if ( + self.batched + and self.batch_size is not None + and len(pa_table) < self.batch_size + and self.drop_last_batch + ): + return + # first build the batch + function_args = ( + [formatter.format_batch(pa_table)] + if self.input_columns is None + else [pa_table[col] for col in self.input_columns] + ) + if self.with_indices: + if self.batched: + function_args.append([current_idx + i for i in range(len(pa_table))]) + else: + function_args.append(current_idx) + # then apply the transform + output = self.function(*function_args, **self.fn_kwargs) + output_table = _table_output_to_arrow(output) + if not isinstance(output_table, pa.Table): + raise TypeError( + f"Provided `function` which is applied to {formatter.table_type} returns a variable of type " + f"{type(output)}. Make sure provided `function` returns a {formatter.table_type} to update the dataset." + ) + # we don't need to merge results for consistency with Dataset.map which merges iif both input and output are dicts + # then remove the unwanted columns + if self.remove_columns: + for column in self.remove_columns: + if column in output_table.column_names: + output_table = output_table.remove_column(output_table.column_names.index(column)) + # return output + if max_chunksize is None: + current_idx += len(pa_table) + if self._state_dict: + self._state_dict["previous_state_example_idx"] += len(pa_table) + yield key, output_table + else: + for i, pa_subtable in enumerate(output_table.to_reader(max_chunksize=max_chunksize)): + current_idx += 1 + if self._state_dict: + self._state_dict["num_examples_since_previous_state"] += 1 + if num_examples_to_skip > 0: + num_examples_to_skip -= 1 + continue + yield f"{key}_{i}", pa_subtable + if self._state_dict: + self._state_dict["previous_state"] = self.ex_iterable.state_dict() + self._state_dict["num_examples_since_previous_state"] = 0 + self._state_dict["previous_state_example_idx"] += len(pa_table) + + def shuffle_data_sources(self, generator: np.random.Generator) -> "MappedExamplesIterable": + """Shuffle the wrapped examples iterable.""" + return MappedExamplesIterable( + self.ex_iterable.shuffle_data_sources(generator), + function=self.function, + with_indices=self.with_indices, + input_columns=self.input_columns, + batched=self.batched, + batch_size=self.batch_size, + drop_last_batch=self.drop_last_batch, + remove_columns=self.remove_columns, + fn_kwargs=self.fn_kwargs, + formatting=self.formatting, + features=self.features, + max_num_running_async_map_functions_in_parallel=self.max_num_running_async_map_functions_in_parallel, + ) + + def shard_data_sources(self, num_shards: int, index: int, contiguous=True) -> "MappedExamplesIterable": + """Keep only the requested shard.""" + return MappedExamplesIterable( + self.ex_iterable.shard_data_sources(num_shards, index, contiguous=contiguous), + function=self.function, + with_indices=self.with_indices, + input_columns=self.input_columns, + batched=self.batched, + batch_size=self.batch_size, + drop_last_batch=self.drop_last_batch, + remove_columns=self.remove_columns, + fn_kwargs=self.fn_kwargs, + formatting=self.formatting, + features=self.features, + max_num_running_async_map_functions_in_parallel=self.max_num_running_async_map_functions_in_parallel, + ) + + @property + def num_shards(self) -> int: + return self.ex_iterable.num_shards + + +def _add_mask( + input: Union[dict, pa.Table], + mask: Union[bool, list, pa.Array, pa.ChunkedArray, pa.BooleanScalar], + mask_column_name: str, +): + if isinstance(input, pa.Table): + if not isinstance(mask, (list, pa.Array, pa.ChunkedArray)): + mask = pa.array([mask], type=pa.bool_()) + return input.append_column(mask_column_name, mask) + else: + return {mask_column_name: mask} + + +def add_mask(mask_function: Callable, input: Union[dict, pa.Table], *args, mask_column_name: str, **kwargs): + mask = mask_function(input, *args, **kwargs) + return _add_mask(input, mask, mask_column_name) + + +async def async_add_mask( + mask_function: Callable, input: Union[dict, pa.Table], *args, mask_column_name: str, **kwargs +): + mask = await mask_function(input, *args, **kwargs) + return _add_mask(input, mask, mask_column_name) + + +class FilteredExamplesIterable(MappedExamplesIterable): + mask_column_name = "===MASK===" + + def __init__( + self, + ex_iterable: _BaseExamplesIterable, + function: Callable, + with_indices: bool = False, + input_columns: Optional[list[str]] = None, + batched: bool = False, + batch_size: Optional[int] = 1000, + fn_kwargs: Optional[dict] = None, + formatting: Optional["FormattingConfig"] = None, + ): + self.mask_function = function + if ex_iterable.is_typed: + features = Features({**ex_iterable.features, self.mask_column_name: Value("bool")}) + else: + features = None + super().__init__( + ex_iterable=ex_iterable, + function=partial( + async_add_mask if inspect.iscoroutinefunction(function) else add_mask, + function, + mask_column_name=self.mask_column_name, + ), + with_indices=with_indices, + input_columns=input_columns, + batched=batched, + batch_size=batch_size, + fn_kwargs=fn_kwargs, + formatting=formatting, + features=features, + ) + + def _iter(self): + for key, example in super()._iter(): + example = dict(example) + if example.pop(self.mask_column_name): + yield key, example + + def _iter_arrow(self, max_chunksize: Optional[int] = None): + for key, pa_table in super()._iter_arrow(max_chunksize=max_chunksize): + mask = pa_table[self.mask_column_name] + yield key, pa_table.drop(self.mask_column_name).filter(mask) + + def shuffle_data_sources(self, seed: Optional[int]) -> "FilteredExamplesIterable": + """Shuffle the wrapped examples iterable.""" + return FilteredExamplesIterable( + self.ex_iterable.shuffle_data_sources(seed), + function=self.mask_function, + with_indices=self.with_indices, + input_columns=self.input_columns, + batched=self.batched, + batch_size=self.batch_size, + fn_kwargs=self.fn_kwargs, + formatting=self.formatting, + ) + + def shard_data_sources(self, num_shards: int, index: int, contiguous=True) -> "FilteredExamplesIterable": + """Keep only the requested shard.""" + return FilteredExamplesIterable( + self.ex_iterable.shard_data_sources(num_shards, index, contiguous=contiguous), + function=self.mask_function, + with_indices=self.with_indices, + input_columns=self.input_columns, + batched=self.batched, + batch_size=self.batch_size, + fn_kwargs=self.fn_kwargs, + formatting=self.formatting, + ) + + @property + def num_shards(self) -> int: + return self.ex_iterable.num_shards + + +class BufferShuffledExamplesIterable(_BaseExamplesIterable): + def __init__(self, ex_iterable: _BaseExamplesIterable, buffer_size: int, generator: np.random.Generator): + super().__init__() + self.ex_iterable = ex_iterable + self.buffer_size = buffer_size + self.generator = generator + # TODO(QL): implement iter_arrow + + @property + def is_typed(self): + return self.ex_iterable.is_typed + + @property + def features(self): + return self.ex_iterable.features + + def _init_state_dict(self) -> dict: + self._state_dict = self.ex_iterable._init_state_dict() + self._original_state_dict = self.state_dict() + return self._state_dict + + def load_state_dict(self, state_dict: dict) -> dict: + if self._state_dict: + if state_dict != self._original_state_dict: + logger.warning( + "Loading a state dict of a shuffle buffer of a dataset without the buffer content." + "The shuffle buffer will be refilled before starting to yield new examples." + ) + return super().load_state_dict(state_dict) + + @staticmethod + def _iter_random_indices(rng: np.random.Generator, buffer_size: int, random_batch_size=1000) -> Iterator[int]: + while True: + yield from (int(i) for i in rng.integers(0, buffer_size, size=random_batch_size)) + + def __iter__(self): + buffer_size = self.buffer_size + rng = deepcopy(self.generator) + indices_iterator = self._iter_random_indices(rng, buffer_size) + # this is the shuffle buffer that we keep in memory + mem_buffer = [] + for x in self.ex_iterable: + if len(mem_buffer) == buffer_size: # if the buffer is full, pick and example from it + i = next(indices_iterator) + yield mem_buffer[i] + mem_buffer[i] = x # replace the picked example by a new one + else: # otherwise, keep filling the buffer + mem_buffer.append(x) + # when we run out of examples, we shuffle the remaining examples in the buffer and yield them + rng.shuffle(mem_buffer) + yield from mem_buffer + + def shuffle_data_sources(self, generator: np.random.Generator) -> "BufferShuffledExamplesIterable": + """Shuffle the wrapped examples iterable as well as the shuffling buffer.""" + return BufferShuffledExamplesIterable( + self.ex_iterable.shuffle_data_sources(generator), buffer_size=self.buffer_size, generator=generator + ) + + def shard_data_sources(self, num_shards: int, index: int, contiguous=True) -> "BufferShuffledExamplesIterable": + """Keep only the requested shard.""" + return BufferShuffledExamplesIterable( + self.ex_iterable.shard_data_sources(num_shards, index, contiguous=contiguous), + buffer_size=self.buffer_size, + generator=self.generator, + ) + + @property + def num_shards(self) -> int: + return self.ex_iterable.num_shards + + +class SkipExamplesIterable(_BaseExamplesIterable): + def __init__( + self, + ex_iterable: _BaseExamplesIterable, + n: int, + block_sources_order_when_shuffling: bool = True, + split_when_sharding: bool = True, + ): + super().__init__() + self.ex_iterable = ex_iterable + self.n = n + self.block_sources_order_when_shuffling = block_sources_order_when_shuffling + self.split_when_sharding = split_when_sharding + # TODO(QL): implement iter_arrow + + @property + def is_typed(self): + return self.ex_iterable.is_typed + + @property + def features(self): + return self.ex_iterable.features + + def _init_state_dict(self) -> dict: + self._state_dict = { + "skipped": False, + "examples_iterable": self.ex_iterable._init_state_dict(), + "type": self.__class__.__name__, + } + return self._state_dict + + def __iter__(self): + ex_iterable_idx_start = 0 if self._state_dict and self._state_dict["skipped"] else self.n + if self._state_dict: + self._state_dict["skipped"] = True + yield from islice(self.ex_iterable, ex_iterable_idx_start, None) + + @staticmethod + def split_number(num, n): + quotient = num // n + remainder = num % n + result = [quotient] * n + for i in range(remainder): + result[i] += 1 + return result + + def shuffle_data_sources(self, generator: np.random.Generator) -> "SkipExamplesIterable": + """May not shuffle the wrapped examples iterable since it would skip examples from other shards instead.""" + if self.block_sources_order_when_shuffling: + return self + else: + return SkipExamplesIterable( + self.ex_iterable.shuffle_data_sources(generator), + n=self.n, + block_sources_order_when_shuffling=self.block_sources_order_when_shuffling, + split_when_sharding=self.split_when_sharding, + ) + + def shard_data_sources(self, num_shards: int, index: int, contiguous=True) -> "SkipExamplesIterable": + """Keep only the requested shard.""" + if self.split_when_sharding: + return SkipExamplesIterable( + self.ex_iterable.shard_data_sources(num_shards, index, contiguous=contiguous), + n=self.split_number(self.n, num_shards)[index], + block_sources_order_when_shuffling=self.block_sources_order_when_shuffling, + split_when_sharding=self.split_when_sharding, + ) + else: + return self + + @property + def num_shards(self) -> int: + return self.ex_iterable.num_shards + + +class RepeatExamplesIterable(_BaseExamplesIterable): + """ + Iterable that repeats the underlying iterable a given number of times. + """ + + def __init__( + self, + ex_iterable: _BaseExamplesIterable, + num_times: Optional[int], + ): + super().__init__() + self.ex_iterable = ex_iterable + self.num_times = num_times + + def _init_state_dict(self) -> dict: + self._state_dict = { + "repeat_index": 0, + "examples_iterable": self.ex_iterable._init_state_dict(), + "type": self.__class__.__name__, + } + return self._state_dict + + def __iter__(self): + repeat_index = self._state_dict["repeat_index"] if self._state_dict else 0 + while True: + if self.num_times is not None and repeat_index >= max(self.num_times, 0): + break + yield from self.ex_iterable + repeat_index += 1 + if self._state_dict: + self._state_dict["repeat_index"] = repeat_index + self._state_dict["examples_iterable"] = self.ex_iterable._init_state_dict() + + def shuffle_data_sources(self, generator: np.random.Generator) -> "RepeatExamplesIterable": + """Shuffle the underlying iterable, then repeat.""" + return RepeatExamplesIterable(self.ex_iterable.shuffle_data_sources(generator), num_times=self.num_times) + + def shard_data_sources(self, worker_id: int, num_workers: int) -> "RepeatExamplesIterable": + """Shard, then repeat shards.""" + return RepeatExamplesIterable( + self.ex_iterable.shard_data_sources(worker_id, num_workers), + num_times=self.num_times, + ) + + @property + def n_shards(self) -> int: + return self.ex_iterable.n_shards + + +class TakeExamplesIterable(_BaseExamplesIterable): + def __init__( + self, + ex_iterable: _BaseExamplesIterable, + n: int, + block_sources_order_when_shuffling: bool = True, + split_when_sharding: bool = True, + ): + super().__init__() + self.ex_iterable = ex_iterable + self.n = n + self.block_sources_order_when_shuffling = block_sources_order_when_shuffling + self.split_when_sharding = split_when_sharding + # TODO(QL): implement iter_arrow + + @property + def is_typed(self): + return self.ex_iterable.is_typed + + @property + def features(self): + return self.ex_iterable.features + + def _init_state_dict(self) -> dict: + self._state_dict = { + "num_taken": 0, + "examples_iterable": self.ex_iterable._init_state_dict(), + "type": self.__class__.__name__, + } + return self._state_dict + + def __iter__(self): + ex_iterable_num_taken = self._state_dict["num_taken"] if self._state_dict else 0 + for key_example in islice(self.ex_iterable, self.n - ex_iterable_num_taken): + if self._state_dict: + self._state_dict["num_taken"] += 1 + yield key_example + + @staticmethod + def split_number(num, n): + quotient = num // n + remainder = num % n + result = [quotient] * n + for i in range(remainder): + result[i] += 1 + return result + + def shuffle_data_sources(self, generator: np.random.Generator) -> "TakeExamplesIterable": + """May not shuffle the wrapped examples iterable since it would take examples from other shards instead.""" + if self.block_sources_order_when_shuffling: + return self + else: + return TakeExamplesIterable( + self.ex_iterable.shuffle_data_sources(generator), + n=self.n, + block_sources_order_when_shuffling=self.block_sources_order_when_shuffling, + split_when_sharding=self.split_when_sharding, + ) + + def shard_data_sources(self, num_shards: int, index: int, contiguous=True) -> "TakeExamplesIterable": + """Keep only the requested shard.""" + if self.split_when_sharding: + return TakeExamplesIterable( + self.ex_iterable.shard_data_sources(num_shards, index, contiguous=contiguous), + n=self.split_number(self.n, num_shards)[index], + block_sources_order_when_shuffling=self.block_sources_order_when_shuffling, + split_when_sharding=self.split_when_sharding, + ) + else: + return TakeExamplesIterable( + self.ex_iterable.shard_data_sources(num_shards, index, contiguous=contiguous), + n=self.n, + block_sources_order_when_shuffling=self.block_sources_order_when_shuffling, + split_when_sharding=self.split_when_sharding, + ) + + @property + def num_shards(self) -> int: + return self.ex_iterable.num_shards + + +def _apply_feature_types_on_example( + example: dict, features: Features, token_per_repo_id: dict[str, Union[str, bool, None]] +) -> dict: + example = dict(example) + # add missing columns + for column_name in features: + if column_name not in example: + example[column_name] = None + # we encode the example for ClassLabel feature types for example + encoded_example = features.encode_example(example) + # Decode example for Audio feature, e.g. + decoded_example = features.decode_example(encoded_example, token_per_repo_id=token_per_repo_id) + return decoded_example + + +def _apply_feature_types_on_batch( + batch: dict, features: Features, token_per_repo_id: dict[str, Union[str, bool, None]] +) -> dict: + batch = dict(batch) + # add missing columns + n_examples = len(batch[next(iter(batch))]) + for column_name in features: + if column_name not in batch: + batch[column_name] = [None] * n_examples + # we encode the batch for ClassLabel feature types for example + encoded_batch = features.encode_batch(batch) + # Decode batch for Audio feature, e.g. + decoded_batch = features.decode_batch(encoded_batch, token_per_repo_id=token_per_repo_id) + return decoded_batch + + +@dataclass +class FormattingConfig: + format_type: Optional[str] + + @property + def is_table(self) -> bool: + return isinstance(get_formatter(self.format_type), TableFormatter) + + @property + def is_tensor(self) -> bool: + return isinstance(get_formatter(self.format_type), TensorFormatter) + + +class FormattedExamplesIterable(_BaseExamplesIterable): + def __init__( + self, + ex_iterable: _BaseExamplesIterable, + formatting: Optional[FormattingConfig], + features: Optional[Features], + token_per_repo_id: dict[str, Union[str, bool, None]], + ): + super().__init__() + self.ex_iterable = ex_iterable + self._features = features + self.formatting = formatting + self.token_per_repo_id = token_per_repo_id + + @property + def iter_arrow(self): + if self.ex_iterable.iter_arrow and (not self.formatting or self.formatting.is_table): + return self._iter_arrow + + @property + def is_typed(self): + return self.ex_iterable.is_typed or self._features is not None + + @property + def features(self): + return self._features + + def _init_state_dict(self) -> dict: + self._state_dict = self.ex_iterable._init_state_dict() + return self._state_dict + + def __iter__(self): + if not self.formatting or self.formatting.is_table: + formatter = PythonFormatter(features=self._features if not self.ex_iterable.is_typed else None) + else: + formatter = get_formatter( + self.formatting.format_type, + features=self._features if not self.ex_iterable.is_typed else None, + token_per_repo_id=self.token_per_repo_id, + ) + if self.ex_iterable.iter_arrow: + # feature casting (inc column addition) handled within self._iter_arrow() + for key, pa_table in self._iter_arrow(): + batch = formatter.format_batch(pa_table) + for example in _batch_to_examples(batch): + yield key, example + else: + format_dict = ( + formatter.recursive_tensorize + if isinstance(formatter, TensorFormatter) + else None # cast in case features is None + ) + for key, example in self.ex_iterable: + # don't apply feature types if already applied by ex_iterable (e.g. in case of chained with_format) + if self.features and not self.ex_iterable.is_typed: + example = _apply_feature_types_on_example( + example, self.features, token_per_repo_id=self.token_per_repo_id + ) + if format_dict: + example = format_dict(example) + yield key, example + + def _iter_arrow(self) -> Iterator[tuple[Key, pa.Table]]: + if not self.features: + yield from self.ex_iterable._iter_arrow() + for key, pa_table in self.ex_iterable._iter_arrow(): + columns = set(pa_table.column_names) + schema = self.features.arrow_schema + # add missing columns + for column_name in self.features: + if column_name not in columns: + col = pa.NullArray.from_buffers(pa.null(), len(pa_table), [None]) + pa_table = pa_table.append_column(column_name, col) + if pa_table.schema != schema: + pa_table = cast_table_to_features(pa_table, self.features) + yield key, pa_table + + def shuffle_data_sources(self, generator: np.random.Generator) -> "FormattedExamplesIterable": + """Shuffle the wrapped examples iterable.""" + return FormattedExamplesIterable( + self.ex_iterable.shuffle_data_sources(generator), + features=self.features, + token_per_repo_id=self.token_per_repo_id, + formatting=self.formatting, + ) + + def shard_data_sources(self, num_shards: int, index: int, contiguous=True) -> "FormattedExamplesIterable": + """Keep only the requested shard.""" + return FormattedExamplesIterable( + self.ex_iterable.shard_data_sources(num_shards, index, contiguous=contiguous), + features=self.features, + token_per_repo_id=self.token_per_repo_id, + formatting=self.formatting, + ) + + @property + def num_shards(self) -> int: + return self.ex_iterable.num_shards + + +@dataclass +class ShufflingConfig: + generator: np.random.Generator + _original_seed: Optional[int] = None + + +@dataclass +class DistributedConfig: + rank: int + world_size: int + + +def _maybe_add_torch_iterable_dataset_parent_class(cls): + """Add torch.utils.data.IterableDataset as a parent class if 'torch' is available""" + if config.TORCH_AVAILABLE: + import torch.utils.data + + if torch.utils.data.IterableDataset not in cls.__bases__: + cls.__bases__ += (torch.utils.data.IterableDataset,) + + +def _maybe_share_with_torch_persistent_workers(value: Union[int, "torch.Tensor"]) -> Union[int, "torch.Tensor"]: + if config.TORCH_AVAILABLE: + import torch + + if isinstance(value, torch.Tensor): + return value.share_memory_() + else: + return torch.tensor(value).share_memory_() + else: + return value + + +class IterableDataset(DatasetInfoMixin): + """A Dataset backed by an iterable.""" + + def __init__( + self, + ex_iterable: _BaseExamplesIterable, + info: Optional[DatasetInfo] = None, + split: Optional[NamedSplit] = None, + formatting: Optional[FormattingConfig] = None, + shuffling: Optional[ShufflingConfig] = None, + distributed: Optional[DistributedConfig] = None, + token_per_repo_id: Optional[dict[str, Union[str, bool, None]]] = None, + ): + if distributed and distributed.world_size > 1 and shuffling and shuffling._original_seed is None: + raise RuntimeError( + "The dataset doesn't have a fixed random seed across nodes to shuffle and split the list of dataset shards by node. " + "Please pass e.g. `seed=42` in `.shuffle()` to make all the nodes use the same seed. " + ) + + info = info.copy() if info is not None else DatasetInfo() + DatasetInfoMixin.__init__(self, info=info, split=split) + + self._ex_iterable = copy.copy(ex_iterable) + self._formatting = formatting + self._shuffling = shuffling + self._distributed = distributed + self._token_per_repo_id: dict[str, Union[str, bool, None]] = token_per_repo_id or {} + self._epoch: Union[int, "torch.Tensor"] = _maybe_share_with_torch_persistent_workers(0) + self._starting_state_dict: Optional[dict] = None + self._prepare_ex_iterable_for_iteration() # set state_dict + _maybe_add_torch_iterable_dataset_parent_class(self.__class__) # subclass of torch IterableDataset + + def state_dict(self) -> dict: + """Get the current state_dict of the dataset. + It corresponds to the state at the latest example it yielded. + + Resuming returns exactly where the checkpoint was saved except in two cases: + + 1. examples from shuffle buffers are lost when resuming and the buffers are refilled with new data + 2. combinations of `.with_format(arrow)` and batched `.map()` may skip one batch. + + Returns: + `dict` + + Example: + + ```py + >>> from datasets import Dataset, concatenate_datasets + >>> ds = Dataset.from_dict({"a": range(6)}).to_iterable_dataset(num_shards=3) + >>> for idx, example in enumerate(ds): + ... print(example) + ... if idx == 2: + ... state_dict = ds.state_dict() + ... print("checkpoint") + ... break + >>> ds.load_state_dict(state_dict) + >>> print(f"restart from checkpoint") + >>> for example in ds: + ... print(example) + ``` + + which returns: + ``` + {'a': 0} + {'a': 1} + {'a': 2} + checkpoint + restart from checkpoint + {'a': 3} + {'a': 4} + {'a': 5} + ``` + + ```py + >>> from torchdata.stateful_dataloader import StatefulDataLoader + >>> ds = load_dataset("deepmind/code_contests", streaming=True, split="train") + >>> dataloader = StatefulDataLoader(ds, batch_size=32, num_workers=4) + >>> # checkpoint + >>> state_dict = dataloader.state_dict() # uses ds.state_dict() under the hood + >>> # resume from checkpoint + >>> dataloader.load_state_dict(state_dict) # uses ds.load_state_dict() under the hood + ``` + """ + return copy.deepcopy(self._state_dict) + + def load_state_dict(self, state_dict: dict) -> None: + """Load the state_dict of the dataset. + The iteration will restart at the next example from when the state was saved. + + Resuming returns exactly where the checkpoint was saved except in two cases: + + 1. examples from shuffle buffers are lost when resuming and the buffers are refilled with new data + 2. combinations of `.with_format(arrow)` and batched `.map()` may skip one batch. + + Example: + + ```py + >>> from datasets import Dataset, concatenate_datasets + >>> ds = Dataset.from_dict({"a": range(6)}).to_iterable_dataset(num_shards=3) + >>> for idx, example in enumerate(ds): + ... print(example) + ... if idx == 2: + ... state_dict = ds.state_dict() + ... print("checkpoint") + ... break + >>> ds.load_state_dict(state_dict) + >>> print(f"restart from checkpoint") + >>> for example in ds: + ... print(example) + ``` + + which returns: + ``` + {'a': 0} + {'a': 1} + {'a': 2} + checkpoint + restart from checkpoint + {'a': 3} + {'a': 4} + {'a': 5} + ``` + + ```py + >>> from torchdata.stateful_dataloader import StatefulDataLoader + >>> ds = load_dataset("deepmind/code_contests", streaming=True, split="train") + >>> dataloader = StatefulDataLoader(ds, batch_size=32, num_workers=4) + >>> # checkpoint + >>> state_dict = dataloader.state_dict() # uses ds.state_dict() under the hood + >>> # resume from checkpoint + >>> dataloader.load_state_dict(state_dict) # uses ds.load_state_dict() under the hood + ``` + """ + self._starting_state_dict = state_dict + + def __repr__(self): + return f"IterableDataset({{\n features: {list(self._info.features.keys()) if self._info.features is not None else 'Unknown'},\n num_shards: {self.num_shards}\n}})" + + def __getstate__(self): + return self.__dict__ + + def __setstate__(self, d): + self.__dict__ = d + # Re-add torch shared memory, since shared memory is not always kept when pickling + self._epoch = _maybe_share_with_torch_persistent_workers(self._epoch) + # Re-add torch iterable dataset as a parent class, since dynamically added parent classes are not kept when pickling + _maybe_add_torch_iterable_dataset_parent_class(self.__class__) + + def _head(self, n=5): + return next(iter(self.iter(batch_size=n))) + + @property + def epoch(self) -> int: + return int(self._epoch) + + def _effective_generator(self): + if self._shuffling and self.epoch == 0: + return self._shuffling.generator + elif self._shuffling: + # Create effective seed using self.epoch (we subtract in order to avoir overflow in long_scalars) + effective_seed = deepcopy(self._shuffling.generator).integers(0, 1 << 63) - self.epoch + effective_seed = (1 << 63) + effective_seed if effective_seed < 0 else effective_seed + return np.random.default_rng(effective_seed) + else: + raise ValueError("This dataset is not shuffled") + + @property + def num_shards(self) -> int: + if self._distributed and self._ex_iterable.num_shards % self._distributed.world_size == 0: + return self._ex_iterable.num_shards // self._distributed.world_size + return self._ex_iterable.num_shards + + @property + def n_shards(self) -> int: # backward compatibility + return self.num_shards + + def _iter_pytorch(self): + ex_iterable = self._prepare_ex_iterable_for_iteration() + # Fix for fsspec when using multiprocess to avoid hanging in the ML training loop. (only required for fsspec >= 0.9.0) + # See https://github.com/fsspec/gcsfs/issues/379 + fsspec.asyn.reset_lock() + # check if there aren't too many workers + import torch.utils.data + + worker_info = torch.utils.data.get_worker_info() + if self._is_main_process() and ex_iterable.num_shards < worker_info.num_workers: + logger.warning( + f"Too many dataloader workers: {worker_info.num_workers} (max is dataset.num_shards={ex_iterable.num_shards}). " + f"Stopping {worker_info.num_workers - ex_iterable.num_shards} dataloader workers." + ) + logger.info( + f"To parallelize data loading, we give each process some shards (or data sources) to process. " + f"Therefore it's unnecessary to have a number of workers greater than dataset.num_shards={ex_iterable.num_shards}. " + f"To enable more parallelism, please split the dataset in more files than {ex_iterable.num_shards}." + ) + # split workload + _log_prefix = f"node#{self._distributed.rank} " if self._distributed else "" + shards_indices = ex_iterable.split_shard_indices_by_worker( + num_shards=worker_info.num_workers, index=worker_info.id, contiguous=False + ) + if shards_indices: + logger.debug( + f"{_log_prefix}dataloader worker#{worker_info.id}, ': Starting to iterate over {len(shards_indices)}/{ex_iterable.num_shards} shards." + ) + ex_iterable = ex_iterable.shard_data_sources( + num_shards=worker_info.num_workers, index=worker_info.id, contiguous=False + ) + self._state_dict = { + "examples_iterable": ex_iterable._init_state_dict(), + "epoch": self.epoch, + } + if self._starting_state_dict and self.epoch == self._starting_state_dict["epoch"]: + ex_iterable.load_state_dict(self._starting_state_dict["examples_iterable"]) + + if self._formatting and (ex_iterable.iter_arrow or self._formatting.is_table): + formatter = get_formatter(self._formatting.format_type, features=self.features) + if ex_iterable.iter_arrow: + iterator = ex_iterable.iter_arrow() + else: + iterator = _convert_to_arrow(ex_iterable, batch_size=1) + for key, pa_table in iterator: + yield formatter.format_row(pa_table) + return + else: + for key, example in ex_iterable: + # no need to format thanks to FormattedExamplesIterable + yield example + logger.debug( + f"{_log_prefix}dataloader worker#{worker_info.id}, ': Finished iterating over {len(shards_indices)}/{ex_iterable.num_shards} shards." + ) + else: + logger.debug( + f"{_log_prefix}dataloader worker#{worker_info.id}, ': Stopping... Number of dataset shards < num_workers ({ex_iterable.num_shards}<{worker_info.num_workers})." + ) + + def _is_main_process(self): + if self._distributed and self._distributed.rank > 0: + return False + if "torch" in sys.modules: + import torch.utils.data + + worker_info = torch.utils.data.get_worker_info() + if worker_info is not None and worker_info.id > 0: + return False + return True + + def _prepare_ex_iterable_for_iteration( + self, batch_size: int = 1, drop_last_batch: bool = False + ) -> _BaseExamplesIterable: + ex_iterable = self._ex_iterable + if ( + self._formatting + and (ex_iterable.iter_arrow or self._formatting.is_table) + or (self.features and ex_iterable.features != self.features) + ): + ex_iterable = RebatchedArrowExamplesIterable( + ex_iterable, batch_size=batch_size, drop_last_batch=drop_last_batch + ) + if self._shuffling: + ex_iterable = ex_iterable.shuffle_data_sources(self._effective_generator()) + else: + ex_iterable = ex_iterable + + if self._distributed: + rank = self._distributed.rank + world_size = self._distributed.world_size + if ex_iterable.num_shards % world_size == 0: + if self._is_main_process(): + num_shards_per_node = ex_iterable.num_shards // world_size + plural = "s" if num_shards_per_node > 1 else "" + logger.info( + f"Assigning {num_shards_per_node} shard{plural} (or data source{plural}) of the dataset to each node." + ) + ex_iterable = ex_iterable.shard_data_sources(num_shards=world_size, index=rank, contiguous=False) + else: + if self._is_main_process(): + logger.info( + f"Assigning 1 out of {world_size} examples of the dataset to each node. The others are skipped during the iteration." + ) + logger.info( + f"It is more optimized to distribute the dataset shards (or data sources) across nodes. " + f"You can do that by using a dataset with number of shards that is a factor of world_size={world_size}. " + f"The current dataset has {ex_iterable.num_shards} which is not a factor of {world_size}" + ) + ex_iterable = StepExamplesIterable(ex_iterable, step=world_size, offset=rank) + + if self._formatting or (self.features and ex_iterable.features != self.features): + ex_iterable = FormattedExamplesIterable( + ex_iterable, + formatting=self._formatting, + features=self.features, + token_per_repo_id=self._token_per_repo_id, + ) + + self._state_dict = { + "examples_iterable": ex_iterable._init_state_dict(), + "epoch": self.epoch, + } + if self._starting_state_dict and self.epoch == self._starting_state_dict["epoch"]: + ex_iterable.load_state_dict(self._starting_state_dict["examples_iterable"]) + return ex_iterable + + def __iter__(self): + if "torch" in sys.modules: + import torch.utils.data + + worker_info = torch.utils.data.get_worker_info() + if isinstance(self, torch.utils.data.IterableDataset) and worker_info is not None: + # We're a torch.utils.data.IterableDataset in a PyTorch worker process + yield from self._iter_pytorch() + return + + ex_iterable = self._prepare_ex_iterable_for_iteration() + if self._formatting and (ex_iterable.iter_arrow or self._formatting.is_table): + formatter = get_formatter(self._formatting.format_type, features=self.features) + if ex_iterable.iter_arrow: + iterator = ex_iterable.iter_arrow() + else: + iterator = _convert_to_arrow(ex_iterable, batch_size=1) + for key, pa_table in iterator: + yield formatter.format_row(pa_table) + return + + for key, example in ex_iterable: + # no need to format thanks to FormattedExamplesIterable + yield example + + def iter(self, batch_size: int, drop_last_batch: bool = False): + """Iterate through the batches of size `batch_size`. + + Args: + batch_size (:obj:`int`): size of each batch to yield. + drop_last_batch (:obj:`bool`, default `False`): Whether a last batch smaller than the batch_size should be + dropped + """ + + if self._formatting: + formatter = get_formatter(self._formatting.format_type, features=self.features) + format_dict = formatter.recursive_tensorize if isinstance(formatter, TensorFormatter) else None + else: + format_dict = None + + ex_iterable = self._prepare_ex_iterable_for_iteration(batch_size=batch_size, drop_last_batch=drop_last_batch) + if self._formatting and (ex_iterable.iter_arrow or self._formatting.is_table): + if ex_iterable.iter_arrow: + iterator = ex_iterable.iter_arrow() + else: + iterator = _convert_to_arrow(ex_iterable, batch_size=batch_size, drop_last_batch=drop_last_batch) + for key, pa_table in iterator: + yield formatter.format_batch(pa_table) + return + + iterator = iter(ex_iterable) + for key, example in iterator: + # If batched, first build the batch + examples = [example] + [example for key, example in islice(iterator, batch_size - 1)] + if drop_last_batch and len(examples) < batch_size: # ignore last batch + return + batch = _examples_to_batch(examples) + # we need to format here in case we need to stack tensors together + yield format_dict(batch) if format_dict else batch + + @staticmethod + def from_generator( + generator: Callable, + features: Optional[Features] = None, + gen_kwargs: Optional[dict] = None, + split: NamedSplit = Split.TRAIN, + ) -> "IterableDataset": + """Create an Iterable Dataset from a generator. + + Args: + generator (`Callable`): + A generator function that `yields` examples. + features (`Features`, *optional*): + Dataset features. + gen_kwargs(`dict`, *optional*): + Keyword arguments to be passed to the `generator` callable. + You can define a sharded iterable dataset by passing the list of shards in `gen_kwargs`. + This can be used to improve shuffling and when iterating over the dataset with multiple workers. + split ([`NamedSplit`], defaults to `Split.TRAIN`): + Split name to be assigned to the dataset. + + + Returns: + `IterableDataset` + + Example: + + ```py + >>> def gen(): + ... yield {"text": "Good", "label": 0} + ... yield {"text": "Bad", "label": 1} + ... + >>> ds = IterableDataset.from_generator(gen) + ``` + + ```py + >>> def gen(shards): + ... for shard in shards: + ... with open(shard) as f: + ... for line in f: + ... yield {"line": line} + ... + >>> shards = [f"data{i}.txt" for i in range(32)] + >>> ds = IterableDataset.from_generator(gen, gen_kwargs={"shards": shards}) + >>> ds = ds.shuffle(seed=42, buffer_size=10_000) # shuffles the shards order + uses a shuffle buffer + >>> from torch.utils.data import DataLoader + >>> dataloader = DataLoader(ds.with_format("torch"), num_workers=4) # give each worker a subset of 32/4=8 shards + ``` + """ + from .io.generator import GeneratorDatasetInputStream + + return GeneratorDatasetInputStream( + generator=generator, features=features, gen_kwargs=gen_kwargs, streaming=True, split=split + ).read() + + @staticmethod + def from_spark( + df: "pyspark.sql.DataFrame", + split: Optional[NamedSplit] = None, + features: Optional[Features] = None, + **kwargs, + ) -> "IterableDataset": + """Create an IterableDataset from Spark DataFrame. The dataset is streamed to the driver in batches. + + Args: + df (`pyspark.sql.DataFrame`): + The DataFrame containing the desired data. + split (`NamedSplit`, *optional*): + Split name to be assigned to the dataset. + features (`Features`, *optional*): + Dataset features. + + Returns: + [`IterableDataset`] + + Example: + + ```py + >>> df = spark.createDataFrame( + >>> data=[[1, "Elia"], [2, "Teo"], [3, "Fang"]], + >>> columns=["id", "name"], + >>> ) + >>> ds = IterableDataset.from_spark(df) + ``` + """ + from .io.spark import SparkDatasetReader + + if sys.platform == "win32": + raise OSError("IterableDataset.from_spark is not currently supported on Windows") + + return SparkDatasetReader( + df, + split=split, + features=features, + streaming=True, + **kwargs, + ).read() + + @staticmethod + def from_file(filename: str) -> "IterableDataset": + """Instantiate a IterableDataset from Arrow table at filename. + + Args: + filename (`str`): + File name of the dataset. + + Returns: + [`IterableDataset`] + """ + pa_table_schema = read_schema_from_file(filename) + inferred_features = Features.from_arrow_schema(pa_table_schema) + ex_iterable = ArrowExamplesIterable(Dataset._generate_tables_from_cache_file, kwargs={"filename": filename}) + return IterableDataset(ex_iterable=ex_iterable, info=DatasetInfo(features=inferred_features)) + + def with_format( + self, + type: Optional[str] = None, + ) -> "IterableDataset": + """ + Return a dataset with the specified format. + + Args: + + type (`str`, *optional*): + Either output type selected in `[None, 'numpy', 'torch', 'tensorflow', 'jax', 'arrow', 'pandas', 'polars']`. + `None` means it returns python objects (default). + + Example: + + ```py + >>> from datasets import load_dataset + >>> from transformers import AutoTokenizer + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation", streaming=True) + >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-cased") + >>> ds = ds.map(lambda x: tokenizer(x['text'], truncation=True, padding=True), batched=True) + >>> ds = ds.with_format("torch") + >>> next(iter(ds)) + {'text': 'compassionately explores the seemingly irreconcilable situation between conservative christian parents and their estranged gay and lesbian children .', + 'label': tensor(1), + 'input_ids': tensor([ 101, 18027, 16310, 16001, 1103, 9321, 178, 11604, 7235, 6617, + 1742, 2165, 2820, 1206, 6588, 22572, 12937, 1811, 2153, 1105, + 1147, 12890, 19587, 6463, 1105, 15026, 1482, 119, 102, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0]), + 'token_type_ids': tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]), + 'attention_mask': tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])} + ``` + """ + type = get_format_type_from_alias(type) + # TODO(QL): add format_kwargs + # TODO(QL): add format_columns and return_all_columns + # TODO(QL): add pandas format + return IterableDataset( + ex_iterable=self._ex_iterable, + info=self._info.copy(), + split=self._split, + formatting=FormattingConfig(format_type=type), + shuffling=copy.deepcopy(self._shuffling), + distributed=copy.deepcopy(self._distributed), + token_per_repo_id=self._token_per_repo_id, + ) + + def map( + self, + function: Optional[Callable] = None, + with_indices: bool = False, + input_columns: Optional[Union[str, list[str]]] = None, + batched: bool = False, + batch_size: Optional[int] = 1000, + drop_last_batch: bool = False, + remove_columns: Optional[Union[str, list[str]]] = None, + features: Optional[Features] = None, + fn_kwargs: Optional[dict] = None, + ) -> "IterableDataset": + """ + Apply a function to all the examples in the iterable dataset (individually or in batches) and update them. + If your function returns a column that already exists, then it overwrites it. + The function is applied on-the-fly on the examples when iterating over the dataset. + + You can specify whether the function should be batched or not with the `batched` parameter: + + - If batched is `False`, then the function takes 1 example in and should return 1 example. + An example is a dictionary, e.g. `{"text": "Hello there !"}`. + - If batched is `True` and `batch_size` is 1, then the function takes a batch of 1 example as input and can return a batch with 1 or more examples. + A batch is a dictionary, e.g. a batch of 1 example is {"text": ["Hello there !"]}. + - If batched is `True` and `batch_size` is `n` > 1, then the function takes a batch of `n` examples as input and can return a batch with `n` examples, or with an arbitrary number of examples. + Note that the last batch may have less than `n` examples. + A batch is a dictionary, e.g. a batch of `n` examples is `{"text": ["Hello there !"] * n}`. + + If the function is asynchronous, then `map` will run your function in parallel, with up to one thousand simulatenous calls. + It is recommended to use a `asyncio.Semaphore` in your function if you want to set a maximum number of operations that can run at the same time. + + Args: + function (`Callable`, *optional*, defaults to `None`): + Function applied on-the-fly on the examples when you iterate on the dataset. + It must have one of the following signatures: + + - `function(example: Dict[str, Any]) -> Dict[str, Any]` if `batched=False` and `with_indices=False` + - `function(example: Dict[str, Any], idx: int) -> Dict[str, Any]` if `batched=False` and `with_indices=True` + - `function(batch: Dict[str, List]) -> Dict[str, List]` if `batched=True` and `with_indices=False` + - `function(batch: Dict[str, List], indices: List[int]) -> Dict[str, List]` if `batched=True` and `with_indices=True` + + For advanced usage, the function can also return a `pyarrow.Table`. + If the function is asynchronous, then `map` will run your function in parallel. + Moreover if your function returns nothing (`None`), then `map` will run your function and return the dataset unchanged. + If no function is provided, default to identity function: `lambda x: x`. + with_indices (`bool`, defaults to `False`): + Provide example indices to `function`. Note that in this case the signature of `function` should be `def function(example, idx[, rank]): ...`. + input_columns (`Optional[Union[str, List[str]]]`, defaults to `None`): + The columns to be passed into `function` + as positional arguments. If `None`, a dict mapping to all formatted columns is passed as one argument. + batched (`bool`, defaults to `False`): + Provide batch of examples to `function`. + batch_size (`int`, *optional*, defaults to `1000`): + Number of examples per batch provided to `function` if `batched=True`. + `batch_size <= 0` or `batch_size == None` then provide the full dataset as a single batch to `function`. + drop_last_batch (`bool`, defaults to `False`): + Whether a last batch smaller than the batch_size should be + dropped instead of being processed by the function. + remove_columns (`[List[str]]`, *optional*, defaults to `None`): + Remove a selection of columns while doing the mapping. + Columns will be removed before updating the examples with the output of `function`, i.e. if `function` is adding + columns with names in `remove_columns`, these columns will be kept. + features (`[Features]`, *optional*, defaults to `None`): + Feature types of the resulting dataset. + fn_kwargs (`Dict`, *optional*, default `None`): + Keyword arguments to be passed to `function`. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train", streaming=True) + >>> def add_prefix(example): + ... example["text"] = "Review: " + example["text"] + ... return example + >>> ds = ds.map(add_prefix) + >>> list(ds.take(3)) + [{'label': 1, + 'text': 'Review: the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'}, + {'label': 1, + 'text': 'Review: the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson\'s expanded vision of j . r . r . tolkien\'s middle-earth .'}, + {'label': 1, 'text': 'Review: effective but too-tepid biopic'}] + ``` + """ + if isinstance(input_columns, str): + input_columns = [input_columns] + if isinstance(remove_columns, str): + remove_columns = [remove_columns] + if function is None: + function = identity_func + if fn_kwargs is None: + fn_kwargs = {} + + ex_iterable = self._ex_iterable + # no need to apply features if ex_iterable is typed and if there was no cast_column() + input_features = ( + None + if (ex_iterable.is_typed and (self._info.features is None or self._info.features == ex_iterable.features)) + else self._info.features + ) + + if self._formatting and self._formatting.is_table: + # apply formatting before iter_arrow to keep map examples iterable happy + ex_iterable = FormattedExamplesIterable( + ex_iterable, + formatting=copy.deepcopy(self._formatting), + features=input_features, + token_per_repo_id=self._token_per_repo_id, + ) + ex_iterable = RebatchedArrowExamplesIterable( + ex_iterable, batch_size=batch_size if batched else 1, drop_last_batch=drop_last_batch + ) + else: + if self._formatting and self._ex_iterable.iter_arrow: + ex_iterable = RebatchedArrowExamplesIterable( + self._ex_iterable, batch_size=batch_size if batched else 1, drop_last_batch=drop_last_batch + ) + if self._formatting or input_features: + # apply formatting after iter_arrow to avoid re-encoding the examples + ex_iterable = FormattedExamplesIterable( + ex_iterable, + formatting=copy.deepcopy(self._formatting), + features=input_features, + token_per_repo_id=self._token_per_repo_id, + ) + + ex_iterable = MappedExamplesIterable( + ex_iterable, + function=function, + with_indices=with_indices, + input_columns=input_columns, + batched=batched, + batch_size=batch_size, + drop_last_batch=drop_last_batch, + remove_columns=remove_columns, + fn_kwargs=fn_kwargs, + formatting=self._formatting, + features=features, + ) + info = self.info.copy() + info.features = features + return IterableDataset( + ex_iterable=ex_iterable, + info=info, + split=self._split, + formatting=self._formatting, + shuffling=copy.deepcopy(self._shuffling), + distributed=copy.deepcopy(self._distributed), + token_per_repo_id=self._token_per_repo_id, + ) + + def filter( + self, + function: Optional[Callable] = None, + with_indices=False, + input_columns: Optional[Union[str, list[str]]] = None, + batched: bool = False, + batch_size: Optional[int] = 1000, + fn_kwargs: Optional[dict] = None, + ) -> "IterableDataset": + """Apply a filter function to all the elements so that the dataset only includes examples according to the filter function. + The filtering is done on-the-fly when iterating over the dataset. + + If the function is asynchronous, then `filter` will run your function in parallel, with up to one thousand simulatenous calls (configurable). + It is recommended to use a `asyncio.Semaphore` in your function if you want to set a maximum number of operations that can run at the same time. + + Args: + function (`Callable`): + Callable with one of the following signatures: + + - `function(example: Dict[str, Any]) -> bool` if `with_indices=False, batched=False` + - `function(example: Dict[str, Any], indices: int) -> bool` if `with_indices=True, batched=False` + - `function(example: Dict[str, List]) -> List[bool]` if `with_indices=False, batched=True` + - `function(example: Dict[str, List], indices: List[int]) -> List[bool]` if `with_indices=True, batched=True` + + If the function is asynchronous, then `filter` will run your function in parallel. + If no function is provided, defaults to an always True function: `lambda x: True`. + with_indices (`bool`, defaults to `False`): + Provide example indices to `function`. Note that in this case the signature of `function` should be `def function(example, idx): ...`. + input_columns (`str` or `List[str]`, *optional*): + The columns to be passed into `function` as + positional arguments. If `None`, a dict mapping to all formatted columns is passed as one argument. + batched (`bool`, defaults to `False`): + Provide batch of examples to `function`. + batch_size (`int`, *optional*, default `1000`): + Number of examples per batch provided to `function` if `batched=True`. + fn_kwargs (`Dict`, *optional*, default `None`): + Keyword arguments to be passed to `function`. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train", streaming=True) + >>> ds = ds.filter(lambda x: x["label"] == 0) + >>> list(ds.take(3)) + [{'label': 0, 'movie_review': 'simplistic , silly and tedious .'}, + {'label': 0, + 'movie_review': "it's so laddish and juvenile , only teenage boys could possibly find it funny ."}, + {'label': 0, + 'movie_review': 'exploitative and largely devoid of the depth or sophistication that would make watching such a graphic treatment of the crimes bearable .'}] + ``` + """ + if isinstance(input_columns, str): + input_columns = [input_columns] + + # We need the examples to be decoded for certain feature types like Image or Audio, + # format and type before filtering + ex_iterable = self._ex_iterable + if self._info.features or self._formatting: + ex_iterable = FormattedExamplesIterable( + ex_iterable, + formatting=self._formatting, + features=None if ex_iterable.is_typed else self._info.features, + token_per_repo_id=self._token_per_repo_id, + ) + + ex_iterable = FilteredExamplesIterable( + ex_iterable, + function=function, + with_indices=with_indices, + input_columns=input_columns, + batched=batched, + batch_size=batch_size, + fn_kwargs=fn_kwargs, + formatting=self._formatting, + ) + return IterableDataset( + ex_iterable=ex_iterable, + info=self._info, + split=self._split, + formatting=self._formatting, + shuffling=copy.deepcopy(self._shuffling), + distributed=copy.deepcopy(self._distributed), + token_per_repo_id=self._token_per_repo_id, + ) + + def shuffle( + self, seed=None, generator: Optional[np.random.Generator] = None, buffer_size: int = 1000 + ) -> "IterableDataset": + """ + Randomly shuffles the elements of this dataset. + + This dataset fills a buffer with `buffer_size` elements, then randomly samples elements from this buffer, + replacing the selected elements with new elements. For perfect shuffling, a buffer size greater than or + equal to the full size of the dataset is required. + + For instance, if your dataset contains 10,000 elements but `buffer_size` is set to 1000, then `shuffle` will + initially select a random element from only the first 1000 elements in the buffer. Once an element is + selected, its space in the buffer is replaced by the next (i.e. 1,001-st) element, + maintaining the 1000 element buffer. + + If the dataset is made of several shards, it also does shuffle the order of the shards. + However if the order has been fixed by using [`~datasets.IterableDataset.skip`] or [`~datasets.IterableDataset.take`] + then the order of the shards is kept unchanged. + + Args: + seed (`int`, *optional*, defaults to `None`): + Random seed that will be used to shuffle the dataset. + It is used to sample from the shuffle buffer and also to shuffle the data shards. + generator (`numpy.random.Generator`, *optional*): + Numpy random Generator to use to compute the permutation of the dataset rows. + If `generator=None` (default), uses `np.random.default_rng` (the default BitGenerator (PCG64) of NumPy). + buffer_size (`int`, defaults to `1000`): + Size of the buffer. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train", streaming=True) + >>> list(ds.take(3)) + [{'label': 1, + 'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'}, + {'label': 1, + 'text': 'the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson\'s expanded vision of j . r . r . tolkien\'s middle-earth .'}, + {'label': 1, 'text': 'effective but too-tepid biopic'}] + >>> shuffled_ds = ds.shuffle(seed=42) + >>> list(shuffled_ds.take(3)) + [{'label': 1, + 'text': "a sports movie with action that's exciting on the field and a story you care about off it ."}, + {'label': 1, + 'text': 'at its best , the good girl is a refreshingly adult take on adultery . . .'}, + {'label': 1, + 'text': "sam jones became a very lucky filmmaker the day wilco got dropped from their record label , proving that one man's ruin may be another's fortune ."}] + ``` + """ + if generator is None: + generator = np.random.default_rng(seed) + else: + generator = deepcopy(generator) + shuffling = ShufflingConfig(generator=generator, _original_seed=seed) + return IterableDataset( + ex_iterable=BufferShuffledExamplesIterable( + self._ex_iterable, buffer_size=buffer_size, generator=generator + ), + info=self._info.copy(), + split=self._split, + formatting=self._formatting, + shuffling=shuffling, + distributed=copy.deepcopy(self._distributed), + token_per_repo_id=self._token_per_repo_id, + ) + + def set_epoch(self, epoch: int): + self._epoch += epoch - self._epoch # update torch value in shared memory in-place + + def skip(self, n: int) -> "IterableDataset": + """ + Create a new [`IterableDataset`] that skips the first `n` elements. + + Args: + n (`int`): + Number of elements to skip. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train", streaming=True) + >>> list(ds.take(3)) + [{'label': 1, + 'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'}, + {'label': 1, + 'text': 'the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson\'s expanded vision of j . r . r . tolkien\'s middle-earth .'}, + {'label': 1, 'text': 'effective but too-tepid biopic'}] + >>> ds = ds.skip(1) + >>> list(ds.take(3)) + [{'label': 1, + 'text': 'the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson\'s expanded vision of j . r . r . tolkien\'s middle-earth .'}, + {'label': 1, 'text': 'effective but too-tepid biopic'}, + {'label': 1, + 'text': 'if you sometimes like to go to the movies to have fun , wasabi is a good place to start .'}] + ``` + """ + ex_iterable = SkipExamplesIterable( + self._ex_iterable, + n, + block_sources_order_when_shuffling=self._shuffling is None, + split_when_sharding=self._distributed is None, + ) + return IterableDataset( + ex_iterable=ex_iterable, + info=self._info.copy(), + split=self._split, + formatting=self._formatting, + shuffling=copy.deepcopy(self._shuffling), + distributed=copy.deepcopy(self._distributed), + token_per_repo_id=self._token_per_repo_id, + ) + + def repeat(self, num_times: Optional[int]) -> "IterableDataset": + """ + Create a new [`IterableDataset`] that repeats the underlying dataset `num_times` times. + + N.B. The effect of calling shuffle after repeat depends significantly on buffer size. + With buffer_size 1, duplicate data is never seen in the same iteration, even after shuffling: + ds.repeat(n).shuffle(seed=42, buffer_size=1) is equivalent to ds.shuffle(seed=42, buffer_size=1).repeat(n), + and only shuffles shard orders within each iteration. + With buffer size >= (num samples in the dataset * num_times), we get full shuffling of the repeated data, i.e. we can observe duplicates in + the same iteration. + + Args: + num_times (`int`) or (`None`): + Number of times to repeat the dataset. If `None`, the dataset will be repeated indefinitely. + + Example: + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train") + >>> ds = ds.take(2).repeat(2) + >>> list(ds) + [{'label': 1, + 'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'}, + {'label': 1, + 'text': 'the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson\'s expanded vision of j . r . r . tolkien\'s middle-earth .'}, + {'label': 1, 'text': 'effective but too-tepid biopic'}, + {'label': 1, + 'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'}, + {'label': 1, + 'text': 'the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson\'s expanded vision of j . r . r . tolkien\'s middle-earth .'}, + {'label': 1, 'text': 'effective but too-tepid biopic'}] + ``` + """ + return IterableDataset( + ex_iterable=RepeatExamplesIterable(self._ex_iterable, num_times=num_times), + info=self._info, + split=self._split, + formatting=self._formatting, + shuffling=copy.deepcopy(self._shuffling), + distributed=copy.deepcopy(self._distributed), + token_per_repo_id=self._token_per_repo_id, + ) + + def take(self, n: int) -> "IterableDataset": + """ + Create a new [`IterableDataset`] with only the first `n` elements. + + Args: + n (`int`): + Number of elements to take. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train", streaming=True) + >>> small_ds = ds.take(2) + >>> list(small_ds) + [{'label': 1, + 'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'}, + {'label': 1, + 'text': 'the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson\'s expanded vision of j . r . r . tolkien\'s middle-earth .'}] + ``` + """ + ex_iterable = TakeExamplesIterable( + self._ex_iterable, + n, + block_sources_order_when_shuffling=self._shuffling is None, + split_when_sharding=self._distributed is None, + ) + return IterableDataset( + ex_iterable=ex_iterable, + info=self._info.copy(), + split=self._split, + formatting=self._formatting, + shuffling=copy.deepcopy(self._shuffling), + distributed=copy.deepcopy(self._distributed), + token_per_repo_id=self._token_per_repo_id, + ) + + def shard( + self, + num_shards: int, + index: int, + contiguous: bool = True, + ) -> "IterableDataset": + """Return the `index`-nth shard from dataset split into `num_shards` pieces. + + This shards deterministically. `dataset.shard(n, i)` splits the dataset into contiguous chunks, + so it can be easily concatenated back together after processing. If `dataset.num_shards % n == l`, then the + first `l` datasets each have `(dataset.num_shards // n) + 1` shards, and the remaining datasets have `(dataset.num_shards // n)` shards. + `datasets.concatenate_datasets([dset.shard(n, i) for i in range(n)])` returns a dataset with the same order as the original. + In particular, `dataset.shard(dataset.num_shards, i)` returns a dataset with 1 shard. + + Note: n should be less or equal to the number of shards in the dataset `dataset.num_shards`. + + On the other hand, `dataset.shard(n, i, contiguous=False)` contains all the shards of the dataset whose index mod `n = i`. + + Be sure to shard before using any randomizing operator (such as `shuffle`). + It is best if the shard operator is used early in the dataset pipeline. + + Args: + num_shards (`int`): + How many shards to split the dataset into. + index (`int`): + Which shard to select and return. + contiguous: (`bool`, defaults to `True`): + Whether to select contiguous blocks of indices for shards. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("amazon_polarity", split="train", streaming=True) + >>> ds + Dataset({ + features: ['label', 'title', 'content'], + num_shards: 4 + }) + >>> ds.shard(num_shards=2, index=0) + Dataset({ + features: ['label', 'title', 'content'], + num_shards: 2 + }) + ``` + """ + ex_iterable = self._ex_iterable.shard_data_sources(num_shards=num_shards, index=index, contiguous=contiguous) + return IterableDataset( + ex_iterable=ex_iterable, + info=self._info.copy(), + split=self._split, + formatting=self._formatting, + shuffling=copy.deepcopy(self._shuffling), + distributed=copy.deepcopy(self._distributed), + token_per_repo_id=self._token_per_repo_id, + ) + + @property + def column_names(self) -> Optional[list[str]]: + """Names of the columns in the dataset. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation", streaming=True) + >>> ds.column_names + ['text', 'label'] + ``` + """ + return list(self._info.features.keys()) if self._info.features is not None else None + + def add_column(self, name: str, column: Union[list, np.array]) -> "IterableDataset": + """Add column to Dataset. + + Args: + name (str): Column name. + column (list or np.array): Column data to be added. + + Returns: + `IterableDataset` + """ + return self.map(partial(add_column_fn, name=name, column=column), with_indices=True) + + def rename_column(self, original_column_name: str, new_column_name: str) -> "IterableDataset": + """ + Rename a column in the dataset, and move the features associated to the original column under the new column + name. + + Args: + original_column_name (`str`): + Name of the column to rename. + new_column_name (`str`): + New name for the column. + + Returns: + `IterableDataset`: A copy of the dataset with a renamed column. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train", streaming=True) + >>> next(iter(ds)) + {'label': 1, + 'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'} + >>> ds = ds.rename_column("text", "movie_review") + >>> next(iter(ds)) + {'label': 1, + 'movie_review': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'} + ``` + """ + return self.rename_columns({original_column_name: new_column_name}) + + def rename_columns(self, column_mapping: dict[str, str]) -> "IterableDataset": + """ + Rename several columns in the dataset, and move the features associated to the original columns under + the new column names. + + Args: + column_mapping (`Dict[str, str]`): A mapping of columns to rename to their new names + + Returns: + `IterableDataset`: A copy of the dataset with renamed columns + """ + + original_features = self._info.features.copy() if self._info.features else None + ds_iterable = self.map( + partial(_rename_columns_fn, column_mapping=column_mapping), remove_columns=list(column_mapping) + ) + if original_features is not None: + ds_iterable._info.features = Features( + { + column_mapping[col] if col in column_mapping.keys() else col: feature + for col, feature in original_features.items() + } + ) + return ds_iterable + + def remove_columns(self, column_names: Union[str, list[str]]) -> "IterableDataset": + """ + Remove one or several column(s) in the dataset and the features associated to them. + The removal is done on-the-fly on the examples when iterating over the dataset. + + + Args: + column_names (`Union[str, List[str]]`): + Name of the column(s) to remove. + + Returns: + `IterableDataset`: A copy of the dataset object without the columns to remove. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train", streaming=True) + >>> next(iter(ds)) + {'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .', 'label': 1} + >>> ds = ds.remove_columns("label") + >>> next(iter(ds)) + {'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'} + ``` + """ + original_features = self._info.features.copy() if self._info.features else None + ds_iterable = self.map(remove_columns=column_names) + if original_features is not None: + ds_iterable._info.features = original_features.copy() + for col, _ in original_features.items(): + if col in column_names: + del ds_iterable._info.features[col] + + return ds_iterable + + def select_columns(self, column_names: Union[str, list[str]]) -> "IterableDataset": + """Select one or several column(s) in the dataset and the features + associated to them. The selection is done on-the-fly on the examples + when iterating over the dataset. + + + Args: + column_names (`Union[str, List[str]]`): + Name of the column(s) to select. + + Returns: + `IterableDataset`: A copy of the dataset object with selected columns. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train", streaming=True) + >>> next(iter(ds)) + {'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .', 'label': 1} + >>> ds = ds.select_columns("text") + >>> next(iter(ds)) + {'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'} + ``` + """ + if isinstance(column_names, str): + column_names = [column_names] + + if self._info: + info = copy.deepcopy(self._info) + if self._info.features is not None: + missing_columns = set(column_names) - set(self._info.features.keys()) + if missing_columns: + raise ValueError( + f"Column name {list(missing_columns)} not in the " + "dataset. Columns in the dataset: " + f"{list(self._info.features.keys())}." + ) + info.features = Features({c: info.features[c] for c in column_names}) + + ex_iterable = SelectColumnsIterable(self._ex_iterable, column_names) + return IterableDataset( + ex_iterable=ex_iterable, + info=info, + split=self._split, + formatting=self._formatting, + shuffling=self._shuffling, + distributed=self._distributed, + token_per_repo_id=self._token_per_repo_id, + ) + + def cast_column(self, column: str, feature: FeatureType) -> "IterableDataset": + """Cast column to feature for decoding. + + Args: + column (`str`): + Column name. + feature (`Feature`): + Target feature. + + Returns: + `IterableDataset` + + Example: + + ```py + >>> from datasets import load_dataset, Audio + >>> ds = load_dataset("PolyAI/minds14", name="en-US", split="train", streaming=True) + >>> ds.features + {'audio': Audio(sampling_rate=8000, mono=True, decode=True, id=None), + 'english_transcription': Value(dtype='string', id=None), + 'intent_class': ClassLabel(num_classes=14, names=['abroad', 'address', 'app_error', 'atm_limit', 'balance', 'business_loan', 'card_issues', 'cash_deposit', 'direct_debit', 'freeze', 'high_value_payment', 'joint_account', 'latest_transactions', 'pay_bill'], id=None), + 'lang_id': ClassLabel(num_classes=14, names=['cs-CZ', 'de-DE', 'en-AU', 'en-GB', 'en-US', 'es-ES', 'fr-FR', 'it-IT', 'ko-KR', 'nl-NL', 'pl-PL', 'pt-PT', 'ru-RU', 'zh-CN'], id=None), + 'path': Value(dtype='string', id=None), + 'transcription': Value(dtype='string', id=None)} + >>> ds = ds.cast_column("audio", Audio(sampling_rate=16000)) + >>> ds.features + {'audio': Audio(sampling_rate=16000, mono=True, decode=True, id=None), + 'english_transcription': Value(dtype='string', id=None), + 'intent_class': ClassLabel(num_classes=14, names=['abroad', 'address', 'app_error', 'atm_limit', 'balance', 'business_loan', 'card_issues', 'cash_deposit', 'direct_debit', 'freeze', 'high_value_payment', 'joint_account', 'latest_transactions', 'pay_bill'], id=None), + 'lang_id': ClassLabel(num_classes=14, names=['cs-CZ', 'de-DE', 'en-AU', 'en-GB', 'en-US', 'es-ES', 'fr-FR', 'it-IT', 'ko-KR', 'nl-NL', 'pl-PL', 'pt-PT', 'ru-RU', 'zh-CN'], id=None), + 'path': Value(dtype='string', id=None), + 'transcription': Value(dtype='string', id=None)} + ``` + """ + info = self._info.copy() + info.features[column] = feature + return IterableDataset( + ex_iterable=self._ex_iterable, + info=info, + split=self._split, + formatting=self._formatting, + shuffling=copy.deepcopy(self._shuffling), + distributed=copy.deepcopy(self._distributed), + token_per_repo_id=self._token_per_repo_id, + ) + + def cast( + self, + features: Features, + ) -> "IterableDataset": + """ + Cast the dataset to a new set of features. + + Args: + features ([`Features`]): + New features to cast the dataset to. + The name of the fields in the features must match the current column names. + The type of the data must also be convertible from one type to the other. + For non-trivial conversion, e.g. `string` <-> `ClassLabel` you should use [`~Dataset.map`] to update the Dataset. + + Returns: + `IterableDataset`: A copy of the dataset with casted features. + + Example: + + ```py + >>> from datasets import load_dataset, ClassLabel, Value + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train", streaming=True) + >>> ds.features + {'label': ClassLabel(names=['neg', 'pos'], id=None), + 'text': Value(dtype='string', id=None)} + >>> new_features = ds.features.copy() + >>> new_features["label"] = ClassLabel(names=["bad", "good"]) + >>> new_features["text"] = Value("large_string") + >>> ds = ds.cast(new_features) + >>> ds.features + {'label': ClassLabel(names=['bad', 'good'], id=None), + 'text': Value(dtype='large_string', id=None)} + ``` + """ + info = self._info.copy() + info.features = features + return IterableDataset( + ex_iterable=self._ex_iterable, + info=info, + split=self._split, + formatting=self._formatting, + shuffling=copy.deepcopy(self._shuffling), + distributed=copy.deepcopy(self._distributed), + token_per_repo_id=self._token_per_repo_id, + ) + + def decode(self, enable: bool = True, num_threads: int = 0) -> "IterableDataset": + """ + Enable or disable the dataset features decoding for audio, image, video. + + When enabled (default), media types are decoded: + + * audio -> dict of "array" and "sampling_rate" and "path" + * image -> PIL.Image + * video -> torchvision.io.VideoReader + + You can enable multithreading using `num_threads`. This is especially useful to speed up remote + data streaming. However it can be slower than `num_threads=0` for local data on fast disks. + + Disabling decoding is useful if you want to iterate on the paths or bytes of the media files + without actually decoding their content. To disable decoding you can use `.decode(False)`, which + is equivalent to calling `.cast()` or `.cast_column()` with all the Audio, Image and Video types + set to `decode=False`. + + Args: + enable (`bool`, defaults to `True`): + Enable or disable features decoding. + num_threads (`int`, defaults to `0`): + Enable multithreading for features decoding. + + Returns: + `IterableDataset`: A copy of the dataset with casted features. + + Examples: + + Disable decoding: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("sshh12/planet-textures", split="train", streaming=True) + >>> next(iter(ds)) + {'image': , + 'text': 'A distant celestial object with an icy crust, displaying a light blue shade, covered with round pits and rugged terrains.'} + >>> ds = ds.decode(False) + >>> ds.features + {'image': Image(mode=None, decode=False, id=None), + 'text': Value(dtype='string', id=None)} + >>> next(iter(ds)) + { + 'image': { + 'path': 'hf://datasets/sshh12/planet-textures@69dc4cef7a5c4b2cfe387727ec8ea73d4bff7302/train/textures/0000.png', + 'bytes': None + }, + 'text': 'A distant celestial object with an icy crust, displaying a light blue shade, covered with round pits and rugged terrains.' + } + ``` + + Speed up streaming with multithreading: + + ```py + >>> import os + >>> from datasets import load_dataset + >>> from tqdm import tqdm + >>> ds = load_dataset("sshh12/planet-textures", split="train", streaming=True) + >>> num_threads = min(32, (os.cpu_count() or 1) + 4) + >>> ds = ds.decode(num_threads=num_threads) + >>> for _ in tqdm(ds): # 20 times faster ! + ... ... + ``` + """ + if not self.features: + raise ValueError( + "Features decoding is only available for datasets with known features, but features are Unknown. " + "Please set the datasets features with `ds = ds.cast(features)`." + ) + ds = self + + def set_decoding(decode: bool, feature): + if hasattr(feature, "decode"): + feature.decode = decode + + if enable and num_threads > 0: + disabled_decoding_features = self.features.copy() + enabled_decoding_features = self.features.copy() + + _visit(disabled_decoding_features, partial(set_decoding, False)) + _visit(enabled_decoding_features, partial(set_decoding, True)) + ds = ds.cast(disabled_decoding_features) + pool = multiprocessing.pool.ThreadPool(num_threads) + func = partial(_apply_async, pool, enabled_decoding_features.decode_example) + ds = ds.map(func, features=enabled_decoding_features) + assert isinstance(ds._ex_iterable, MappedExamplesIterable) + ds._ex_iterable.max_num_running_async_map_functions_in_parallel = 2 * num_threads + else: + features = ds.features.copy() + _visit(features, partial(set_decoding, enable)) + ds = ds.cast(features) + return ds + + def _step(self, step: int, offset: int) -> "IterableDataset": + ex_iterable = StepExamplesIterable(self._ex_iterable, step=step, offset=offset) + return IterableDataset( + ex_iterable=ex_iterable, + info=self._info.copy(), + split=self._split, + formatting=self._formatting, + shuffling=copy.deepcopy(self._shuffling), + distributed=copy.deepcopy(self._distributed), + token_per_repo_id=self._token_per_repo_id, + ) + + def _resolve_features(self): + if self.features is not None: + return self + elif self._ex_iterable.is_typed: + features = self._ex_iterable.features + else: + features = _infer_features_from_batch(self.with_format(None)._head()) + info = self.info.copy() + info.features = features + return IterableDataset( + ex_iterable=self._ex_iterable, + info=info, + split=self._split, + formatting=self._formatting, + shuffling=copy.deepcopy(self._shuffling), + distributed=copy.deepcopy(self._distributed), + token_per_repo_id=self._token_per_repo_id, + ) + + def batch(self, batch_size: int, drop_last_batch: bool = False) -> "IterableDataset": + """ + Group samples from the dataset into batches. + + Args: + batch_size (`int`): The number of samples in each batch. + drop_last_batch (`bool`, defaults to `False`): Whether to drop the last incomplete batch. + + Example: + ```py + >>> ds = load_dataset("some_dataset", streaming=True) + >>> batched_ds = ds.batch(batch_size=32) + ``` + """ + + def batch_fn(unbatched): + return {k: [v] for k, v in unbatched.items()} + + if self.features: + features = Features({col: [feature] for col, feature in self.features.items()}) + else: + features = None + return self.map( + batch_fn, batched=True, batch_size=batch_size, drop_last_batch=drop_last_batch, features=features + ) + + +def _concatenate_iterable_datasets( + dsets: list[IterableDataset], + info: Optional[DatasetInfo] = None, + split: Optional[NamedSplit] = None, + axis: int = 0, +) -> IterableDataset: + """ + Converts a list of `IterableDataset` with the same schema into a single `IterableDataset`. + Missing data are filled with None values. + + + + Args: + dsets (`List[datasets.IterableDataset]`): List of Datasets to concatenate. + info (`DatasetInfo`, optional): Dataset information, like description, citation, etc. + split (`NamedSplit`, optional): Name of the dataset split. + axis (``{0, 1}``, default ``0``, meaning over rows): + Axis to concatenate over, where ``0`` means over rows (vertically) and ``1`` means over columns + (horizontally). + + *New in version 1.6.0* + + Example: + + ```py + >>> ds3 = _concatenate_iterable_datasets([ds1, ds2]) + ``` + """ + dsets = [d._resolve_features() for d in dsets] + + # Perform checks (and a potentional cast if axis=0) + if axis == 0: + _check_if_features_can_be_aligned([dset.features for dset in dsets]) + else: + _check_column_names([col_name for dset in dsets for col_name in dset.features]) + + # TODO: improve this to account for a mix of ClassLabel and Value for example + # right now it would keep the type of the first dataset in the list + features = Features( + {k: v for features in _align_features([dset.features for dset in dsets]) for k, v in features.items()} + ) + + ex_iterables = [copy.deepcopy(d._ex_iterable) for d in dsets] + if axis == 0: + ex_iterable = VerticallyConcatenatedMultiSourcesExamplesIterable(ex_iterables) + else: + ex_iterable = HorizontallyConcatenatedMultiSourcesExamplesIterable(ex_iterables) + # Set new info - we update the features + # setting the features also ensures to fill missing columns with None + if info is None: + info = DatasetInfo.from_merge([d.info for d in dsets]) + else: + info = info.copy() + info.features = features + # Get all the auth tokens per repository - in case the datasets come from different private repositories + token_per_repo_id = {repo_id: token for dataset in dsets for repo_id, token in dataset._token_per_repo_id.items()} + # Return new daset + return IterableDataset(ex_iterable=ex_iterable, info=info, split=split, token_per_repo_id=token_per_repo_id) + + +def _interleave_iterable_datasets( + datasets: list[IterableDataset], + probabilities: Optional[list[float]] = None, + seed: Optional[int] = None, + info: Optional[DatasetInfo] = None, + split: Optional[NamedSplit] = None, + stopping_strategy: Literal["first_exhausted", "all_exhausted"] = "first_exhausted", +) -> IterableDataset: + """ + Interleave several iterable datasets (sources) into a single iterable dataset. + The new iterable dataset alternates between the sources to yield examples. + If `probabilities = None` (default) the iterable dataset will cycles through the sources in order for each next example in the iteration. + If `probabilities` is not `None, the iterable dataset will sample a random source according to the provided probabilities for each next examples in the iteration. + + + + Args: + datasets (`List[IterableDataset]`): list of datasets to interleave + probabilities (`List[float]`, optional, default None): If specified, the new iterable dataset samples + examples from one source at a time according to these probabilities. + seed (`int`, optional, default None): The random seed used to choose a source for each example. + stopping_strategy (`str`, defaults to `first_exhausted`): + Two strategies are proposed right now. + By default, `first_exhausted` is an undersampling strategy, i.e the dataset construction is stopped as soon as one dataset has ran out of samples. + If the strategy is `all_exhausted`, we use an oversampling strategy, i.e the dataset construction is stopped as soon as every samples of every dataset has been added at least once. + Note that if the strategy is `all_exhausted`, the interleaved dataset size can get enormous: + - with no probabilities, the resulting dataset will have max_length_datasets*nb_dataset samples. + - with given probabilities, the resulting dataset will have more samples if some datasets have really low probability of visiting. + + Output: + `datasets.IterableDataset` + """ + datasets = [d._resolve_features() for d in datasets] + + # Perform checks + _check_if_features_can_be_aligned([dset.features for dset in datasets]) + + # TODO: improve this to account for a mix of ClassLabel and Value for example + # right now it would keep the type of the first dataset in the list + features = Features( + {k: v for features in _align_features([dset.features for dset in datasets]) for k, v in features.items()} + ) + + ex_iterables = [copy.deepcopy(d._ex_iterable) for d in datasets] + + # Use cycling or random cycling of sources + if probabilities is None: + ex_iterable = CyclingMultiSourcesExamplesIterable(ex_iterables, stopping_strategy=stopping_strategy) + else: + generator = np.random.default_rng(seed) + ex_iterable = RandomlyCyclingMultiSourcesExamplesIterable( + ex_iterables, generator=generator, probabilities=probabilities, stopping_strategy=stopping_strategy + ) + # Set new info - we update the features + # setting the features also ensures to fill missing columns with None + if info is None: + info = DatasetInfo.from_merge([d.info for d in datasets]) + else: + info = info.copy() + info.features = features + # Get all the auth tokens per repository - in case the datasets come from different private repositories + token_per_repo_id = { + repo_id: token for dataset in datasets for repo_id, token in dataset._token_per_repo_id.items() + } + # Return new daset + return IterableDataset(ex_iterable=ex_iterable, info=info, split=split, token_per_repo_id=token_per_repo_id) + + +def _split_by_node_iterable_dataset(dataset: IterableDataset, rank: int, world_size: int) -> IterableDataset: + """ + Split an iterable dataset for the node at rank `rank` in a pool of nodes of size `world_size`. + + If the dataset has a number of shards that is a factor of `world_size` (i.e. if `dataset.num_shards % world_size == 0`), + then the shards are evenly assigned across the nodes, which is the most optimized. + Otherwise, each node keeps 1 example out of `world_size`, skipping the other examples. + + Args: + dataset ([`IterableDataset`]): + The iterable dataset to split by node. + rank (`int`): + Rank of the current node. + world_size (`int`): + Total number of nodes. + + Returns: + [`IterableDataset`]: The iterable dataset to be used on the node at rank `rank`. + """ + if dataset._distributed: + rank = world_size * dataset._distributed.rank + rank + world_size = world_size * dataset._distributed.world_size + distributed = DistributedConfig(rank=rank, world_size=world_size) + return IterableDataset( + ex_iterable=dataset._ex_iterable, + info=dataset._info.copy(), + split=dataset._split, + formatting=dataset._formatting, + shuffling=copy.deepcopy(dataset._shuffling), + distributed=distributed, + token_per_repo_id=dataset._token_per_repo_id, + ) + + +async def _apply_async(pool, func, x): + future = pool.apply_async(func, (x,)) + while True: + if future.ready(): + return future.get() + else: + await asyncio.sleep(0) diff --git a/lib/python3.12/site-packages/datasets/keyhash.py b/lib/python3.12/site-packages/datasets/keyhash.py new file mode 100644 index 0000000000000000000000000000000000000000..5ba2686e259868e19ee7439d21b995dfd4c8de51 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/keyhash.py @@ -0,0 +1,104 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 + +""" +Hashing function for dataset keys using `hashlib.md5` + +Requirements for the hash function: + +- Provides a uniformly distributed hash from random space +- Adequately fast speed +- Working with multiple input types (in this case, `str`, `int` or `bytes`) +- Should be platform independent (generates same hash on different OS and systems) + +The hashing function provides a unique 128-bit integer hash of the key provided. + +The split name is being used here as the hash salt to avoid having same hashes +in different splits due to same keys +""" + +from typing import Union + +from huggingface_hub.utils import insecure_hashlib + + +def _as_bytes(hash_data: Union[str, int, bytes, bytearray]) -> bytes: + """ + Returns the input hash_data in its bytes form + + Args: + hash_data: the hash salt/key to be converted to bytes + """ + if isinstance(hash_data, (bytes, bytearray)): + # Data already in bytes, returns as it as + return hash_data + elif isinstance(hash_data, str): + # We keep the data as it as for it ot be later encoded to UTF-8 + # However replace `\\` with `/` for Windows compatibility + hash_data = hash_data.replace("\\", "/") + elif isinstance(hash_data, int): + hash_data = str(hash_data) + else: + # If data is not of the required type, raise error + raise InvalidKeyError(hash_data) + + return hash_data.encode("utf-8") + + +class InvalidKeyError(Exception): + """Raises an error when given key is of invalid datatype.""" + + def __init__(self, hash_data): + self.prefix = "\nFAILURE TO GENERATE DATASET: Invalid key type detected" + self.err_msg = f"\nFound Key {hash_data} of type {type(hash_data)}" + self.suffix = "\nKeys should be either str, int or bytes type" + super().__init__(f"{self.prefix}{self.err_msg}{self.suffix}") + + +class DuplicatedKeysError(Exception): + """Raise an error when duplicate key found.""" + + def __init__(self, key, duplicate_key_indices, fix_msg=""): + self.key = key + self.duplicate_key_indices = duplicate_key_indices + self.fix_msg = fix_msg + self.prefix = "Found multiple examples generated with the same key" + if len(duplicate_key_indices) <= 20: + self.err_msg = f"\nThe examples at index {', '.join(duplicate_key_indices)} have the key {key}" + else: + self.err_msg = f"\nThe examples at index {', '.join(duplicate_key_indices[:20])}... ({len(duplicate_key_indices) - 20} more) have the key {key}" + self.suffix = "\n" + fix_msg if fix_msg else "" + super().__init__(f"{self.prefix}{self.err_msg}{self.suffix}") + + +class KeyHasher: + """KeyHasher class for providing hash using md5""" + + def __init__(self, hash_salt: str): + self._split_md5 = insecure_hashlib.md5(_as_bytes(hash_salt)) + + def hash(self, key: Union[str, int, bytes]) -> int: + """Returns 128-bits unique hash of input key + + Args: + key: the input key to be hashed (should be str, int or bytes) + + Returns: 128-bit int hash key""" + md5 = self._split_md5.copy() + byte_key = _as_bytes(key) + md5.update(byte_key) + # Convert to integer with hexadecimal conversion + return int(md5.hexdigest(), 16) diff --git a/lib/python3.12/site-packages/datasets/load.py b/lib/python3.12/site-packages/datasets/load.py new file mode 100644 index 0000000000000000000000000000000000000000..98e5f3b0b8fb4c89a8dc3e626e504eb0b6205d5a --- /dev/null +++ b/lib/python3.12/site-packages/datasets/load.py @@ -0,0 +1,2150 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +"""Access datasets.""" + +import filecmp +import glob +import importlib +import inspect +import json +import os +import posixpath +import shutil +import signal +import time +import warnings +from collections import Counter +from collections.abc import Mapping, Sequence +from contextlib import nullcontext +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Optional, Union + +import fsspec +import requests +import yaml +from fsspec.core import url_to_fs +from huggingface_hub import DatasetCard, DatasetCardData, HfApi +from huggingface_hub.utils import ( + EntryNotFoundError, + GatedRepoError, + LocalEntryNotFoundError, + OfflineModeIsEnabled, + RepositoryNotFoundError, + RevisionNotFoundError, + get_session, +) + +from . import __version__, config +from .arrow_dataset import Dataset +from .builder import BuilderConfig, DatasetBuilder +from .data_files import ( + DataFilesDict, + DataFilesList, + DataFilesPatternsDict, + EmptyDatasetError, + get_data_patterns, + sanitize_patterns, +) +from .dataset_dict import DatasetDict, IterableDatasetDict +from .download.download_config import DownloadConfig +from .download.download_manager import DownloadMode +from .download.streaming_download_manager import StreamingDownloadManager, xbasename, xglob, xjoin +from .exceptions import DataFilesNotFoundError, DatasetNotFoundError +from .features import Features +from .fingerprint import Hasher +from .info import DatasetInfo, DatasetInfosDict +from .iterable_dataset import IterableDataset +from .naming import camelcase_to_snakecase, snakecase_to_camelcase +from .packaged_modules import ( + _EXTENSION_TO_MODULE, + _MODULE_TO_EXTENSIONS, + _MODULE_TO_METADATA_FILE_NAMES, + _PACKAGED_DATASETS_MODULES, + _hash_python_lines, +) +from .packaged_modules.folder_based_builder.folder_based_builder import FolderBasedBuilder +from .splits import Split +from .utils import _dataset_viewer +from .utils.file_utils import ( + _raise_if_offline_mode_is_enabled, + cached_path, + get_datasets_user_agent, + init_hf_modules, + is_relative_path, + relative_to_absolute_path, + url_or_path_join, +) +from .utils.hub import hf_dataset_url +from .utils.info_utils import VerificationMode, is_small_dataset +from .utils.logging import get_logger +from .utils.metadata import MetadataConfigs +from .utils.py_utils import get_imports, lock_importable_file +from .utils.typing import PathLike +from .utils.version import Version + + +logger = get_logger(__name__) + +ALL_ALLOWED_EXTENSIONS = list(_EXTENSION_TO_MODULE.keys()) + [".zip"] + + +def _raise_timeout_error(signum, frame): + raise ValueError( + "Loading this dataset requires you to execute custom code contained in the dataset repository on your local " + "machine. Please set the option `trust_remote_code=True` to permit loading of this dataset." + ) + + +def resolve_trust_remote_code(trust_remote_code: Optional[bool], repo_id: str) -> bool: + """ + Copied and adapted from Transformers + https://github.com/huggingface/transformers/blob/2098d343cc4b4b9d2aea84b3cf1eb5a1e610deff/src/transformers/dynamic_module_utils.py#L589 + """ + trust_remote_code = trust_remote_code if trust_remote_code is not None else config.HF_DATASETS_TRUST_REMOTE_CODE + if trust_remote_code is None: + if config.TIME_OUT_REMOTE_CODE > 0: + try: + signal.signal(signal.SIGALRM, _raise_timeout_error) + signal.alarm(config.TIME_OUT_REMOTE_CODE) + while trust_remote_code is None: + answer = input( + f"The repository for {repo_id} contains custom code which must be executed to correctly " + f"load the dataset. You can inspect the repository content at https://hf.co/datasets/{repo_id}.\n" + f"You can avoid this prompt in future by passing the argument `trust_remote_code=True`.\n\n" + f"Do you wish to run the custom code? [y/N] " + ) + if answer.lower() in ["yes", "y", "1"]: + trust_remote_code = True + elif answer.lower() in ["no", "n", "0", ""]: + trust_remote_code = False + signal.alarm(0) + except Exception: + # OS which does not support signal.SIGALRM + raise ValueError( + f"The repository for {repo_id} contains custom code which must be executed to correctly " + f"load the dataset. You can inspect the repository content at https://hf.co/datasets/{repo_id}.\n" + f"Please pass the argument `trust_remote_code=True` to allow custom code to be run." + ) + else: + # For the CI which might put the timeout at 0 + _raise_timeout_error(None, None) + return trust_remote_code + + +def init_dynamic_modules( + name: str = config.MODULE_NAME_FOR_DYNAMIC_MODULES, hf_modules_cache: Optional[Union[Path, str]] = None +): + """ + Create a module with name `name` in which you can add dynamic modules + such as datasets. The module can be imported using its name. + The module is created in the HF_MODULE_CACHE directory by default (~/.cache/huggingface/modules) but it can + be overridden by specifying a path to another directory in `hf_modules_cache`. + """ + hf_modules_cache = init_hf_modules(hf_modules_cache) + dynamic_modules_path = os.path.join(hf_modules_cache, name) + os.makedirs(dynamic_modules_path, exist_ok=True) + if not os.path.exists(os.path.join(dynamic_modules_path, "__init__.py")): + with open(os.path.join(dynamic_modules_path, "__init__.py"), "w"): + pass + return dynamic_modules_path + + +def import_main_class(module_path) -> Optional[type[DatasetBuilder]]: + """Import a module at module_path and return its main class: a DatasetBuilder""" + module = importlib.import_module(module_path) + # Find the main class in our imported module + module_main_cls = None + for name, obj in module.__dict__.items(): + if inspect.isclass(obj) and issubclass(obj, DatasetBuilder): + if inspect.isabstract(obj): + continue + module_main_cls = obj + obj_module = inspect.getmodule(obj) + if obj_module is not None and module == obj_module: + break + + return module_main_cls + + +class _InitializeConfiguredDatasetBuilder: + """ + From https://stackoverflow.com/questions/4647566/pickle-a-dynamically-parameterized-sub-class + See also ConfiguredDatasetBuilder.__reduce__ + When called with the param value as the only argument, returns an + un-initialized instance of the parameterized class. Subsequent __setstate__ + will be called by pickle. + """ + + def __call__(self, builder_cls, metadata_configs, default_config_name, name): + # make a simple object which has no complex __init__ (this one will do) + obj = _InitializeConfiguredDatasetBuilder() + obj.__class__ = configure_builder_class( + builder_cls, metadata_configs, default_config_name=default_config_name, dataset_name=name + ) + return obj + + +def configure_builder_class( + builder_cls: type[DatasetBuilder], + builder_configs: list[BuilderConfig], + default_config_name: Optional[str], + dataset_name: str, +) -> type[DatasetBuilder]: + """ + Dynamically create a builder class with custom builder configs parsed from README.md file, + i.e. set BUILDER_CONFIGS class variable of a builder class to custom configs list. + """ + + class ConfiguredDatasetBuilder(builder_cls): + BUILDER_CONFIGS = builder_configs + DEFAULT_CONFIG_NAME = default_config_name + + __module__ = builder_cls.__module__ # so that the actual packaged builder can be imported + + def __reduce__(self): # to make dynamically created class pickable, see _InitializeParameterizedDatasetBuilder + parent_builder_cls = self.__class__.__mro__[1] + return ( + _InitializeConfiguredDatasetBuilder(), + ( + parent_builder_cls, + self.BUILDER_CONFIGS, + self.DEFAULT_CONFIG_NAME, + self.dataset_name, + ), + self.__dict__.copy(), + ) + + ConfiguredDatasetBuilder.__name__ = ( + f"{builder_cls.__name__.lower().capitalize()}{snakecase_to_camelcase(dataset_name)}" + ) + ConfiguredDatasetBuilder.__qualname__ = ( + f"{builder_cls.__name__.lower().capitalize()}{snakecase_to_camelcase(dataset_name)}" + ) + + return ConfiguredDatasetBuilder + + +def get_dataset_builder_class( + dataset_module: "DatasetModule", dataset_name: Optional[str] = None +) -> type[DatasetBuilder]: + with ( + lock_importable_file(dataset_module.importable_file_path) + if dataset_module.importable_file_path + else nullcontext() + ): + builder_cls = import_main_class(dataset_module.module_path) + if dataset_module.builder_configs_parameters.builder_configs: + dataset_name = dataset_name or dataset_module.builder_kwargs.get("dataset_name") + if dataset_name is None: + raise ValueError("dataset_name should be specified but got None") + builder_cls = configure_builder_class( + builder_cls, + builder_configs=dataset_module.builder_configs_parameters.builder_configs, + default_config_name=dataset_module.builder_configs_parameters.default_config_name, + dataset_name=dataset_name, + ) + return builder_cls + + +def files_to_hash(file_paths: list[str]) -> str: + """ + Convert a list of scripts or text files provided in file_paths into a hashed filename in a repeatable way. + """ + # List all python files in directories if directories are supplied as part of external imports + to_use_files: list[Union[Path, str]] = [] + for file_path in file_paths: + if os.path.isdir(file_path): + to_use_files.extend(list(Path(file_path).rglob("*.[pP][yY]"))) + else: + to_use_files.append(file_path) + + # Get the code from all these files + lines = [] + for file_path in to_use_files: + with open(file_path, encoding="utf-8") as f: + lines.extend(f.readlines()) + return _hash_python_lines(lines) + + +def increase_load_count(name: str): + """Update the download count of a dataset.""" + if not config.HF_HUB_OFFLINE and config.HF_UPDATE_DOWNLOAD_COUNTS: + try: + get_session().head( + "/".join((config.S3_DATASETS_BUCKET_PREFIX, name, name + ".py")), + headers={"User-Agent": get_datasets_user_agent()}, + timeout=3, + ) + except Exception: + pass + + +def _download_additional_modules( + name: str, base_path: str, imports: tuple[str, str, str, str], download_config: Optional[DownloadConfig] +) -> tuple[list[tuple[str, str]], list[tuple[str, str]]]: + """ + Download additional module for a module .py at URL (or local path) /.py + The imports must have been parsed first using ``get_imports``. + + If some modules need to be installed with pip, an error is raised showing how to install them. + This function return the list of downloaded modules as tuples (import_name, module_file_path). + + The downloaded modules can then be moved into an importable directory with ``_copy_script_and_other_resources_in_importable_dir``. + """ + local_imports = [] + library_imports = [] + download_config = download_config.copy() + if download_config.download_desc is None: + download_config.download_desc = "Downloading extra modules" + for import_type, import_name, import_path, sub_directory in imports: + if import_type == "library": + library_imports.append((import_name, import_path)) # Import from a library + continue + + if import_name == name: + raise ValueError( + f"Error in the {name} script, importing relative {import_name} module " + f"but {import_name} is the name of the script. " + f"Please change relative import {import_name} to another name and add a '# From: URL_OR_PATH' " + f"comment pointing to the original relative import file path." + ) + if import_type == "internal": + url_or_filename = url_or_path_join(base_path, import_path + ".py") + elif import_type == "external": + url_or_filename = import_path + else: + raise ValueError("Wrong import_type") + + local_import_path = cached_path( + url_or_filename, + download_config=download_config, + ) + if sub_directory is not None: + local_import_path = os.path.join(local_import_path, sub_directory) + local_imports.append((import_name, local_import_path)) + + return local_imports, library_imports + + +def _check_library_imports(name: str, library_imports: list[tuple[str, str]]) -> None: + # Check library imports + needs_to_be_installed = {} + for library_import_name, library_import_path in library_imports: + try: + lib = importlib.import_module(library_import_name) # noqa F841 + except ImportError: + if library_import_name not in needs_to_be_installed or library_import_path != library_import_name: + needs_to_be_installed[library_import_name] = library_import_path + if needs_to_be_installed: + _dependencies_str = "dependencies" if len(needs_to_be_installed) > 1 else "dependency" + _them_str = "them" if len(needs_to_be_installed) > 1 else "it" + if "sklearn" in needs_to_be_installed.keys(): + needs_to_be_installed["sklearn"] = "scikit-learn" + if "Bio" in needs_to_be_installed.keys(): + needs_to_be_installed["Bio"] = "biopython" + raise ImportError( + f"To be able to use {name}, you need to install the following {_dependencies_str}: " + f"{', '.join(needs_to_be_installed)}.\nPlease install {_them_str} using 'pip install " + f"{' '.join(needs_to_be_installed.values())}' for instance." + ) + + +def _copy_script_and_other_resources_in_importable_dir( + name: str, + importable_directory_path: str, + subdirectory_name: str, + original_local_path: str, + local_imports: list[tuple[str, str]], + additional_files: list[tuple[str, str]], + download_mode: Optional[Union[DownloadMode, str]], +) -> str: + """Copy a script and its required imports to an importable directory + + Args: + name (str): name of the resource to load + importable_directory_path (str): path to the loadable folder in the dynamic modules directory + subdirectory_name (str): name of the subdirectory in importable_directory_path in which to place the script + original_local_path (str): local path to the resource script + local_imports (List[Tuple[str, str]]): list of (destination_filename, import_file_to_copy) + additional_files (List[Tuple[str, str]]): list of (destination_filename, additional_file_to_copy) + download_mode (Optional[Union[DownloadMode, str]]): download mode + + Return: + importable_file: path to an importable module with importlib.import_module + """ + # Define a directory with a unique name in our dataset folder + # path is: ./datasets/dataset_name/hash_from_code/script.py + # we use a hash as subdirectory_name to be able to have multiple versions of a dataset processing file together + importable_subdirectory = os.path.join(importable_directory_path, subdirectory_name) + importable_file = os.path.join(importable_subdirectory, name + ".py") + # Prevent parallel disk operations + with lock_importable_file(importable_file): + # Create main dataset folder if needed + if download_mode == DownloadMode.FORCE_REDOWNLOAD and os.path.exists(importable_directory_path): + shutil.rmtree(importable_directory_path) + os.makedirs(importable_directory_path, exist_ok=True) + + # add an __init__ file to the main dataset folder if needed + init_file_path = os.path.join(importable_directory_path, "__init__.py") + if not os.path.exists(init_file_path): + with open(init_file_path, "w"): + pass + + # Create hash dataset folder if needed + os.makedirs(importable_subdirectory, exist_ok=True) + # add an __init__ file to the hash dataset folder if needed + init_file_path = os.path.join(importable_subdirectory, "__init__.py") + if not os.path.exists(init_file_path): + with open(init_file_path, "w"): + pass + + # Copy dataset.py file in hash folder if needed + if not os.path.exists(importable_file): + shutil.copyfile(original_local_path, importable_file) + # Record metadata associating original dataset path with local unique folder + # Use os.path.splitext to split extension from importable_local_file + meta_path = os.path.splitext(importable_file)[0] + ".json" + if not os.path.exists(meta_path): + meta = {"original file path": original_local_path, "local file path": importable_file} + # the filename is *.py in our case, so better rename to filename.json instead of filename.py.json + with open(meta_path, "w", encoding="utf-8") as meta_file: + json.dump(meta, meta_file) + + # Copy all the additional imports + for import_name, import_path in local_imports: + if os.path.isfile(import_path): + full_path_local_import = os.path.join(importable_subdirectory, import_name + ".py") + if not os.path.exists(full_path_local_import): + shutil.copyfile(import_path, full_path_local_import) + elif os.path.isdir(import_path): + full_path_local_import = os.path.join(importable_subdirectory, import_name) + if not os.path.exists(full_path_local_import): + shutil.copytree(import_path, full_path_local_import) + else: + raise ImportError(f"Error with local import at {import_path}") + + # Copy additional files like dataset_infos.json file if needed + for file_name, original_path in additional_files: + destination_additional_path = os.path.join(importable_subdirectory, file_name) + if not os.path.exists(destination_additional_path) or not filecmp.cmp( + original_path, destination_additional_path + ): + shutil.copyfile(original_path, destination_additional_path) + return importable_file + + +def _get_importable_file_path( + dynamic_modules_path: str, + module_namespace: str, + subdirectory_name: str, + name: str, +) -> str: + importable_directory_path = os.path.join(dynamic_modules_path, module_namespace, name.replace("/", "--")) + return os.path.join(importable_directory_path, subdirectory_name, name.split("/")[-1] + ".py") + + +def _create_importable_file( + local_path: str, + local_imports: list[tuple[str, str]], + additional_files: list[tuple[str, str]], + dynamic_modules_path: str, + module_namespace: str, + subdirectory_name: str, + name: str, + download_mode: DownloadMode, +) -> None: + importable_directory_path = os.path.join(dynamic_modules_path, module_namespace, name.replace("/", "--")) + Path(importable_directory_path).mkdir(parents=True, exist_ok=True) + (Path(importable_directory_path).parent / "__init__.py").touch(exist_ok=True) + importable_local_file = _copy_script_and_other_resources_in_importable_dir( + name=name.split("/")[-1], + importable_directory_path=importable_directory_path, + subdirectory_name=subdirectory_name, + original_local_path=local_path, + local_imports=local_imports, + additional_files=additional_files, + download_mode=download_mode, + ) + logger.debug(f"Created importable dataset file at {importable_local_file}") + + +def _load_importable_file( + dynamic_modules_path: str, + module_namespace: str, + subdirectory_name: str, + name: str, +) -> tuple[str, str]: + module_path = ".".join( + [ + os.path.basename(dynamic_modules_path), + module_namespace, + name.replace("/", "--"), + subdirectory_name, + name.split("/")[-1], + ] + ) + return module_path, subdirectory_name + + +def infer_module_for_data_files_list( + data_files_list: DataFilesList, download_config: Optional[DownloadConfig] = None +) -> tuple[Optional[str], dict]: + """Infer module (and builder kwargs) from list of data files. + + It picks the module based on the most common file extension. + In case of a draw ".parquet" is the favorite, and then alphabetical order. + + Args: + data_files_list (DataFilesList): List of data files. + download_config (bool or str, optional): Mainly use `token` or `storage_options` to support different platforms and auth types. + + Returns: + tuple[str, dict[str, Any]]: Tuple with + - inferred module name + - dict of builder kwargs + """ + extensions_counter = Counter( + ("." + suffix.lower(), xbasename(filepath) in FolderBasedBuilder.METADATA_FILENAMES) + for filepath in data_files_list[: config.DATA_FILES_MAX_NUMBER_FOR_MODULE_INFERENCE] + for suffix in xbasename(filepath).split(".")[1:] + ) + if extensions_counter: + + def sort_key(ext_count: tuple[tuple[str, bool], int]) -> tuple[int, bool]: + """Sort by count and set ".parquet" as the favorite in case of a draw, and ignore metadata files""" + (ext, is_metadata), count = ext_count + return (not is_metadata, count, ext == ".parquet", ext == ".jsonl", ext == ".json", ext == ".csv", ext) + + for (ext, _), _ in sorted(extensions_counter.items(), key=sort_key, reverse=True): + if ext in _EXTENSION_TO_MODULE: + return _EXTENSION_TO_MODULE[ext] + elif ext == ".zip": + return infer_module_for_data_files_list_in_archives(data_files_list, download_config=download_config) + return None, {} + + +def infer_module_for_data_files_list_in_archives( + data_files_list: DataFilesList, download_config: Optional[DownloadConfig] = None +) -> tuple[Optional[str], dict]: + """Infer module (and builder kwargs) from list of archive data files. + + Args: + data_files_list (DataFilesList): List of data files. + download_config (bool or str, optional): Mainly use `token` or `storage_options` to support different platforms and auth types. + + Returns: + tuple[str, dict[str, Any]]: Tuple with + - inferred module name + - dict of builder kwargs + """ + archived_files = [] + archive_files_counter = 0 + for filepath in data_files_list: + if str(filepath).endswith(".zip"): + archive_files_counter += 1 + if archive_files_counter > config.GLOBBED_DATA_FILES_MAX_NUMBER_FOR_MODULE_INFERENCE: + break + extracted = xjoin(StreamingDownloadManager().extract(filepath), "**") + archived_files += [ + f.split("::")[0] + for f in xglob(extracted, recursive=True, download_config=download_config)[ + : config.ARCHIVED_DATA_FILES_MAX_NUMBER_FOR_MODULE_INFERENCE + ] + ] + extensions_counter = Counter( + "." + suffix.lower() for filepath in archived_files for suffix in xbasename(filepath).split(".")[1:] + ) + if extensions_counter: + most_common = extensions_counter.most_common(1)[0][0] + if most_common in _EXTENSION_TO_MODULE: + return _EXTENSION_TO_MODULE[most_common] + return None, {} + + +def infer_module_for_data_files( + data_files: DataFilesDict, path: Optional[str] = None, download_config: Optional[DownloadConfig] = None +) -> tuple[Optional[str], dict[str, Any]]: + """Infer module (and builder kwargs) from data files. Raise if module names for different splits don't match. + + Args: + data_files ([`DataFilesDict`]): Dict of list of data files. + path (str, *optional*): Dataset name or path. + download_config ([`DownloadConfig`], *optional*): + Specific download configuration parameters to authenticate on the Hugging Face Hub for private remote files. + + Returns: + tuple[str, dict[str, Any]]: Tuple with + - inferred module name + - builder kwargs + """ + split_modules = { + split: infer_module_for_data_files_list(data_files_list, download_config=download_config) + for split, data_files_list in data_files.items() + } + module_name, default_builder_kwargs = next(iter(split_modules.values())) + if any((module_name, default_builder_kwargs) != split_module for split_module in split_modules.values()): + raise ValueError(f"Couldn't infer the same data file format for all splits. Got {split_modules}") + if not module_name: + raise DataFilesNotFoundError("No (supported) data files found" + (f" in {path}" if path else "")) + return module_name, default_builder_kwargs + + +def create_builder_configs_from_metadata_configs( + module_path: str, + metadata_configs: MetadataConfigs, + base_path: Optional[str] = None, + default_builder_kwargs: dict[str, Any] = None, + download_config: Optional[DownloadConfig] = None, +) -> tuple[list[BuilderConfig], str]: + builder_cls = import_main_class(module_path) + builder_config_cls = builder_cls.BUILDER_CONFIG_CLASS + default_config_name = metadata_configs.get_default_config_name() + builder_configs = [] + default_builder_kwargs = {} if default_builder_kwargs is None else default_builder_kwargs + + base_path = base_path if base_path is not None else "" + for config_name, config_params in metadata_configs.items(): + config_data_files = config_params.get("data_files") + config_data_dir = config_params.get("data_dir") + config_base_path = xjoin(base_path, config_data_dir) if config_data_dir else base_path + try: + config_patterns = ( + sanitize_patterns(config_data_files) + if config_data_files is not None + else get_data_patterns(config_base_path, download_config=download_config) + ) + config_data_files_dict = DataFilesPatternsDict.from_patterns( + config_patterns, + allowed_extensions=ALL_ALLOWED_EXTENSIONS, + ) + except EmptyDatasetError as e: + raise EmptyDatasetError( + f"Dataset at '{base_path}' doesn't contain data files matching the patterns for config '{config_name}'," + f" check `data_files` and `data_fir` parameters in the `configs` YAML field in README.md. " + ) from e + ignored_params = [ + param for param in config_params if not hasattr(builder_config_cls, param) and param != "default" + ] + if ignored_params: + logger.warning( + f"Some datasets params were ignored: {ignored_params}. " + "Make sure to use only valid params for the dataset builder and to have " + "a up-to-date version of the `datasets` library." + ) + builder_configs.append( + builder_config_cls( + name=config_name, + data_files=config_data_files_dict, + data_dir=config_data_dir, + **{ + param: value + for param, value in {**default_builder_kwargs, **config_params}.items() + if hasattr(builder_config_cls, param) and param not in ("default", "data_files", "data_dir") + }, + ) + ) + return builder_configs, default_config_name + + +@dataclass +class BuilderConfigsParameters: + """Dataclass containing objects related to creation of builder configurations from yaml's metadata content. + + Attributes: + metadata_configs (`MetadataConfigs`, *optional*): + Configs parsed from yaml's metadata. + builder_configs (`list[BuilderConfig]`, *optional*): + List of BuilderConfig objects created from metadata_configs above. + default_config_name (`str`): + Name of default config taken from yaml's metadata. + """ + + metadata_configs: Optional[MetadataConfigs] = None + builder_configs: Optional[list[BuilderConfig]] = None + default_config_name: Optional[str] = None + + +@dataclass +class DatasetModule: + module_path: str + hash: str + builder_kwargs: dict + builder_configs_parameters: BuilderConfigsParameters = field(default_factory=BuilderConfigsParameters) + dataset_infos: Optional[DatasetInfosDict] = None + importable_file_path: Optional[str] = None + + +class _DatasetModuleFactory: + def get_module(self) -> DatasetModule: + raise NotImplementedError + + +class LocalDatasetModuleFactoryWithScript(_DatasetModuleFactory): + """Get the module of a local dataset. The dataset script is loaded from a local script.""" + + def __init__( + self, + path: str, + download_config: Optional[DownloadConfig] = None, + download_mode: Optional[Union[DownloadMode, str]] = None, + dynamic_modules_path: Optional[str] = None, + trust_remote_code: Optional[bool] = None, + ): + self.path = path + self.name = Path(path).stem + self.download_config = download_config or DownloadConfig() + self.download_mode = download_mode + self.dynamic_modules_path = dynamic_modules_path + self.trust_remote_code = trust_remote_code + + def get_module(self) -> DatasetModule: + if config.HF_DATASETS_TRUST_REMOTE_CODE and self.trust_remote_code is None: + warnings.warn( + f"The repository for {self.name} contains custom code which must be executed to correctly " + f"load the dataset. You can inspect the repository content at {self.path}\n" + f"You can avoid this message in future by passing the argument `trust_remote_code=True`.\n" + f"Passing `trust_remote_code=True` will be mandatory to load this dataset from the next major release of `datasets`.", + FutureWarning, + ) + # get script and other files + dataset_infos_path = Path(self.path).parent / config.DATASETDICT_INFOS_FILENAME + dataset_readme_path = Path(self.path).parent / config.REPOCARD_FILENAME + imports = get_imports(self.path) + local_imports, library_imports = _download_additional_modules( + name=self.name, + base_path=str(Path(self.path).parent), + imports=imports, + download_config=self.download_config, + ) + additional_files = [] + if dataset_infos_path.is_file(): + additional_files.append((config.DATASETDICT_INFOS_FILENAME, str(dataset_infos_path))) + if dataset_readme_path.is_file(): + additional_files.append((config.REPOCARD_FILENAME, dataset_readme_path)) + # copy the script and the files in an importable directory + dynamic_modules_path = self.dynamic_modules_path if self.dynamic_modules_path else init_dynamic_modules() + hash = files_to_hash([self.path] + [loc[1] for loc in local_imports]) + importable_file_path = _get_importable_file_path( + dynamic_modules_path=dynamic_modules_path, + module_namespace="datasets", + subdirectory_name=hash, + name=self.name, + ) + if not os.path.exists(importable_file_path): + trust_remote_code = resolve_trust_remote_code(self.trust_remote_code, self.name) + if trust_remote_code: + _create_importable_file( + local_path=self.path, + local_imports=local_imports, + additional_files=additional_files, + dynamic_modules_path=dynamic_modules_path, + module_namespace="datasets", + subdirectory_name=hash, + name=self.name, + download_mode=self.download_mode, + ) + else: + raise ValueError( + f"Loading {self.name} requires you to execute the dataset script in that" + " repo on your local machine. Make sure you have read the code there to avoid malicious use, then" + " set the option `trust_remote_code=True` to remove this error." + ) + _check_library_imports(name=self.name, library_imports=library_imports) + module_path, hash = _load_importable_file( + dynamic_modules_path=dynamic_modules_path, + module_namespace="datasets", + subdirectory_name=hash, + name=self.name, + ) + + # make the new module to be noticed by the import system + importlib.invalidate_caches() + builder_kwargs = {"base_path": str(Path(self.path).parent)} + return DatasetModule(module_path, hash, builder_kwargs, importable_file_path=importable_file_path) + + +class LocalDatasetModuleFactoryWithoutScript(_DatasetModuleFactory): + """Get the module of a dataset loaded from the user's data files. The dataset builder module to use is inferred + from the data files extensions.""" + + def __init__( + self, + path: str, + data_dir: Optional[str] = None, + data_files: Optional[Union[str, list, dict]] = None, + download_mode: Optional[Union[DownloadMode, str]] = None, + ): + if data_dir and os.path.isabs(data_dir): + raise ValueError(f"`data_dir` must be relative to a dataset directory's root: {path}") + + self.path = Path(path).as_posix() + self.name = Path(path).stem + self.data_files = data_files + self.data_dir = data_dir + self.download_mode = download_mode + + def get_module(self) -> DatasetModule: + readme_path = os.path.join(self.path, config.REPOCARD_FILENAME) + standalone_yaml_path = os.path.join(self.path, config.REPOYAML_FILENAME) + dataset_card_data = DatasetCard.load(readme_path).data if os.path.isfile(readme_path) else DatasetCardData() + if os.path.exists(standalone_yaml_path): + with open(standalone_yaml_path, encoding="utf-8") as f: + standalone_yaml_data = yaml.safe_load(f.read()) + if standalone_yaml_data: + _dataset_card_data_dict = dataset_card_data.to_dict() + _dataset_card_data_dict.update(standalone_yaml_data) + dataset_card_data = DatasetCardData(**_dataset_card_data_dict) + metadata_configs = MetadataConfigs.from_dataset_card_data(dataset_card_data) + dataset_infos = DatasetInfosDict.from_dataset_card_data(dataset_card_data) + # we need a set of data files to find which dataset builder to use + # because we need to infer module name by files extensions + base_path = Path(self.path, self.data_dir or "").expanduser().resolve().as_posix() + if self.data_files is not None: + patterns = sanitize_patterns(self.data_files) + elif metadata_configs and not self.data_dir and "data_files" in next(iter(metadata_configs.values())): + patterns = sanitize_patterns(next(iter(metadata_configs.values()))["data_files"]) + else: + patterns = get_data_patterns(base_path) + data_files = DataFilesDict.from_patterns( + patterns, + base_path=base_path, + allowed_extensions=ALL_ALLOWED_EXTENSIONS, + ) + module_name, default_builder_kwargs = infer_module_for_data_files( + data_files=data_files, + path=self.path, + ) + data_files = data_files.filter( + extensions=_MODULE_TO_EXTENSIONS[module_name], file_names=_MODULE_TO_METADATA_FILE_NAMES[module_name] + ) + module_path, _ = _PACKAGED_DATASETS_MODULES[module_name] + if metadata_configs: + builder_configs, default_config_name = create_builder_configs_from_metadata_configs( + module_path, + metadata_configs, + base_path=base_path, + default_builder_kwargs=default_builder_kwargs, + ) + else: + builder_configs: list[BuilderConfig] = [ + import_main_class(module_path).BUILDER_CONFIG_CLASS( + data_files=data_files, + **default_builder_kwargs, + ) + ] + default_config_name = None + builder_kwargs = { + "base_path": self.path, + "dataset_name": camelcase_to_snakecase(Path(self.path).name), + } + if self.data_dir: + builder_kwargs["data_files"] = data_files + # this file is deprecated and was created automatically in old versions of push_to_hub + if os.path.isfile(os.path.join(self.path, config.DATASETDICT_INFOS_FILENAME)): + with open(os.path.join(self.path, config.DATASETDICT_INFOS_FILENAME), encoding="utf-8") as f: + legacy_dataset_infos = DatasetInfosDict( + { + config_name: DatasetInfo.from_dict(dataset_info_dict) + for config_name, dataset_info_dict in json.load(f).items() + } + ) + if len(legacy_dataset_infos) == 1: + # old config e.g. named "username--dataset_name" + legacy_config_name = next(iter(legacy_dataset_infos)) + legacy_dataset_infos["default"] = legacy_dataset_infos.pop(legacy_config_name) + legacy_dataset_infos.update(dataset_infos) + dataset_infos = legacy_dataset_infos + if default_config_name is None and len(dataset_infos) == 1: + default_config_name = next(iter(dataset_infos)) + + hash = Hasher.hash({"dataset_infos": dataset_infos, "builder_configs": builder_configs}) + return DatasetModule( + module_path, + hash, + builder_kwargs, + dataset_infos=dataset_infos, + builder_configs_parameters=BuilderConfigsParameters( + metadata_configs=metadata_configs, + builder_configs=builder_configs, + default_config_name=default_config_name, + ), + ) + + +class PackagedDatasetModuleFactory(_DatasetModuleFactory): + """Get the dataset builder module from the ones that are packaged with the library: csv, json, etc.""" + + def __init__( + self, + name: str, + data_dir: Optional[str] = None, + data_files: Optional[Union[str, list, dict]] = None, + download_config: Optional[DownloadConfig] = None, + download_mode: Optional[Union[DownloadMode, str]] = None, + ): + self.name = name + self.data_files = data_files + self.data_dir = data_dir + self.download_config = download_config + self.download_mode = download_mode + increase_load_count(name) + + def get_module(self) -> DatasetModule: + base_path = Path(self.data_dir or "").expanduser().resolve().as_posix() + patterns = ( + sanitize_patterns(self.data_files) + if self.data_files is not None + else get_data_patterns(base_path, download_config=self.download_config) + ) + data_files = DataFilesDict.from_patterns( + patterns, + download_config=self.download_config, + base_path=base_path, + ) + + module_path, hash = _PACKAGED_DATASETS_MODULES[self.name] + + builder_kwargs = { + "data_files": data_files, + "dataset_name": self.name, + } + + return DatasetModule(module_path, hash, builder_kwargs) + + +class HubDatasetModuleFactoryWithoutScript(_DatasetModuleFactory): + """ + Get the module of a dataset loaded from data files of a dataset repository. + The dataset builder module to use is inferred from the data files extensions. + """ + + def __init__( + self, + name: str, + commit_hash: str, + data_dir: Optional[str] = None, + data_files: Optional[Union[str, list, dict]] = None, + download_config: Optional[DownloadConfig] = None, + download_mode: Optional[Union[DownloadMode, str]] = None, + use_exported_dataset_infos: bool = False, + ): + self.name = name + self.commit_hash = commit_hash + self.data_files = data_files + self.data_dir = data_dir + self.download_config = download_config or DownloadConfig() + self.download_mode = download_mode + self.use_exported_dataset_infos = use_exported_dataset_infos + increase_load_count(name) + + def get_module(self) -> DatasetModule: + # Get the Dataset Card and fix the revision in case there are new commits in the meantime + api = HfApi( + endpoint=config.HF_ENDPOINT, + token=self.download_config.token, + library_name="datasets", + library_version=__version__, + user_agent=get_datasets_user_agent(self.download_config.user_agent), + ) + try: + dataset_readme_path = api.hf_hub_download( + repo_id=self.name, + filename=config.REPOCARD_FILENAME, + repo_type="dataset", + revision=self.commit_hash, + proxies=self.download_config.proxies, + ) + dataset_card_data = DatasetCard.load(dataset_readme_path).data + except EntryNotFoundError: + dataset_card_data = DatasetCardData() + download_config = self.download_config.copy() + if download_config.download_desc is None: + download_config.download_desc = "Downloading standalone yaml" + try: + standalone_yaml_path = cached_path( + hf_dataset_url(self.name, config.REPOYAML_FILENAME, revision=self.commit_hash), + download_config=download_config, + ) + with open(standalone_yaml_path, encoding="utf-8") as f: + standalone_yaml_data = yaml.safe_load(f.read()) + if standalone_yaml_data: + _dataset_card_data_dict = dataset_card_data.to_dict() + _dataset_card_data_dict.update(standalone_yaml_data) + dataset_card_data = DatasetCardData(**_dataset_card_data_dict) + except FileNotFoundError: + pass + base_path = f"hf://datasets/{self.name}@{self.commit_hash}/{self.data_dir or ''}".rstrip("/") + metadata_configs = MetadataConfigs.from_dataset_card_data(dataset_card_data) + dataset_infos = DatasetInfosDict.from_dataset_card_data(dataset_card_data) + if config.USE_PARQUET_EXPORT and self.use_exported_dataset_infos: + try: + exported_dataset_infos = _dataset_viewer.get_exported_dataset_infos( + dataset=self.name, commit_hash=self.commit_hash, token=self.download_config.token + ) + exported_dataset_infos = DatasetInfosDict( + { + config_name: DatasetInfo.from_dict(exported_dataset_infos[config_name]) + for config_name in exported_dataset_infos + } + ) + except _dataset_viewer.DatasetViewerError: + exported_dataset_infos = None + else: + exported_dataset_infos = None + if exported_dataset_infos: + exported_dataset_infos.update(dataset_infos) + dataset_infos = exported_dataset_infos + # we need a set of data files to find which dataset builder to use + # because we need to infer module name by files extensions + if self.data_files is not None: + patterns = sanitize_patterns(self.data_files) + elif metadata_configs and not self.data_dir and "data_files" in next(iter(metadata_configs.values())): + patterns = sanitize_patterns(next(iter(metadata_configs.values()))["data_files"]) + else: + patterns = get_data_patterns(base_path, download_config=self.download_config) + data_files = DataFilesDict.from_patterns( + patterns, + base_path=base_path, + allowed_extensions=ALL_ALLOWED_EXTENSIONS, + download_config=self.download_config, + ) + module_name, default_builder_kwargs = infer_module_for_data_files( + data_files=data_files, + path=self.name, + download_config=self.download_config, + ) + data_files = data_files.filter( + extensions=_MODULE_TO_EXTENSIONS[module_name], file_names=_MODULE_TO_METADATA_FILE_NAMES[module_name] + ) + module_path, _ = _PACKAGED_DATASETS_MODULES[module_name] + if metadata_configs: + builder_configs, default_config_name = create_builder_configs_from_metadata_configs( + module_path, + metadata_configs, + base_path=base_path, + default_builder_kwargs=default_builder_kwargs, + download_config=self.download_config, + ) + else: + builder_configs: list[BuilderConfig] = [ + import_main_class(module_path).BUILDER_CONFIG_CLASS( + data_files=data_files, + **default_builder_kwargs, + ) + ] + default_config_name = None + builder_kwargs = { + "base_path": hf_dataset_url(self.name, "", revision=self.commit_hash).rstrip("/"), + "repo_id": self.name, + "dataset_name": camelcase_to_snakecase(Path(self.name).name), + } + if self.data_dir: + builder_kwargs["data_files"] = data_files + download_config = self.download_config.copy() + if download_config.download_desc is None: + download_config.download_desc = "Downloading metadata" + try: + # this file is deprecated and was created automatically in old versions of push_to_hub + dataset_infos_path = cached_path( + hf_dataset_url(self.name, config.DATASETDICT_INFOS_FILENAME, revision=self.commit_hash), + download_config=download_config, + ) + with open(dataset_infos_path, encoding="utf-8") as f: + legacy_dataset_infos = DatasetInfosDict( + { + config_name: DatasetInfo.from_dict(dataset_info_dict) + for config_name, dataset_info_dict in json.load(f).items() + } + ) + if len(legacy_dataset_infos) == 1: + # old config e.g. named "username--dataset_name" + legacy_config_name = next(iter(legacy_dataset_infos)) + legacy_dataset_infos["default"] = legacy_dataset_infos.pop(legacy_config_name) + legacy_dataset_infos.update(dataset_infos) + dataset_infos = legacy_dataset_infos + except FileNotFoundError: + pass + if default_config_name is None and len(dataset_infos) == 1: + default_config_name = next(iter(dataset_infos)) + + return DatasetModule( + module_path, + self.commit_hash, + builder_kwargs, + dataset_infos=dataset_infos, + builder_configs_parameters=BuilderConfigsParameters( + metadata_configs=metadata_configs, + builder_configs=builder_configs, + default_config_name=default_config_name, + ), + ) + + +class HubDatasetModuleFactoryWithParquetExport(_DatasetModuleFactory): + """ + Get the module of a dataset loaded from parquet files of a dataset repository parquet export. + """ + + def __init__( + self, + name: str, + commit_hash: str, + download_config: Optional[DownloadConfig] = None, + ): + self.name = name + self.commit_hash = commit_hash + self.download_config = download_config or DownloadConfig() + increase_load_count(name) + + def get_module(self) -> DatasetModule: + exported_parquet_files = _dataset_viewer.get_exported_parquet_files( + dataset=self.name, commit_hash=self.commit_hash, token=self.download_config.token + ) + exported_dataset_infos = _dataset_viewer.get_exported_dataset_infos( + dataset=self.name, commit_hash=self.commit_hash, token=self.download_config.token + ) + dataset_infos = DatasetInfosDict( + { + config_name: DatasetInfo.from_dict(exported_dataset_infos[config_name]) + for config_name in exported_dataset_infos + } + ) + parquet_commit_hash = ( + HfApi( + endpoint=config.HF_ENDPOINT, + token=self.download_config.token, + library_name="datasets", + library_version=__version__, + user_agent=get_datasets_user_agent(self.download_config.user_agent), + ) + .dataset_info( + self.name, + revision="refs/convert/parquet", + token=self.download_config.token, + timeout=100.0, + ) + .sha + ) # fix the revision in case there are new commits in the meantime + metadata_configs = MetadataConfigs._from_exported_parquet_files_and_dataset_infos( + parquet_commit_hash=parquet_commit_hash, + exported_parquet_files=exported_parquet_files, + dataset_infos=dataset_infos, + ) + module_path, _ = _PACKAGED_DATASETS_MODULES["parquet"] + builder_configs, default_config_name = create_builder_configs_from_metadata_configs( + module_path, + metadata_configs, + download_config=self.download_config, + ) + builder_kwargs = { + "repo_id": self.name, + "dataset_name": camelcase_to_snakecase(Path(self.name).name), + } + + return DatasetModule( + module_path, + self.commit_hash, + builder_kwargs, + dataset_infos=dataset_infos, + builder_configs_parameters=BuilderConfigsParameters( + metadata_configs=metadata_configs, + builder_configs=builder_configs, + default_config_name=default_config_name, + ), + ) + + +class HubDatasetModuleFactoryWithScript(_DatasetModuleFactory): + """ + Get the module of a dataset from a dataset repository. + The dataset script comes from the script inside the dataset repository. + """ + + def __init__( + self, + name: str, + commit_hash: str, + download_config: Optional[DownloadConfig] = None, + download_mode: Optional[Union[DownloadMode, str]] = None, + dynamic_modules_path: Optional[str] = None, + trust_remote_code: Optional[bool] = None, + ): + self.name = name + self.commit_hash = commit_hash + self.download_config = download_config or DownloadConfig() + self.download_mode = download_mode + self.dynamic_modules_path = dynamic_modules_path + self.trust_remote_code = trust_remote_code + increase_load_count(name) + + def download_loading_script(self) -> str: + file_path = hf_dataset_url(self.name, self.name.split("/")[-1] + ".py", revision=self.commit_hash) + download_config = self.download_config.copy() + if download_config.download_desc is None: + download_config.download_desc = "Downloading builder script" + return cached_path(file_path, download_config=download_config) + + def download_dataset_infos_file(self) -> str: + dataset_infos = hf_dataset_url(self.name, config.DATASETDICT_INFOS_FILENAME, revision=self.commit_hash) + # Download the dataset infos file if available + download_config = self.download_config.copy() + if download_config.download_desc is None: + download_config.download_desc = "Downloading metadata" + try: + return cached_path( + dataset_infos, + download_config=download_config, + ) + except (FileNotFoundError, ConnectionError): + return None + + def download_dataset_readme_file(self) -> str: + readme_url = hf_dataset_url(self.name, config.REPOCARD_FILENAME, revision=self.commit_hash) + # Download the dataset infos file if available + download_config = self.download_config.copy() + if download_config.download_desc is None: + download_config.download_desc = "Downloading readme" + try: + return cached_path( + readme_url, + download_config=download_config, + ) + except (FileNotFoundError, ConnectionError): + return None + + def get_module(self) -> DatasetModule: + if config.HF_DATASETS_TRUST_REMOTE_CODE and self.trust_remote_code is None: + warnings.warn( + f"The repository for {self.name} contains custom code which must be executed to correctly " + f"load the dataset. You can inspect the repository content at https://hf.co/datasets/{self.name}\n" + f"You can avoid this message in future by passing the argument `trust_remote_code=True`.\n" + f"Passing `trust_remote_code=True` will be mandatory to load this dataset from the next major release of `datasets`.", + FutureWarning, + ) + # get script and other files + local_path = self.download_loading_script() + dataset_infos_path = self.download_dataset_infos_file() + dataset_readme_path = self.download_dataset_readme_file() + imports = get_imports(local_path) + local_imports, library_imports = _download_additional_modules( + name=self.name, + base_path=hf_dataset_url(self.name, "", revision=self.commit_hash), + imports=imports, + download_config=self.download_config, + ) + additional_files = [] + if dataset_infos_path: + additional_files.append((config.DATASETDICT_INFOS_FILENAME, dataset_infos_path)) + if dataset_readme_path: + additional_files.append((config.REPOCARD_FILENAME, dataset_readme_path)) + # copy the script and the files in an importable directory + dynamic_modules_path = self.dynamic_modules_path if self.dynamic_modules_path else init_dynamic_modules() + hash = files_to_hash([local_path] + [loc[1] for loc in local_imports]) + importable_file_path = _get_importable_file_path( + dynamic_modules_path=dynamic_modules_path, + module_namespace="datasets", + subdirectory_name=hash, + name=self.name, + ) + if not os.path.exists(importable_file_path): + trust_remote_code = resolve_trust_remote_code(self.trust_remote_code, self.name) + if trust_remote_code: + _create_importable_file( + local_path=local_path, + local_imports=local_imports, + additional_files=additional_files, + dynamic_modules_path=dynamic_modules_path, + module_namespace="datasets", + subdirectory_name=hash, + name=self.name, + download_mode=self.download_mode, + ) + else: + raise ValueError( + f"Loading {self.name} requires you to execute the dataset script in that" + " repo on your local machine. Make sure you have read the code there to avoid malicious use, then" + " set the option `trust_remote_code=True` to remove this error." + ) + _check_library_imports(name=self.name, library_imports=library_imports) + module_path, hash = _load_importable_file( + dynamic_modules_path=dynamic_modules_path, + module_namespace="datasets", + subdirectory_name=hash, + name=self.name, + ) + # make the new module to be noticed by the import system + importlib.invalidate_caches() + builder_kwargs = { + "base_path": hf_dataset_url(self.name, "", revision=self.commit_hash).rstrip("/"), + "repo_id": self.name, + } + return DatasetModule(module_path, hash, builder_kwargs, importable_file_path=importable_file_path) + + +class CachedDatasetModuleFactory(_DatasetModuleFactory): + """ + Get the module of a dataset that has been loaded once already and cached. + The script that is loaded from the cache is the most recent one with a matching name. + """ + + def __init__( + self, + name: str, + cache_dir: Optional[str] = None, + dynamic_modules_path: Optional[str] = None, + ): + self.name = name + self.cache_dir = cache_dir + self.dynamic_modules_path = dynamic_modules_path + assert self.name.count("/") <= 1 + + def get_module(self) -> DatasetModule: + dynamic_modules_path = self.dynamic_modules_path if self.dynamic_modules_path else init_dynamic_modules() + importable_directory_path = os.path.join(dynamic_modules_path, "datasets", self.name.replace("/", "--")) + hashes = ( + [h for h in os.listdir(importable_directory_path) if len(h) == 64] + if os.path.isdir(importable_directory_path) + else None + ) + if hashes: + # get most recent + def _get_modification_time(module_hash): + return ( + (Path(importable_directory_path) / module_hash / (self.name.split("/")[-1] + ".py")) + .stat() + .st_mtime + ) + + hash = sorted(hashes, key=_get_modification_time)[-1] + warning_msg = ( + f"Using the latest cached version of the module from {os.path.join(importable_directory_path, hash)} " + f"(last modified on {time.ctime(_get_modification_time(hash))}) since it " + f"couldn't be found locally at {self.name}" + ) + if not config.HF_HUB_OFFLINE: + warning_msg += ", or remotely on the Hugging Face Hub." + logger.warning(warning_msg) + importable_file_path = _get_importable_file_path( + dynamic_modules_path=dynamic_modules_path, + module_namespace="datasets", + subdirectory_name=hash, + name=self.name, + ) + module_path, hash = _load_importable_file( + dynamic_modules_path=dynamic_modules_path, + module_namespace="datasets", + subdirectory_name=hash, + name=self.name, + ) + # make the new module to be noticed by the import system + importlib.invalidate_caches() + builder_kwargs = { + "repo_id": self.name, + } + return DatasetModule(module_path, hash, builder_kwargs, importable_file_path=importable_file_path) + cache_dir = os.path.expanduser(str(self.cache_dir or config.HF_DATASETS_CACHE)) + namespace_and_dataset_name = self.name.split("/") + namespace_and_dataset_name[-1] = camelcase_to_snakecase(namespace_and_dataset_name[-1]) + cached_relative_path = "___".join(namespace_and_dataset_name) + cached_datasets_directory_path_root = os.path.join(cache_dir, cached_relative_path) + cached_directory_paths = [ + cached_directory_path + for cached_directory_path in glob.glob(os.path.join(cached_datasets_directory_path_root, "*", "*", "*")) + if os.path.isdir(cached_directory_path) + ] + if cached_directory_paths: + builder_kwargs = { + "repo_id": self.name, + "dataset_name": self.name.split("/")[-1], + } + warning_msg = f"Using the latest cached version of the dataset since {self.name} couldn't be found on the Hugging Face Hub" + if config.HF_HUB_OFFLINE: + warning_msg += " (offline mode is enabled)." + logger.warning(warning_msg) + return DatasetModule( + "datasets.packaged_modules.cache.cache", + "auto", + {**builder_kwargs, "version": "auto"}, + ) + raise FileNotFoundError(f"Dataset {self.name} is not cached in {self.cache_dir}") + + +def dataset_module_factory( + path: str, + revision: Optional[Union[str, Version]] = None, + download_config: Optional[DownloadConfig] = None, + download_mode: Optional[Union[DownloadMode, str]] = None, + dynamic_modules_path: Optional[str] = None, + data_dir: Optional[str] = None, + data_files: Optional[Union[dict, list, str, DataFilesDict]] = None, + cache_dir: Optional[str] = None, + trust_remote_code: Optional[bool] = None, + _require_default_config_name=True, + _require_custom_configs=False, + **download_kwargs, +) -> DatasetModule: + """ + Download/extract/cache a dataset module. + + Dataset codes are cached inside the dynamic modules cache to allow easy import (avoid ugly sys.path tweaks). + + Args: + + path (str): Path or name of the dataset. + Depending on ``path``, the dataset builder that is used comes from a generic dataset script (JSON, CSV, Parquet, text etc.) or from the dataset script (a python file) inside the dataset directory. + + For local datasets: + + - if ``path`` is a local directory (containing data files only) + -> load a generic dataset builder (csv, json, text etc.) based on the content of the directory + e.g. ``'./path/to/directory/with/my/csv/data'``. + - if ``path`` is a local dataset script or a directory containing a local dataset script (if the script has the same name as the directory): + -> load the dataset builder from the dataset script + e.g. ``'./dataset/squad'`` or ``'./dataset/squad/squad.py'``. + + For datasets on the Hugging Face Hub (list all available datasets with ``huggingface_hub.list_datasets()``) + + - if ``path`` is a dataset repository on the HF hub (containing data files only) + -> load a generic dataset builder (csv, text etc.) based on the content of the repository + e.g. ``'username/dataset_name'``, a dataset repository on the HF hub containing your data files. + - if ``path`` is a dataset repository on the HF hub with a dataset script (if the script has the same name as the directory) + -> load the dataset builder from the dataset script in the dataset repository + e.g. ``glue``, ``squad``, ``'username/dataset_name'``, a dataset repository on the HF hub containing a dataset script `'dataset_name.py'`. + + revision (:class:`~utils.Version` or :obj:`str`, optional): Version of the dataset script to load. + As datasets have their own git repository on the Datasets Hub, the default version "main" corresponds to their "main" branch. + You can specify a different version than the default "main" by using a commit SHA or a git tag of the dataset repository. + download_config (:class:`DownloadConfig`, optional): Specific download configuration parameters. + download_mode (:class:`DownloadMode` or :obj:`str`, default ``REUSE_DATASET_IF_EXISTS``): Download/generate mode. + dynamic_modules_path (Optional str, defaults to HF_MODULES_CACHE / "datasets_modules", i.e. ~/.cache/huggingface/modules/datasets_modules): + Optional path to the directory in which the dynamic modules are saved. It must have been initialized with :obj:`init_dynamic_modules`. + By default, the datasets are stored inside the `datasets_modules` module. + data_dir (:obj:`str`, optional): Directory with the data files. Used only if `data_files` is not specified, + in which case it's equal to pass `os.path.join(data_dir, "**")` as `data_files`. + data_files (:obj:`Union[Dict, List, str]`, optional): Defining the data_files of the dataset configuration. + cache_dir (`str`, *optional*): + Directory to read/write data. Defaults to `"~/.cache/huggingface/datasets"`. + + + trust_remote_code (`bool`, *optional*, defaults to `None`): + Whether or not to allow for datasets defined on the Hub using a dataset script. This option + should only be set to `True` for repositories you trust and in which you have read the code, as it will + execute code present on the Hub on your local machine. + + + + + + `trust_remote_code` defaults to `False` if not specified. + + + + **download_kwargs (additional keyword arguments): optional attributes for DownloadConfig() which will override + the attributes in download_config if supplied. + + Returns: + DatasetModule + """ + if download_config is None: + download_config = DownloadConfig(**download_kwargs) + download_mode = DownloadMode(download_mode or DownloadMode.REUSE_DATASET_IF_EXISTS) + download_config.extract_compressed_file = True + download_config.force_extract = True + download_config.force_download = download_mode == DownloadMode.FORCE_REDOWNLOAD + + filename = list(filter(lambda x: x, path.replace(os.sep, "/").split("/")))[-1] + if not filename.endswith(".py"): + filename = filename + ".py" + combined_path = os.path.join(path, filename) + + # We have several ways to get a dataset builder: + # + # - if path is the name of a packaged dataset module + # -> use the packaged module (json, csv, etc.) + # + # - if os.path.join(path, name) is a local python file + # -> use the module from the python file + # - if path is a local directory (but no python file) + # -> use a packaged module (csv, text etc.) based on content of the directory + # + # - if path has one "/" and is dataset repository on the HF hub with a python file + # -> the module from the python file in the dataset repository + # - if path has one "/" and is dataset repository on the HF hub without a python file + # -> use a packaged module (csv, text etc.) based on content of the repository + + # Try packaged + if path in _PACKAGED_DATASETS_MODULES: + return PackagedDatasetModuleFactory( + path, + data_dir=data_dir, + data_files=data_files, + download_config=download_config, + download_mode=download_mode, + ).get_module() + # Try locally + elif path.endswith(filename): + if os.path.isfile(path): + return LocalDatasetModuleFactoryWithScript( + path, + download_mode=download_mode, + dynamic_modules_path=dynamic_modules_path, + trust_remote_code=trust_remote_code, + ).get_module() + else: + raise FileNotFoundError(f"Couldn't find a dataset script at {relative_to_absolute_path(path)}") + elif os.path.isfile(combined_path): + return LocalDatasetModuleFactoryWithScript( + combined_path, + download_mode=download_mode, + dynamic_modules_path=dynamic_modules_path, + trust_remote_code=trust_remote_code, + ).get_module() + elif os.path.isdir(path): + return LocalDatasetModuleFactoryWithoutScript( + path, data_dir=data_dir, data_files=data_files, download_mode=download_mode + ).get_module() + # Try remotely + elif is_relative_path(path) and path.count("/") <= 1: + try: + # Get the Dataset Card + get the revision + check authentication all at in one call + # We fix the commit_hash in case there are new commits in the meantime + api = HfApi( + endpoint=config.HF_ENDPOINT, + token=download_config.token, + library_name="datasets", + library_version=__version__, + user_agent=get_datasets_user_agent(download_config.user_agent), + ) + try: + _raise_if_offline_mode_is_enabled() + dataset_readme_path = api.hf_hub_download( + repo_id=path, + filename=config.REPOCARD_FILENAME, + repo_type="dataset", + revision=revision, + proxies=download_config.proxies, + ) + commit_hash = os.path.basename(os.path.dirname(dataset_readme_path)) + except LocalEntryNotFoundError as e: + if isinstance( + e.__cause__, + ( + OfflineModeIsEnabled, + requests.exceptions.Timeout, + requests.exceptions.ConnectionError, + ), + ): + raise ConnectionError(f"Couldn't reach '{path}' on the Hub ({e.__class__.__name__})") from e + else: + raise + except EntryNotFoundError: + commit_hash = api.dataset_info( + path, + revision=revision, + timeout=100.0, + ).sha + except ( + OfflineModeIsEnabled, + requests.exceptions.Timeout, + requests.exceptions.ConnectionError, + ) as e: + raise ConnectionError(f"Couldn't reach '{path}' on the Hub ({e.__class__.__name__})") from e + except GatedRepoError as e: + message = f"Dataset '{path}' is a gated dataset on the Hub." + if e.response.status_code == 401: + message += " You must be authenticated to access it." + elif e.response.status_code == 403: + message += f" Visit the dataset page at https://huggingface.co/datasets/{path} to ask for access." + raise DatasetNotFoundError(message) from e + except RevisionNotFoundError as e: + raise DatasetNotFoundError( + f"Revision '{revision}' doesn't exist for dataset '{path}' on the Hub." + ) from e + except RepositoryNotFoundError as e: + raise DatasetNotFoundError(f"Dataset '{path}' doesn't exist on the Hub or cannot be accessed.") from e + try: + dataset_script_path = api.hf_hub_download( + repo_id=path, + filename=filename, + repo_type="dataset", + revision=commit_hash, + proxies=download_config.proxies, + ) + if _require_custom_configs or (revision and revision != "main"): + can_load_config_from_parquet_export = False + elif _require_default_config_name: + with open(dataset_script_path, encoding="utf-8") as f: + can_load_config_from_parquet_export = "DEFAULT_CONFIG_NAME" not in f.read() + else: + can_load_config_from_parquet_export = True + if config.USE_PARQUET_EXPORT and can_load_config_from_parquet_export: + # If the parquet export is ready (parquet files + info available for the current sha), we can use it instead + # This fails when the dataset has multiple configs and a default config and + # the user didn't specify a configuration name (_require_default_config_name=True). + try: + out = HubDatasetModuleFactoryWithParquetExport( + path, download_config=download_config, commit_hash=commit_hash + ).get_module() + logger.info("Loading the dataset from the Parquet export on Hugging Face.") + return out + except _dataset_viewer.DatasetViewerError: + pass + # Otherwise we must use the dataset script if the user trusts it + return HubDatasetModuleFactoryWithScript( + path, + commit_hash=commit_hash, + download_config=download_config, + download_mode=download_mode, + dynamic_modules_path=dynamic_modules_path, + trust_remote_code=trust_remote_code, + ).get_module() + except EntryNotFoundError: + # Use the infos from the parquet export except in some cases: + if data_dir or data_files or (revision and revision != "main"): + use_exported_dataset_infos = False + else: + use_exported_dataset_infos = True + return HubDatasetModuleFactoryWithoutScript( + path, + commit_hash=commit_hash, + data_dir=data_dir, + data_files=data_files, + download_config=download_config, + download_mode=download_mode, + use_exported_dataset_infos=use_exported_dataset_infos, + ).get_module() + except GatedRepoError as e: + message = f"Dataset '{path}' is a gated dataset on the Hub." + if e.response.status_code == 401: + message += " You must be authenticated to access it." + elif e.response.status_code == 403: + message += f" Visit the dataset page at https://huggingface.co/datasets/{path} to ask for access." + raise DatasetNotFoundError(message) from e + except RevisionNotFoundError as e: + raise DatasetNotFoundError( + f"Revision '{revision}' doesn't exist for dataset '{path}' on the Hub." + ) from e + except Exception as e1: + # All the attempts failed, before raising the error we should check if the module is already cached + try: + return CachedDatasetModuleFactory( + path, dynamic_modules_path=dynamic_modules_path, cache_dir=cache_dir + ).get_module() + except Exception: + # If it's not in the cache, then it doesn't exist. + if isinstance(e1, OfflineModeIsEnabled): + raise ConnectionError(f"Couldn't reach the Hugging Face Hub for dataset '{path}': {e1}") from None + if isinstance(e1, (DataFilesNotFoundError, DatasetNotFoundError, EmptyDatasetError)): + raise e1 from None + if isinstance(e1, FileNotFoundError): + if trust_remote_code: + raise FileNotFoundError( + f"Couldn't find a dataset script at {relative_to_absolute_path(combined_path)} or any data file in the same directory. " + f"Couldn't find '{path}' on the Hugging Face Hub either: {type(e1).__name__}: {e1}" + ) from None + else: + raise FileNotFoundError( + f"Couldn't find any data file at {relative_to_absolute_path(path)}. " + f"Couldn't find '{path}' on the Hugging Face Hub either: {type(e1).__name__}: {e1}" + ) from None + raise e1 from None + elif trust_remote_code: + raise FileNotFoundError( + f"Couldn't find a dataset script at {relative_to_absolute_path(combined_path)} or any data file in the same directory." + ) + else: + raise FileNotFoundError(f"Couldn't find any data file at {relative_to_absolute_path(path)}.") + + +def load_dataset_builder( + path: str, + name: Optional[str] = None, + data_dir: Optional[str] = None, + data_files: Optional[Union[str, Sequence[str], Mapping[str, Union[str, Sequence[str]]]]] = None, + cache_dir: Optional[str] = None, + features: Optional[Features] = None, + download_config: Optional[DownloadConfig] = None, + download_mode: Optional[Union[DownloadMode, str]] = None, + revision: Optional[Union[str, Version]] = None, + token: Optional[Union[bool, str]] = None, + storage_options: Optional[dict] = None, + trust_remote_code: Optional[bool] = None, + _require_default_config_name=True, + **config_kwargs, +) -> DatasetBuilder: + """Load a dataset builder which can be used to: + + - Inspect general information that is required to build a dataset (cache directory, config, dataset info, features, data files, etc.) + - Download and prepare the dataset as Arrow files in the cache + - Get a streaming dataset without downloading or caching anything + + You can find the list of datasets on the [Hub](https://huggingface.co/datasets) or with [`huggingface_hub.list_datasets`]. + + A dataset is a directory that contains some data files in generic formats (JSON, CSV, Parquet, etc.) and possibly + in a generic structure (Webdataset, ImageFolder, AudioFolder, VideoFolder, etc.) + + Args: + + path (`str`): + Path or name of the dataset. + + - if `path` is a dataset repository on the HF hub (list all available datasets with [`huggingface_hub.list_datasets`]) + -> load the dataset builder from supported files in the repository (csv, json, parquet, etc.) + e.g. `'username/dataset_name'`, a dataset repository on the HF hub containing the data files. + + - if `path` is a local directory + -> load the dataset builder from supported files in the directory (csv, json, parquet, etc.) + e.g. `'./path/to/directory/with/my/csv/data'`. + + - if `path` is the name of a dataset builder and `data_files` or `data_dir` is specified + (available builders are "json", "csv", "parquet", "arrow", "text", "xml", "webdataset", "imagefolder", "audiofolder", "videofolder") + -> load the dataset builder from the files in `data_files` or `data_dir` + e.g. `'parquet'`. + + It can also point to a local dataset script but this is not recommended. + name (`str`, *optional*): + Defining the name of the dataset configuration. + data_dir (`str`, *optional*): + Defining the `data_dir` of the dataset configuration. If specified for the generic builders (csv, text etc.) or the Hub datasets and `data_files` is `None`, + the behavior is equal to passing `os.path.join(data_dir, **)` as `data_files` to reference all the files in a directory. + data_files (`str` or `Sequence` or `Mapping`, *optional*): + Path(s) to source data file(s). + cache_dir (`str`, *optional*): + Directory to read/write data. Defaults to `"~/.cache/huggingface/datasets"`. + features ([`Features`], *optional*): + Set the features type to use for this dataset. + download_config ([`DownloadConfig`], *optional*): + Specific download configuration parameters. + download_mode ([`DownloadMode`] or `str`, defaults to `REUSE_DATASET_IF_EXISTS`): + Download/generate mode. + revision ([`Version`] or `str`, *optional*): + Version of the dataset script to load. + As datasets have their own git repository on the Datasets Hub, the default version "main" corresponds to their "main" branch. + You can specify a different version than the default "main" by using a commit SHA or a git tag of the dataset repository. + token (`str` or `bool`, *optional*): + Optional string or boolean to use as Bearer token for remote files on the Datasets Hub. + If `True`, or not specified, will get token from `"~/.huggingface"`. + storage_options (`dict`, *optional*, defaults to `None`): + **Experimental**. Key/value pairs to be passed on to the dataset file-system backend, if any. + + + trust_remote_code (`bool`, *optional*, defaults to `None`): + Whether or not to allow for datasets defined on the Hub using a dataset script. This option + should only be set to `True` for repositories you trust and in which you have read the code, as it will + execute code present on the Hub on your local machine. + + + + + + `trust_remote_code` defaults to `False` if not specified. + + + + **config_kwargs (additional keyword arguments): + Keyword arguments to be passed to the [`BuilderConfig`] + and used in the [`DatasetBuilder`]. + + Returns: + [`DatasetBuilder`] + + Example: + + ```py + >>> from datasets import load_dataset_builder + >>> ds_builder = load_dataset_builder('cornell-movie-review-data/rotten_tomatoes') + >>> ds_builder.info.features + {'label': ClassLabel(names=['neg', 'pos'], id=None), + 'text': Value(dtype='string', id=None)} + ``` + """ + download_mode = DownloadMode(download_mode or DownloadMode.REUSE_DATASET_IF_EXISTS) + if token is not None: + download_config = download_config.copy() if download_config else DownloadConfig() + download_config.token = token + if storage_options is not None: + download_config = download_config.copy() if download_config else DownloadConfig() + download_config.storage_options.update(storage_options) + dataset_module = dataset_module_factory( + path, + revision=revision, + download_config=download_config, + download_mode=download_mode, + data_dir=data_dir, + data_files=data_files, + cache_dir=cache_dir, + trust_remote_code=trust_remote_code, + _require_default_config_name=_require_default_config_name, + _require_custom_configs=bool(config_kwargs), + ) + # Get dataset builder class from the processing script + builder_kwargs = dataset_module.builder_kwargs + data_dir = builder_kwargs.pop("data_dir", data_dir) + data_files = builder_kwargs.pop("data_files", data_files) + config_name = builder_kwargs.pop( + "config_name", name or dataset_module.builder_configs_parameters.default_config_name + ) + dataset_name = builder_kwargs.pop("dataset_name", None) + info = dataset_module.dataset_infos.get(config_name) if dataset_module.dataset_infos else None + + if ( + path in _PACKAGED_DATASETS_MODULES + and data_files is None + and dataset_module.builder_configs_parameters.builder_configs[0].data_files is None + ): + error_msg = f"Please specify the data files or data directory to load for the {path} dataset builder." + example_extensions = [ + extension for extension in _EXTENSION_TO_MODULE if _EXTENSION_TO_MODULE[extension] == path + ] + if example_extensions: + error_msg += f'\nFor example `data_files={{"train": "path/to/data/train/*.{example_extensions[0]}"}}`' + raise ValueError(error_msg) + + builder_cls = get_dataset_builder_class(dataset_module, dataset_name=dataset_name) + # Instantiate the dataset builder + builder_instance: DatasetBuilder = builder_cls( + cache_dir=cache_dir, + dataset_name=dataset_name, + config_name=config_name, + data_dir=data_dir, + data_files=data_files, + hash=dataset_module.hash, + info=info, + features=features, + token=token, + storage_options=storage_options, + **builder_kwargs, + **config_kwargs, + ) + builder_instance._use_legacy_cache_dir_if_possible(dataset_module) + + return builder_instance + + +def load_dataset( + path: str, + name: Optional[str] = None, + data_dir: Optional[str] = None, + data_files: Optional[Union[str, Sequence[str], Mapping[str, Union[str, Sequence[str]]]]] = None, + split: Optional[Union[str, Split]] = None, + cache_dir: Optional[str] = None, + features: Optional[Features] = None, + download_config: Optional[DownloadConfig] = None, + download_mode: Optional[Union[DownloadMode, str]] = None, + verification_mode: Optional[Union[VerificationMode, str]] = None, + keep_in_memory: Optional[bool] = None, + save_infos: bool = False, + revision: Optional[Union[str, Version]] = None, + token: Optional[Union[bool, str]] = None, + streaming: bool = False, + num_proc: Optional[int] = None, + storage_options: Optional[dict] = None, + trust_remote_code: Optional[bool] = None, + **config_kwargs, +) -> Union[DatasetDict, Dataset, IterableDatasetDict, IterableDataset]: + """Load a dataset from the Hugging Face Hub, or a local dataset. + + You can find the list of datasets on the [Hub](https://huggingface.co/datasets) or with [`huggingface_hub.list_datasets`]. + + A dataset is a directory that contains some data files in generic formats (JSON, CSV, Parquet, etc.) and possibly + in a generic structure (Webdataset, ImageFolder, AudioFolder, VideoFolder, etc.) + + This function does the following under the hood: + + 1. Load a dataset builder: + + * Find the most common data format in the dataset and pick its associated builder (JSON, CSV, Parquet, Webdataset, ImageFolder, AudioFolder, etc.) + * Find which file goes into which split (e.g. train/test) based on file and directory names or on the YAML configuration + * It is also possible to specify `data_files` manually, and which dataset builder to use (e.g. "parquet"). + + 2. Run the dataset builder: + + In the general case: + + * Download the data files from the dataset if they are not already available locally or cached. + * Process and cache the dataset in typed Arrow tables for caching. + + Arrow table are arbitrarily long, typed tables which can store nested objects and be mapped to numpy/pandas/python generic types. + They can be directly accessed from disk, loaded in RAM or even streamed over the web. + + In the streaming case: + + * Don't download or cache anything. Instead, the dataset is lazily loaded and will be streamed on-the-fly when iterating on it. + + 3. Return a dataset built from the requested splits in `split` (default: all). + + It can also use a custom dataset builder if the dataset contains a dataset script, but this feature is mostly for backward compatibility. + In this case the dataset script file must be named after the dataset repository or directory and end with ".py". + + Args: + + path (`str`): + Path or name of the dataset. + + - if `path` is a dataset repository on the HF hub (list all available datasets with [`huggingface_hub.list_datasets`]) + -> load the dataset from supported files in the repository (csv, json, parquet, etc.) + e.g. `'username/dataset_name'`, a dataset repository on the HF hub containing the data files. + + - if `path` is a local directory + -> load the dataset from supported files in the directory (csv, json, parquet, etc.) + e.g. `'./path/to/directory/with/my/csv/data'`. + + - if `path` is the name of a dataset builder and `data_files` or `data_dir` is specified + (available builders are "json", "csv", "parquet", "arrow", "text", "xml", "webdataset", "imagefolder", "audiofolder", "videofolder") + -> load the dataset from the files in `data_files` or `data_dir` + e.g. `'parquet'`. + + It can also point to a local dataset script but this is not recommended. + name (`str`, *optional*): + Defining the name of the dataset configuration. + data_dir (`str`, *optional*): + Defining the `data_dir` of the dataset configuration. If specified for the generic builders (csv, text etc.) or the Hub datasets and `data_files` is `None`, + the behavior is equal to passing `os.path.join(data_dir, **)` as `data_files` to reference all the files in a directory. + data_files (`str` or `Sequence` or `Mapping`, *optional*): + Path(s) to source data file(s). + split (`Split` or `str`): + Which split of the data to load. + If `None`, will return a `dict` with all splits (typically `datasets.Split.TRAIN` and `datasets.Split.TEST`). + If given, will return a single Dataset. + Splits can be combined and specified like in tensorflow-datasets. + cache_dir (`str`, *optional*): + Directory to read/write data. Defaults to `"~/.cache/huggingface/datasets"`. + features (`Features`, *optional*): + Set the features type to use for this dataset. + download_config ([`DownloadConfig`], *optional*): + Specific download configuration parameters. + download_mode ([`DownloadMode`] or `str`, defaults to `REUSE_DATASET_IF_EXISTS`): + Download/generate mode. + verification_mode ([`VerificationMode`] or `str`, defaults to `BASIC_CHECKS`): + Verification mode determining the checks to run on the downloaded/processed dataset information (checksums/size/splits/...). + + + keep_in_memory (`bool`, defaults to `None`): + Whether to copy the dataset in-memory. If `None`, the dataset + will not be copied in-memory unless explicitly enabled by setting `datasets.config.IN_MEMORY_MAX_SIZE` to + nonzero. See more details in the [improve performance](../cache#improve-performance) section. + save_infos (`bool`, defaults to `False`): + Save the dataset information (checksums/size/splits/...). + revision ([`Version`] or `str`, *optional*): + Version of the dataset script to load. + As datasets have their own git repository on the Datasets Hub, the default version "main" corresponds to their "main" branch. + You can specify a different version than the default "main" by using a commit SHA or a git tag of the dataset repository. + token (`str` or `bool`, *optional*): + Optional string or boolean to use as Bearer token for remote files on the Datasets Hub. + If `True`, or not specified, will get token from `"~/.huggingface"`. + streaming (`bool`, defaults to `False`): + If set to `True`, don't download the data files. Instead, it streams the data progressively while + iterating on the dataset. An [`IterableDataset`] or [`IterableDatasetDict`] is returned instead in this case. + + Note that streaming works for datasets that use data formats that support being iterated over like txt, csv, jsonl for example. + Json files may be downloaded completely. Also streaming from remote zip or gzip files is supported but other compressed formats + like rar and xz are not yet supported. The tgz format doesn't allow streaming. + num_proc (`int`, *optional*, defaults to `None`): + Number of processes when downloading and generating the dataset locally. + Multiprocessing is disabled by default. + + + storage_options (`dict`, *optional*, defaults to `None`): + **Experimental**. Key/value pairs to be passed on to the dataset file-system backend, if any. + + + trust_remote_code (`bool`, *optional*, defaults to `None`): + Whether or not to allow for datasets defined on the Hub using a dataset script. This option + should only be set to `True` for repositories you trust and in which you have read the code, as it will + execute code present on the Hub on your local machine. + + + + + + `trust_remote_code` defaults to `False` if not specified. + + + + **config_kwargs (additional keyword arguments): + Keyword arguments to be passed to the `BuilderConfig` + and used in the [`DatasetBuilder`]. + + Returns: + [`Dataset`] or [`DatasetDict`]: + - if `split` is not `None`: the dataset requested, + - if `split` is `None`, a [`~datasets.DatasetDict`] with each split. + + or [`IterableDataset`] or [`IterableDatasetDict`]: if `streaming=True` + + - if `split` is not `None`, the dataset is requested + - if `split` is `None`, a [`~datasets.streaming.IterableDatasetDict`] with each split. + + Example: + + Load a dataset from the Hugging Face Hub: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset('cornell-movie-review-data/rotten_tomatoes', split='train') + + # Load a subset or dataset configuration (here 'sst2') + >>> from datasets import load_dataset + >>> ds = load_dataset('nyu-mll/glue', 'sst2', split='train') + + # Manual mapping of data files to splits + >>> data_files = {'train': 'train.csv', 'test': 'test.csv'} + >>> ds = load_dataset('namespace/your_dataset_name', data_files=data_files) + + # Manual selection of a directory to load + >>> ds = load_dataset('namespace/your_dataset_name', data_dir='folder_name') + ``` + + Load a local dataset: + + ```py + # Load a CSV file + >>> from datasets import load_dataset + >>> ds = load_dataset('csv', data_files='path/to/local/my_dataset.csv') + + # Load a JSON file + >>> from datasets import load_dataset + >>> ds = load_dataset('json', data_files='path/to/local/my_dataset.json') + + # Load from a local loading script (not recommended) + >>> from datasets import load_dataset + >>> ds = load_dataset('path/to/local/loading_script/loading_script.py', split='train') + ``` + + Load an [`~datasets.IterableDataset`]: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset('cornell-movie-review-data/rotten_tomatoes', split='train', streaming=True) + ``` + + Load an image dataset with the `ImageFolder` dataset builder: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset('imagefolder', data_dir='/path/to/images', split='train') + ``` + """ + if data_files is not None and not data_files: + raise ValueError(f"Empty 'data_files': '{data_files}'. It should be either non-empty or None (default).") + if Path(path, config.DATASET_STATE_JSON_FILENAME).exists(): + raise ValueError( + "You are trying to load a dataset that was saved using `save_to_disk`. " + "Please use `load_from_disk` instead." + ) + + if streaming and num_proc is not None: + raise NotImplementedError( + "Loading a streaming dataset in parallel with `num_proc` is not implemented. " + "To parallelize streaming, you can wrap the dataset with a PyTorch DataLoader using `num_workers` > 1 instead." + ) + + download_mode = DownloadMode(download_mode or DownloadMode.REUSE_DATASET_IF_EXISTS) + verification_mode = VerificationMode( + (verification_mode or VerificationMode.BASIC_CHECKS) if not save_infos else VerificationMode.ALL_CHECKS + ) + + # Create a dataset builder + builder_instance = load_dataset_builder( + path=path, + name=name, + data_dir=data_dir, + data_files=data_files, + cache_dir=cache_dir, + features=features, + download_config=download_config, + download_mode=download_mode, + revision=revision, + token=token, + storage_options=storage_options, + trust_remote_code=trust_remote_code, + _require_default_config_name=name is None, + **config_kwargs, + ) + + # Return iterable dataset in case of streaming + if streaming: + return builder_instance.as_streaming_dataset(split=split) + + # Download and prepare data + builder_instance.download_and_prepare( + download_config=download_config, + download_mode=download_mode, + verification_mode=verification_mode, + num_proc=num_proc, + storage_options=storage_options, + ) + + # Build dataset for splits + keep_in_memory = ( + keep_in_memory if keep_in_memory is not None else is_small_dataset(builder_instance.info.dataset_size) + ) + ds = builder_instance.as_dataset(split=split, verification_mode=verification_mode, in_memory=keep_in_memory) + if save_infos: + builder_instance._save_infos() + + return ds + + +def load_from_disk( + dataset_path: PathLike, keep_in_memory: Optional[bool] = None, storage_options: Optional[dict] = None +) -> Union[Dataset, DatasetDict]: + """ + Loads a dataset that was previously saved using [`~Dataset.save_to_disk`] from a dataset directory, or + from a filesystem using any implementation of `fsspec.spec.AbstractFileSystem`. + + Args: + dataset_path (`path-like`): + Path (e.g. `"dataset/train"`) or remote URI (e.g. `"s3://my-bucket/dataset/train"`) + of the [`Dataset`] or [`DatasetDict`] directory where the dataset/dataset-dict will be + loaded from. + keep_in_memory (`bool`, defaults to `None`): + Whether to copy the dataset in-memory. If `None`, the dataset + will not be copied in-memory unless explicitly enabled by setting `datasets.config.IN_MEMORY_MAX_SIZE` to + nonzero. See more details in the [improve performance](../cache#improve-performance) section. + + storage_options (`dict`, *optional*): + Key/value pairs to be passed on to the file-system backend, if any. + + + + Returns: + [`Dataset`] or [`DatasetDict`]: + - If `dataset_path` is a path of a dataset directory: the dataset requested. + - If `dataset_path` is a path of a dataset dict directory, a [`DatasetDict`] with each split. + + Example: + + ```py + >>> from datasets import load_from_disk + >>> ds = load_from_disk('path/to/dataset/directory') + ``` + """ + fs: fsspec.AbstractFileSystem + fs, *_ = url_to_fs(dataset_path, **(storage_options or {})) + if not fs.exists(dataset_path): + raise FileNotFoundError(f"Directory {dataset_path} not found") + if fs.isfile(posixpath.join(dataset_path, config.DATASET_INFO_FILENAME)) and fs.isfile( + posixpath.join(dataset_path, config.DATASET_STATE_JSON_FILENAME) + ): + return Dataset.load_from_disk(dataset_path, keep_in_memory=keep_in_memory, storage_options=storage_options) + elif fs.isfile(posixpath.join(dataset_path, config.DATASETDICT_JSON_FILENAME)): + return DatasetDict.load_from_disk(dataset_path, keep_in_memory=keep_in_memory, storage_options=storage_options) + else: + raise FileNotFoundError( + f"Directory {dataset_path} is neither a `Dataset` directory nor a `DatasetDict` directory." + ) diff --git a/lib/python3.12/site-packages/datasets/naming.py b/lib/python3.12/site-packages/datasets/naming.py new file mode 100644 index 0000000000000000000000000000000000000000..65e7ede10dcde8701823223ae98e7971f705f945 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/naming.py @@ -0,0 +1,84 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +"""Utilities for file names.""" + +import itertools +import os +import re + + +_uppercase_uppercase_re = re.compile(r"([A-Z]+)([A-Z][a-z])") +_lowercase_uppercase_re = re.compile(r"([a-z\d])([A-Z])") + +_single_underscore_re = re.compile(r"(?:/\|?*" + + +def camelcase_to_snakecase(name): + """Convert camel-case string to snake-case.""" + name = _uppercase_uppercase_re.sub(r"\1_\2", name) + name = _lowercase_uppercase_re.sub(r"\1_\2", name) + return name.lower() + + +def snakecase_to_camelcase(name): + """Convert snake-case string to camel-case string.""" + name = _single_underscore_re.split(name) + name = [_multiple_underscores_re.split(n) for n in name] + return "".join(n.capitalize() for n in itertools.chain.from_iterable(name) if n != "") + + +def filename_prefix_for_name(name): + if os.path.basename(name) != name: + raise ValueError(f"Should be a dataset name, not a path: {name}") + return camelcase_to_snakecase(name) + + +def filename_prefix_for_split(name, split): + if os.path.basename(name) != name: + raise ValueError(f"Should be a dataset name, not a path: {name}") + if not re.match(_split_re, split): + raise ValueError(f"Split name should match '{_split_re}'' but got '{split}'.") + return f"{filename_prefix_for_name(name)}-{split}" + + +def filepattern_for_dataset_split(dataset_name, split, data_dir, filetype_suffix=None): + prefix = filename_prefix_for_split(dataset_name, split) + if filetype_suffix: + prefix += f".{filetype_suffix}" + filepath = os.path.join(data_dir, prefix) + return f"{filepath}*" + + +def filenames_for_dataset_split(path, dataset_name, split, filetype_suffix=None, shard_lengths=None): + prefix = filename_prefix_for_split(dataset_name, split) + prefix = os.path.join(path, prefix) + + if shard_lengths: + num_shards = len(shard_lengths) + filenames = [f"{prefix}-{shard_id:05d}-of-{num_shards:05d}" for shard_id in range(num_shards)] + if filetype_suffix: + filenames = [filename + f".{filetype_suffix}" for filename in filenames] + return filenames + else: + filename = prefix + if filetype_suffix: + filename += f".{filetype_suffix}" + return [filename] diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/__init__.py b/lib/python3.12/site-packages/datasets/packaged_modules/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f61c6ddd3de5795ee592abc30f4cda899e913f22 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/packaged_modules/__init__.py @@ -0,0 +1,104 @@ +import inspect +import re +from typing import Dict, List, Tuple + +from huggingface_hub.utils import insecure_hashlib + +from .arrow import arrow +from .audiofolder import audiofolder +from .cache import cache +from .csv import csv +from .imagefolder import imagefolder +from .json import json +from .pandas import pandas +from .parquet import parquet +from .pdffolder import pdffolder +from .sql import sql +from .text import text +from .videofolder import videofolder +from .webdataset import webdataset +from .xml import xml + + +def _hash_python_lines(lines: list[str]) -> str: + filtered_lines = [] + for line in lines: + line = re.sub(r"#.*", "", line) # remove comments + if line: + filtered_lines.append(line) + full_str = "\n".join(filtered_lines) + + # Make a hash from all this code + full_bytes = full_str.encode("utf-8") + return insecure_hashlib.sha256(full_bytes).hexdigest() + + +# get importable module names and hash for caching +_PACKAGED_DATASETS_MODULES = { + "csv": (csv.__name__, _hash_python_lines(inspect.getsource(csv).splitlines())), + "json": (json.__name__, _hash_python_lines(inspect.getsource(json).splitlines())), + "pandas": (pandas.__name__, _hash_python_lines(inspect.getsource(pandas).splitlines())), + "parquet": (parquet.__name__, _hash_python_lines(inspect.getsource(parquet).splitlines())), + "arrow": (arrow.__name__, _hash_python_lines(inspect.getsource(arrow).splitlines())), + "text": (text.__name__, _hash_python_lines(inspect.getsource(text).splitlines())), + "imagefolder": (imagefolder.__name__, _hash_python_lines(inspect.getsource(imagefolder).splitlines())), + "audiofolder": (audiofolder.__name__, _hash_python_lines(inspect.getsource(audiofolder).splitlines())), + "videofolder": (videofolder.__name__, _hash_python_lines(inspect.getsource(videofolder).splitlines())), + "pdffolder": (pdffolder.__name__, _hash_python_lines(inspect.getsource(pdffolder).splitlines())), + "webdataset": (webdataset.__name__, _hash_python_lines(inspect.getsource(webdataset).splitlines())), + "xml": (xml.__name__, _hash_python_lines(inspect.getsource(xml).splitlines())), +} + +# get importable module names and hash for caching +_PACKAGED_DATASETS_MODULES_2_15_HASHES = { + "csv": "eea64c71ca8b46dd3f537ed218fc9bf495d5707789152eb2764f5c78fa66d59d", + "json": "8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96", + "pandas": "3ac4ffc4563c796122ef66899b9485a3f1a977553e2d2a8a318c72b8cc6f2202", + "parquet": "ca31c69184d9832faed373922c2acccec0b13a0bb5bbbe19371385c3ff26f1d1", + "arrow": "74f69db2c14c2860059d39860b1f400a03d11bf7fb5a8258ca38c501c878c137", + "text": "c4a140d10f020282918b5dd1b8a49f0104729c6177f60a6b49ec2a365ec69f34", + "imagefolder": "7b7ce5247a942be131d49ad4f3de5866083399a0f250901bd8dc202f8c5f7ce5", + "audiofolder": "d3c1655c66c8f72e4efb5c79e952975fa6e2ce538473a6890241ddbddee9071c", +} + +# Used to infer the module to use based on the data files extensions +_EXTENSION_TO_MODULE: dict[str, tuple[str, dict]] = { + ".csv": ("csv", {}), + ".tsv": ("csv", {"sep": "\t"}), + ".json": ("json", {}), + ".jsonl": ("json", {}), + # ndjson is no longer maintained (see: https://github.com/ndjson/ndjson-spec/issues/35#issuecomment-1285673417) + ".ndjson": ("json", {}), + ".parquet": ("parquet", {}), + ".geoparquet": ("parquet", {}), + ".gpq": ("parquet", {}), + ".arrow": ("arrow", {}), + ".txt": ("text", {}), + ".tar": ("webdataset", {}), + ".xml": ("xml", {}), +} +_EXTENSION_TO_MODULE.update({ext: ("imagefolder", {}) for ext in imagefolder.ImageFolder.EXTENSIONS}) +_EXTENSION_TO_MODULE.update({ext.upper(): ("imagefolder", {}) for ext in imagefolder.ImageFolder.EXTENSIONS}) +_EXTENSION_TO_MODULE.update({ext: ("audiofolder", {}) for ext in audiofolder.AudioFolder.EXTENSIONS}) +_EXTENSION_TO_MODULE.update({ext.upper(): ("audiofolder", {}) for ext in audiofolder.AudioFolder.EXTENSIONS}) +_EXTENSION_TO_MODULE.update({ext: ("videofolder", {}) for ext in videofolder.VideoFolder.EXTENSIONS}) +_EXTENSION_TO_MODULE.update({ext.upper(): ("videofolder", {}) for ext in videofolder.VideoFolder.EXTENSIONS}) +_EXTENSION_TO_MODULE.update({ext: ("pdffolder", {}) for ext in pdffolder.PdfFolder.EXTENSIONS}) +_EXTENSION_TO_MODULE.update({ext.upper(): ("pdffolder", {}) for ext in pdffolder.PdfFolder.EXTENSIONS}) + +# Used to filter data files based on extensions given a module name +_MODULE_TO_EXTENSIONS: dict[str, list[str]] = {} +for _ext, (_module, _) in _EXTENSION_TO_MODULE.items(): + _MODULE_TO_EXTENSIONS.setdefault(_module, []).append(_ext) + +for _module in _MODULE_TO_EXTENSIONS: + _MODULE_TO_EXTENSIONS[_module].append(".zip") + +# Used to filter data files based on file names +_MODULE_TO_METADATA_FILE_NAMES: Dict[str, List[str]] = {} +for _module in _MODULE_TO_EXTENSIONS: + _MODULE_TO_METADATA_FILE_NAMES[_module] = [] +_MODULE_TO_METADATA_FILE_NAMES["imagefolder"] = imagefolder.ImageFolder.METADATA_FILENAMES +_MODULE_TO_METADATA_FILE_NAMES["audiofolder"] = imagefolder.ImageFolder.METADATA_FILENAMES +_MODULE_TO_METADATA_FILE_NAMES["videofolder"] = imagefolder.ImageFolder.METADATA_FILENAMES +_MODULE_TO_METADATA_FILE_NAMES["pdffolder"] = imagefolder.ImageFolder.METADATA_FILENAMES diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/csv/__init__.py b/lib/python3.12/site-packages/datasets/packaged_modules/csv/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/csv/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/datasets/packaged_modules/csv/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..145b940d8579a466c5dc49dfd3e386b3289a285f Binary files /dev/null and b/lib/python3.12/site-packages/datasets/packaged_modules/csv/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/csv/__pycache__/csv.cpython-312.pyc b/lib/python3.12/site-packages/datasets/packaged_modules/csv/__pycache__/csv.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..dc913aadb25e02a091780b43e9d94a8f8a712497 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/packaged_modules/csv/__pycache__/csv.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/csv/csv.py b/lib/python3.12/site-packages/datasets/packaged_modules/csv/csv.py new file mode 100644 index 0000000000000000000000000000000000000000..2ae95ff51428b826b84b32a7eaf2825cd7593960 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/packaged_modules/csv/csv.py @@ -0,0 +1,198 @@ +import itertools +from dataclasses import dataclass +from typing import Any, Callable, Optional, Union + +import pandas as pd +import pyarrow as pa + +import datasets +import datasets.config +from datasets.features.features import require_storage_cast +from datasets.table import table_cast +from datasets.utils.py_utils import Literal + + +logger = datasets.utils.logging.get_logger(__name__) + +_PANDAS_READ_CSV_NO_DEFAULT_PARAMETERS = ["names", "prefix"] +_PANDAS_READ_CSV_DEPRECATED_PARAMETERS = ["warn_bad_lines", "error_bad_lines", "mangle_dupe_cols"] +_PANDAS_READ_CSV_NEW_1_3_0_PARAMETERS = ["encoding_errors", "on_bad_lines"] +_PANDAS_READ_CSV_NEW_2_0_0_PARAMETERS = ["date_format"] +_PANDAS_READ_CSV_DEPRECATED_2_2_0_PARAMETERS = ["verbose"] + + +@dataclass +class CsvConfig(datasets.BuilderConfig): + """BuilderConfig for CSV.""" + + sep: str = "," + delimiter: Optional[str] = None + header: Optional[Union[int, list[int], str]] = "infer" + names: Optional[list[str]] = None + column_names: Optional[list[str]] = None + index_col: Optional[Union[int, str, list[int], list[str]]] = None + usecols: Optional[Union[list[int], list[str]]] = None + prefix: Optional[str] = None + mangle_dupe_cols: bool = True + engine: Optional[Literal["c", "python", "pyarrow"]] = None + converters: dict[Union[int, str], Callable[[Any], Any]] = None + true_values: Optional[list] = None + false_values: Optional[list] = None + skipinitialspace: bool = False + skiprows: Optional[Union[int, list[int]]] = None + nrows: Optional[int] = None + na_values: Optional[Union[str, list[str]]] = None + keep_default_na: bool = True + na_filter: bool = True + verbose: bool = False + skip_blank_lines: bool = True + thousands: Optional[str] = None + decimal: str = "." + lineterminator: Optional[str] = None + quotechar: str = '"' + quoting: int = 0 + escapechar: Optional[str] = None + comment: Optional[str] = None + encoding: Optional[str] = None + dialect: Optional[str] = None + error_bad_lines: bool = True + warn_bad_lines: bool = True + skipfooter: int = 0 + doublequote: bool = True + memory_map: bool = False + float_precision: Optional[str] = None + chunksize: int = 10_000 + features: Optional[datasets.Features] = None + encoding_errors: Optional[str] = "strict" + on_bad_lines: Literal["error", "warn", "skip"] = "error" + date_format: Optional[str] = None + + def __post_init__(self): + super().__post_init__() + if self.delimiter is not None: + self.sep = self.delimiter + if self.column_names is not None: + self.names = self.column_names + + @property + def pd_read_csv_kwargs(self): + pd_read_csv_kwargs = { + "sep": self.sep, + "header": self.header, + "names": self.names, + "index_col": self.index_col, + "usecols": self.usecols, + "prefix": self.prefix, + "mangle_dupe_cols": self.mangle_dupe_cols, + "engine": self.engine, + "converters": self.converters, + "true_values": self.true_values, + "false_values": self.false_values, + "skipinitialspace": self.skipinitialspace, + "skiprows": self.skiprows, + "nrows": self.nrows, + "na_values": self.na_values, + "keep_default_na": self.keep_default_na, + "na_filter": self.na_filter, + "verbose": self.verbose, + "skip_blank_lines": self.skip_blank_lines, + "thousands": self.thousands, + "decimal": self.decimal, + "lineterminator": self.lineterminator, + "quotechar": self.quotechar, + "quoting": self.quoting, + "escapechar": self.escapechar, + "comment": self.comment, + "encoding": self.encoding, + "dialect": self.dialect, + "error_bad_lines": self.error_bad_lines, + "warn_bad_lines": self.warn_bad_lines, + "skipfooter": self.skipfooter, + "doublequote": self.doublequote, + "memory_map": self.memory_map, + "float_precision": self.float_precision, + "chunksize": self.chunksize, + "encoding_errors": self.encoding_errors, + "on_bad_lines": self.on_bad_lines, + "date_format": self.date_format, + } + + # some kwargs must not be passed if they don't have a default value + # some others are deprecated and we can also not pass them if they are the default value + for pd_read_csv_parameter in _PANDAS_READ_CSV_NO_DEFAULT_PARAMETERS + _PANDAS_READ_CSV_DEPRECATED_PARAMETERS: + if pd_read_csv_kwargs[pd_read_csv_parameter] == getattr(CsvConfig(), pd_read_csv_parameter): + del pd_read_csv_kwargs[pd_read_csv_parameter] + + # Remove 1.3 new arguments + if not (datasets.config.PANDAS_VERSION.major >= 1 and datasets.config.PANDAS_VERSION.minor >= 3): + for pd_read_csv_parameter in _PANDAS_READ_CSV_NEW_1_3_0_PARAMETERS: + del pd_read_csv_kwargs[pd_read_csv_parameter] + + # Remove 2.0 new arguments + if not (datasets.config.PANDAS_VERSION.major >= 2): + for pd_read_csv_parameter in _PANDAS_READ_CSV_NEW_2_0_0_PARAMETERS: + del pd_read_csv_kwargs[pd_read_csv_parameter] + + # Remove 2.2 deprecated arguments + if datasets.config.PANDAS_VERSION.release >= (2, 2): + for pd_read_csv_parameter in _PANDAS_READ_CSV_DEPRECATED_2_2_0_PARAMETERS: + if pd_read_csv_kwargs[pd_read_csv_parameter] == getattr(CsvConfig(), pd_read_csv_parameter): + del pd_read_csv_kwargs[pd_read_csv_parameter] + + return pd_read_csv_kwargs + + +class Csv(datasets.ArrowBasedBuilder): + BUILDER_CONFIG_CLASS = CsvConfig + + def _info(self): + return datasets.DatasetInfo(features=self.config.features) + + def _split_generators(self, dl_manager): + """We handle string, list and dicts in datafiles""" + if not self.config.data_files: + raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}") + dl_manager.download_config.extract_on_the_fly = True + data_files = dl_manager.download_and_extract(self.config.data_files) + splits = [] + for split_name, files in data_files.items(): + if isinstance(files, str): + files = [files] + files = [dl_manager.iter_files(file) for file in files] + splits.append(datasets.SplitGenerator(name=split_name, gen_kwargs={"files": files})) + return splits + + def _cast_table(self, pa_table: pa.Table) -> pa.Table: + if self.config.features is not None: + schema = self.config.features.arrow_schema + if all(not require_storage_cast(feature) for feature in self.config.features.values()): + # cheaper cast + pa_table = pa.Table.from_arrays([pa_table[field.name] for field in schema], schema=schema) + else: + # more expensive cast; allows str <-> int/float or str to Audio for example + pa_table = table_cast(pa_table, schema) + return pa_table + + def _generate_tables(self, files): + schema = self.config.features.arrow_schema if self.config.features else None + # dtype allows reading an int column as str + dtype = ( + { + name: dtype.to_pandas_dtype() if not require_storage_cast(feature) else object + for name, dtype, feature in zip(schema.names, schema.types, self.config.features.values()) + } + if schema is not None + else None + ) + for file_idx, file in enumerate(itertools.chain.from_iterable(files)): + csv_file_reader = pd.read_csv(file, iterator=True, dtype=dtype, **self.config.pd_read_csv_kwargs) + try: + for batch_idx, df in enumerate(csv_file_reader): + pa_table = pa.Table.from_pandas(df) + # Uncomment for debugging (will print the Arrow table size and elements) + # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}") + # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows))) + yield (file_idx, batch_idx), self._cast_table(pa_table) + except ValueError as e: + logger.error(f"Failed to read file '{file}' with error {type(e)}: {e}") + raise diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/folder_based_builder/__init__.py b/lib/python3.12/site-packages/datasets/packaged_modules/folder_based_builder/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/folder_based_builder/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/datasets/packaged_modules/folder_based_builder/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..152e10a16e7808242b2875f124b22de9e3795628 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/packaged_modules/folder_based_builder/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/folder_based_builder/__pycache__/folder_based_builder.cpython-312.pyc b/lib/python3.12/site-packages/datasets/packaged_modules/folder_based_builder/__pycache__/folder_based_builder.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3e6d859bbdd1f10f36e882741563082e1806f303 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/packaged_modules/folder_based_builder/__pycache__/folder_based_builder.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/folder_based_builder/folder_based_builder.py b/lib/python3.12/site-packages/datasets/packaged_modules/folder_based_builder/folder_based_builder.py new file mode 100644 index 0000000000000000000000000000000000000000..44f604e9ee56083e752ada4ffa07337b5c8d4d5e --- /dev/null +++ b/lib/python3.12/site-packages/datasets/packaged_modules/folder_based_builder/folder_based_builder.py @@ -0,0 +1,417 @@ +import collections +import io +import itertools +import os +from dataclasses import dataclass +from typing import Any, Callable, Iterator, Optional, Union + +import pandas as pd +import pyarrow as pa +import pyarrow.dataset as ds +import pyarrow.json as paj +import pyarrow.parquet as pq + +import datasets +from datasets import config +from datasets.features.features import FeatureType, _visit, _visit_with_path, _VisitPath, require_storage_cast +from datasets.utils.file_utils import readline + + +logger = datasets.utils.logging.get_logger(__name__) + + +def count_path_segments(path): + return path.replace("\\", "/").count("/") + + +@dataclass +class FolderBasedBuilderConfig(datasets.BuilderConfig): + """BuilderConfig for AutoFolder.""" + + features: Optional[datasets.Features] = None + drop_labels: bool = None + drop_metadata: bool = None + filters: Optional[Union[ds.Expression, list[tuple], list[list[tuple]]]] = None + + def __post_init__(self): + super().__post_init__() + + +class FolderBasedBuilder(datasets.GeneratorBasedBuilder): + """ + Base class for generic data loaders for vision and image data. + + + Abstract class attributes to be overridden by a child class: + BASE_FEATURE: feature object to decode data (i.e. datasets.Image, datasets.Audio, ...) + BASE_COLUMN_NAME: string key name of a base feature (i.e. "image", "audio", ...) + BUILDER_CONFIG_CLASS: builder config inherited from `folder_based_builder.FolderBasedBuilderConfig` + EXTENSIONS: list of allowed extensions (only files with these extensions and METADATA_FILENAME files + will be included in a dataset) + """ + + BASE_FEATURE: type[FeatureType] + BASE_COLUMN_NAME: str + BUILDER_CONFIG_CLASS: FolderBasedBuilderConfig + EXTENSIONS: list[str] + + METADATA_FILENAMES: list[str] = ["metadata.csv", "metadata.jsonl", "metadata.parquet"] + + def _info(self): + return datasets.DatasetInfo(features=self.config.features) + + def _split_generators(self, dl_manager): + if not self.config.data_files: + raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}") + dl_manager.download_config.extract_on_the_fly = True + # Do an early pass if: + # * `drop_labels` is None (default) or False, to infer the class labels + # * `drop_metadata` is None (default) or False, to find the metadata files + do_analyze = not self.config.drop_labels or not self.config.drop_metadata + labels, path_depths = set(), set() + metadata_files = collections.defaultdict(set) + + def analyze(files_or_archives, downloaded_files_or_dirs, split): + if len(downloaded_files_or_dirs) == 0: + return + # The files are separated from the archives at this point, so check the first sample + # to see if it's a file or a directory and iterate accordingly + if os.path.isfile(downloaded_files_or_dirs[0]): + original_files, downloaded_files = files_or_archives, downloaded_files_or_dirs + for original_file, downloaded_file in zip(original_files, downloaded_files): + original_file, downloaded_file = str(original_file), str(downloaded_file) + _, original_file_ext = os.path.splitext(original_file) + if original_file_ext.lower() in self.EXTENSIONS: + if not self.config.drop_labels: + labels.add(os.path.basename(os.path.dirname(original_file))) + path_depths.add(count_path_segments(original_file)) + elif os.path.basename(original_file) in self.METADATA_FILENAMES: + metadata_files[split].add((original_file, downloaded_file)) + else: + original_file_name = os.path.basename(original_file) + logger.debug( + f"The file '{original_file_name}' was ignored: it is not a {self.BASE_COLUMN_NAME}, and is not {self.METADATA_FILENAMES} either." + ) + else: + archives, downloaded_dirs = files_or_archives, downloaded_files_or_dirs + for archive, downloaded_dir in zip(archives, downloaded_dirs): + archive, downloaded_dir = str(archive), str(downloaded_dir) + for downloaded_dir_file in dl_manager.iter_files(downloaded_dir): + _, downloaded_dir_file_ext = os.path.splitext(downloaded_dir_file) + if downloaded_dir_file_ext in self.EXTENSIONS: + if not self.config.drop_labels: + labels.add(os.path.basename(os.path.dirname(downloaded_dir_file))) + path_depths.add(count_path_segments(downloaded_dir_file)) + elif os.path.basename(downloaded_dir_file) in self.METADATA_FILENAMES: + metadata_files[split].add((None, downloaded_dir_file)) + else: + archive_file_name = os.path.basename(archive) + original_file_name = os.path.basename(downloaded_dir_file) + logger.debug( + f"The file '{original_file_name}' from the archive '{archive_file_name}' was ignored: it is not a {self.BASE_COLUMN_NAME}, and is not {self.METADATA_FILENAMES} either." + ) + + data_files = self.config.data_files + splits = [] + for split_name, files in data_files.items(): + if isinstance(files, str): + files = [files] + files, archives = self._split_files_and_archives(files) + downloaded_files = dl_manager.download(files) + downloaded_dirs = dl_manager.download_and_extract(archives) + if do_analyze: # drop_metadata is None or False, drop_labels is None or False + logger.info(f"Searching for labels and/or metadata files in {split_name} data files...") + analyze(files, downloaded_files, split_name) + analyze(archives, downloaded_dirs, split_name) + + if metadata_files: + # add metadata if `metadata_files` are found and `drop_metadata` is None (default) or False + add_metadata = not self.config.drop_metadata + # if `metadata_files` are found, don't add labels + add_labels = False + else: + # if `metadata_files` are not found, don't add metadata + add_metadata = False + # if `metadata_files` are not found and `drop_labels` is None (default) - + # add labels if files are on the same level in directory hierarchy and there is more than one label + add_labels = ( + (len(labels) > 1 and len(path_depths) == 1) + if self.config.drop_labels is None + else not self.config.drop_labels + ) + + if add_labels: + logger.info("Adding the labels inferred from data directories to the dataset's features...") + if add_metadata: + logger.info("Adding metadata to the dataset...") + else: + add_labels, add_metadata, metadata_files = False, False, {} + + splits.append( + datasets.SplitGenerator( + name=split_name, + gen_kwargs={ + "files": tuple(zip(files, downloaded_files)) + + tuple((None, dl_manager.iter_files(downloaded_dir)) for downloaded_dir in downloaded_dirs), + "metadata_files": metadata_files.get(split_name, []), + "add_labels": add_labels, + "add_metadata": add_metadata, + }, + ) + ) + + if add_metadata: + # Verify that: + # * all metadata files have the same set of features in each split + # * the `file_name` key is one of the metadata keys and is of type string + features_per_metadata_file: list[tuple[str, datasets.Features]] = [] + + # Check that all metadata files share the same format + metadata_ext = { + os.path.splitext(original_metadata_file or downloaded_metadata_file)[-1] + for original_metadata_file, downloaded_metadata_file in itertools.chain.from_iterable( + metadata_files.values() + ) + } + if len(metadata_ext) > 1: + raise ValueError(f"Found metadata files with different extensions: {list(metadata_ext)}") + metadata_ext = metadata_ext.pop() + + for split_metadata_files in metadata_files.values(): + pa_metadata_table = None + for _, downloaded_metadata_file in split_metadata_files: + for pa_metadata_table in self._read_metadata(downloaded_metadata_file, metadata_ext=metadata_ext): + break # just fetch the first rows + if pa_metadata_table is not None: + features_per_metadata_file.append( + (downloaded_metadata_file, datasets.Features.from_arrow_schema(pa_metadata_table.schema)) + ) + break # no need to fetch all the files + for downloaded_metadata_file, metadata_features in features_per_metadata_file: + if metadata_features != features_per_metadata_file[0][1]: + raise ValueError( + f"Metadata files {downloaded_metadata_file} and {features_per_metadata_file[0][0]} have different features: {features_per_metadata_file[0]} != {metadata_features}" + ) + metadata_features = features_per_metadata_file[0][1] + feature_not_found = True + + def _set_feature(feature): + nonlocal feature_not_found + if isinstance(feature, dict): + out = type(feature)() + for key in feature: + if (key == "file_name" or key.endswith("_file_name")) and feature[key] == datasets.Value( + "string" + ): + key = key[: -len("_file_name")] or self.BASE_COLUMN_NAME + out[key] = self.BASE_FEATURE() + feature_not_found = False + elif (key == "file_names" or key.endswith("_file_names")) and feature[ + key + ] == datasets.Sequence(datasets.Value("string")): + key = key[: -len("_file_names")] or (self.BASE_COLUMN_NAME + "s") + out[key] = datasets.Sequence(self.BASE_FEATURE()) + feature_not_found = False + elif (key == "file_names" or key.endswith("_file_names")) and feature[key] == [ + datasets.Value("string") + ]: + key = key[: -len("_file_names")] or (self.BASE_COLUMN_NAME + "s") + out[key] = [self.BASE_FEATURE()] + feature_not_found = False + else: + out[key] = feature[key] + return out + return feature + + metadata_features = _visit(metadata_features, _set_feature) + + if feature_not_found: + raise ValueError( + "`file_name` or `*_file_name` must be present as dictionary key (with type string) in metadata files" + ) + else: + metadata_features = None + + # Normally, we would do this in _info, but we need to know the labels and/or metadata + # before building the features + if self.config.features is None: + if add_metadata: + self.info.features = metadata_features + elif add_labels: + self.info.features = datasets.Features( + { + self.BASE_COLUMN_NAME: self.BASE_FEATURE(), + "label": datasets.ClassLabel(names=sorted(labels)), + } + ) + else: + self.info.features = datasets.Features({self.BASE_COLUMN_NAME: self.BASE_FEATURE()}) + + return splits + + def _split_files_and_archives(self, data_files): + files, archives = [], [] + for data_file in data_files: + _, data_file_ext = os.path.splitext(data_file) + if data_file_ext.lower() in self.EXTENSIONS: + files.append(data_file) + elif os.path.basename(data_file) in self.METADATA_FILENAMES: + files.append(data_file) + else: + archives.append(data_file) + return files, archives + + def _read_metadata(self, metadata_file: str, metadata_ext: str = "") -> Iterator[pa.Table]: + """using the same logic as the Csv, Json and Parquet dataset builders to stream the data""" + if self.config.filters is not None: + filter_expr = ( + pq.filters_to_expression(self.config.filters) + if isinstance(self.config.filters, list) + else self.config.filters + ) + else: + filter_expr = None + if metadata_ext == ".csv": + chunksize = 10_000 # 10k lines + schema = self.config.features.arrow_schema if self.config.features else None + # dtype allows reading an int column as str + dtype = ( + { + name: dtype.to_pandas_dtype() if not require_storage_cast(feature) else object + for name, dtype, feature in zip(schema.names, schema.types, self.config.features.values()) + } + if schema is not None + else None + ) + csv_file_reader = pd.read_csv(metadata_file, iterator=True, dtype=dtype, chunksize=chunksize) + for df in csv_file_reader: + pa_table = pa.Table.from_pandas(df) + if self.config.filters is not None: + pa_table = pa_table.filter(filter_expr) + if len(pa_table) > 0: + yield pa_table + elif metadata_ext == ".jsonl": + with open(metadata_file, "rb") as f: + chunksize: int = 10 << 20 # 10MB + # Use block_size equal to the chunk size divided by 32 to leverage multithreading + # Set a default minimum value of 16kB if the chunk size is really small + block_size = max(chunksize // 32, 16 << 10) + while True: + batch = f.read(chunksize) + if not batch: + break + # Finish current line + try: + batch += f.readline() + except (AttributeError, io.UnsupportedOperation): + batch += readline(f) + while True: + try: + pa_table = paj.read_json( + io.BytesIO(batch), read_options=paj.ReadOptions(block_size=block_size) + ) + break + except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e: + if ( + isinstance(e, pa.ArrowInvalid) + and "straddling" not in str(e) + or block_size > len(batch) + ): + raise + else: + # Increase the block size in case it was too small. + # The block size will be reset for the next file. + logger.debug( + f"Batch of {len(batch)} bytes couldn't be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}." + ) + block_size *= 2 + if self.config.filters is not None: + pa_table = pa_table.filter(filter_expr) + if len(pa_table) > 0: + yield pa_table + else: + with open(metadata_file, "rb") as f: + parquet_fragment = ds.ParquetFileFormat().make_fragment(f) + if parquet_fragment.row_groups: + batch_size = parquet_fragment.row_groups[0].num_rows + else: + batch_size = config.DEFAULT_MAX_BATCH_SIZE + for record_batch in parquet_fragment.to_batches( + batch_size=batch_size, + filter=filter_expr, + batch_readahead=0, + fragment_readahead=0, + ): + yield pa.Table.from_batches([record_batch]) + + def _generate_examples(self, files, metadata_files, add_metadata, add_labels): + sample_idx = 0 + if add_metadata: + feature_paths = [] + + def find_feature_path(feature, feature_path): + nonlocal feature_paths + if feature_path and isinstance(feature, self.BASE_FEATURE): + feature_paths.append(feature_path) + + _visit_with_path(self.info.features, find_feature_path) + + for original_metadata_file, downloaded_metadata_file in metadata_files: + metadata_ext = os.path.splitext(original_metadata_file or downloaded_metadata_file)[-1] + downloaded_metadata_dir = os.path.dirname(downloaded_metadata_file) + + def set_feature(item, feature_path: _VisitPath): + if len(feature_path) == 2 and isinstance(feature_path[0], str) and feature_path[1] == 0: + item[feature_path[0]] = item.pop("file_names", None) or item.pop( + feature_path[0] + "_file_names", None + ) + elif len(feature_path) == 1 and isinstance(feature_path[0], str): + item[feature_path[0]] = item.pop("file_name", None) or item.pop( + feature_path[0] + "_file_name", None + ) + elif len(feature_path) == 0: + file_relpath = os.path.normpath(item).replace("\\", "/") + item = os.path.join(downloaded_metadata_dir, file_relpath) + return item + + for pa_metadata_table in self._read_metadata(downloaded_metadata_file, metadata_ext=metadata_ext): + for sample in pa_metadata_table.to_pylist(): + for feature_path in feature_paths: + _nested_apply(sample, feature_path, set_feature) + yield sample_idx, sample + sample_idx += 1 + else: + if self.config.filters is not None: + filter_expr = ( + pq.filters_to_expression(self.config.filters) + if isinstance(self.config.filters, list) + else self.config.filters + ) + for original_file, downloaded_file_or_dir in files: + downloaded_files = [downloaded_file_or_dir] if original_file else downloaded_file_or_dir + for downloaded_file in downloaded_files: + original_file_ext = os.path.splitext(original_file or downloaded_file)[-1] + if original_file_ext.lower() not in self.EXTENSIONS: + continue + sample = {self.BASE_COLUMN_NAME: downloaded_file} + if add_labels: + sample["label"] = os.path.basename(os.path.dirname(original_file or downloaded_file)) + if self.config.filters is not None: + pa_table = pa.Table.from_pylist([sample]).filter(filter_expr) + if len(pa_table) == 0: + continue + yield sample_idx, sample + sample_idx += 1 + + +def _nested_apply(item: Any, feature_path: _VisitPath, func: Callable[[Any, _VisitPath], Any]): + # see _visit_with_path() to see how feature paths are constructed + item = func(item, feature_path) + if feature_path: + key = feature_path[0] + if key == 0: + for i in range(len(item)): + item[i] = _nested_apply(item[i], feature_path[1:], func) + else: + item[key] = _nested_apply(item[key], feature_path[1:], func) + return item diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/generator/__init__.py b/lib/python3.12/site-packages/datasets/packaged_modules/generator/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/generator/__pycache__/generator.cpython-312.pyc b/lib/python3.12/site-packages/datasets/packaged_modules/generator/__pycache__/generator.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9d9c11db9c7d43eebf222c7a71f651acc72c35a5 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/packaged_modules/generator/__pycache__/generator.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/generator/generator.py b/lib/python3.12/site-packages/datasets/packaged_modules/generator/generator.py new file mode 100644 index 0000000000000000000000000000000000000000..dac3f0d92dfea56636e24c9c8e0cba9e46e96a90 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/packaged_modules/generator/generator.py @@ -0,0 +1,33 @@ +from dataclasses import dataclass +from typing import Callable, Optional + +import datasets + + +@dataclass +class GeneratorConfig(datasets.BuilderConfig): + generator: Optional[Callable] = None + gen_kwargs: Optional[dict] = None + features: Optional[datasets.Features] = None + split: datasets.NamedSplit = datasets.Split.TRAIN + + def __post_init__(self): + super().__post_init__() + if self.generator is None: + raise ValueError("generator must be specified") + + if self.gen_kwargs is None: + self.gen_kwargs = {} + + +class Generator(datasets.GeneratorBasedBuilder): + BUILDER_CONFIG_CLASS = GeneratorConfig + + def _info(self): + return datasets.DatasetInfo(features=self.config.features) + + def _split_generators(self, dl_manager): + return [datasets.SplitGenerator(name=self.config.split, gen_kwargs=self.config.gen_kwargs)] + + def _generate_examples(self, **gen_kwargs): + yield from enumerate(self.config.generator(**gen_kwargs)) diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/imagefolder/__init__.py b/lib/python3.12/site-packages/datasets/packaged_modules/imagefolder/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/imagefolder/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/datasets/packaged_modules/imagefolder/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3a90648d861016ae6532b3e1af6b84a0671dd694 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/packaged_modules/imagefolder/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/imagefolder/__pycache__/imagefolder.cpython-312.pyc b/lib/python3.12/site-packages/datasets/packaged_modules/imagefolder/__pycache__/imagefolder.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..68adabb5c12ef636263a11cdb08607ed1d98bc0b Binary files /dev/null and b/lib/python3.12/site-packages/datasets/packaged_modules/imagefolder/__pycache__/imagefolder.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/imagefolder/imagefolder.py b/lib/python3.12/site-packages/datasets/packaged_modules/imagefolder/imagefolder.py new file mode 100644 index 0000000000000000000000000000000000000000..aba9452ad4e12833c60ef78df8b3a0cae519f4a4 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/packaged_modules/imagefolder/imagefolder.py @@ -0,0 +1,103 @@ +import datasets + +from ..folder_based_builder import folder_based_builder + + +logger = datasets.utils.logging.get_logger(__name__) + + +class ImageFolderConfig(folder_based_builder.FolderBasedBuilderConfig): + """BuilderConfig for ImageFolder.""" + + drop_labels: bool = None + drop_metadata: bool = None + + def __post_init__(self): + super().__post_init__() + + +class ImageFolder(folder_based_builder.FolderBasedBuilder): + BASE_FEATURE = datasets.Image + BASE_COLUMN_NAME = "image" + BUILDER_CONFIG_CLASS = ImageFolderConfig + EXTENSIONS: list[str] # definition at the bottom of the script + + +# Obtained with: +# ``` +# import PIL.Image +# IMAGE_EXTENSIONS = [] +# PIL.Image.init() +# for ext, format in PIL.Image.EXTENSION.items(): +# if format in PIL.Image.OPEN: +# IMAGE_EXTENSIONS.append(ext[1:]) +# ``` +# We intentionally do not run this code on launch because: +# (1) Pillow is an optional dependency, so importing Pillow in global namespace is not allowed +# (2) To ensure the list of supported extensions is deterministic +IMAGE_EXTENSIONS = [ + ".blp", + ".bmp", + ".dib", + ".bufr", + ".cur", + ".pcx", + ".dcx", + ".dds", + ".ps", + ".eps", + ".fit", + ".fits", + ".fli", + ".flc", + ".ftc", + ".ftu", + ".gbr", + ".gif", + ".grib", + # ".h5", # may contain zero or several images + # ".hdf", # may contain zero or several images + ".png", + ".apng", + ".jp2", + ".j2k", + ".jpc", + ".jpf", + ".jpx", + ".j2c", + ".icns", + ".ico", + ".im", + ".iim", + ".tif", + ".tiff", + ".jfif", + ".jpe", + ".jpg", + ".jpeg", + ".mpg", + ".mpeg", + ".msp", + ".pcd", + ".pxr", + ".pbm", + ".pgm", + ".ppm", + ".pnm", + ".psd", + ".bw", + ".rgb", + ".rgba", + ".sgi", + ".ras", + ".tga", + ".icb", + ".vda", + ".vst", + ".webp", + ".wmf", + ".emf", + ".xbm", + ".xpm", +] +ImageFolder.EXTENSIONS = IMAGE_EXTENSIONS diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/sql/__init__.py b/lib/python3.12/site-packages/datasets/packaged_modules/sql/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/sql/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/datasets/packaged_modules/sql/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7773fdf3f6de5e0373ebef6c327edd006172f43b Binary files /dev/null and b/lib/python3.12/site-packages/datasets/packaged_modules/sql/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/sql/__pycache__/sql.cpython-312.pyc b/lib/python3.12/site-packages/datasets/packaged_modules/sql/__pycache__/sql.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fc098b5dd883ca0a24dd50d729e419a9b3a021b1 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/packaged_modules/sql/__pycache__/sql.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/sql/sql.py b/lib/python3.12/site-packages/datasets/packaged_modules/sql/sql.py new file mode 100644 index 0000000000000000000000000000000000000000..d8e71664c8e1fde656f681249f56bb6d3ab07da8 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/packaged_modules/sql/sql.py @@ -0,0 +1,119 @@ +import sys +from dataclasses import dataclass +from typing import TYPE_CHECKING, Optional, Union + +import pandas as pd +import pyarrow as pa + +import datasets +import datasets.config +from datasets.features.features import require_storage_cast +from datasets.table import table_cast + + +if TYPE_CHECKING: + import sqlite3 + + import sqlalchemy + + +logger = datasets.utils.logging.get_logger(__name__) + + +@dataclass +class SqlConfig(datasets.BuilderConfig): + """BuilderConfig for SQL.""" + + sql: Union[str, "sqlalchemy.sql.Selectable"] = None + con: Union[str, "sqlalchemy.engine.Connection", "sqlalchemy.engine.Engine", "sqlite3.Connection"] = None + index_col: Optional[Union[str, list[str]]] = None + coerce_float: bool = True + params: Optional[Union[list, tuple, dict]] = None + parse_dates: Optional[Union[list, dict]] = None + columns: Optional[list[str]] = None + chunksize: Optional[int] = 10_000 + features: Optional[datasets.Features] = None + + def __post_init__(self): + super().__post_init__() + if self.sql is None: + raise ValueError("sql must be specified") + if self.con is None: + raise ValueError("con must be specified") + + def create_config_id( + self, + config_kwargs: dict, + custom_features: Optional[datasets.Features] = None, + ) -> str: + config_kwargs = config_kwargs.copy() + # We need to stringify the Selectable object to make its hash deterministic + + # The process of stringifying is explained here: http://docs.sqlalchemy.org/en/latest/faq/sqlexpressions.html + sql = config_kwargs["sql"] + if not isinstance(sql, str): + if datasets.config.SQLALCHEMY_AVAILABLE and "sqlalchemy" in sys.modules: + import sqlalchemy + + if isinstance(sql, sqlalchemy.sql.Selectable): + engine = sqlalchemy.create_engine(config_kwargs["con"].split("://")[0] + "://") + sql_str = str(sql.compile(dialect=engine.dialect)) + config_kwargs["sql"] = sql_str + else: + raise TypeError( + f"Supported types for 'sql' are string and sqlalchemy.sql.Selectable but got {type(sql)}: {sql}" + ) + else: + raise TypeError( + f"Supported types for 'sql' are string and sqlalchemy.sql.Selectable but got {type(sql)}: {sql}" + ) + con = config_kwargs["con"] + if not isinstance(con, str): + config_kwargs["con"] = id(con) + logger.info( + f"SQL connection 'con' of type {type(con)} couldn't be hashed properly. To enable hashing, specify 'con' as URI string instead." + ) + + return super().create_config_id(config_kwargs, custom_features=custom_features) + + @property + def pd_read_sql_kwargs(self): + pd_read_sql_kwargs = { + "index_col": self.index_col, + "columns": self.columns, + "params": self.params, + "coerce_float": self.coerce_float, + "parse_dates": self.parse_dates, + } + return pd_read_sql_kwargs + + +class Sql(datasets.ArrowBasedBuilder): + BUILDER_CONFIG_CLASS = SqlConfig + + def _info(self): + return datasets.DatasetInfo(features=self.config.features) + + def _split_generators(self, dl_manager): + return [datasets.SplitGenerator(name=datasets.Split.TRAIN, gen_kwargs={})] + + def _cast_table(self, pa_table: pa.Table) -> pa.Table: + if self.config.features is not None: + schema = self.config.features.arrow_schema + if all(not require_storage_cast(feature) for feature in self.config.features.values()): + # cheaper cast + pa_table = pa.Table.from_arrays([pa_table[field.name] for field in schema], schema=schema) + else: + # more expensive cast; allows str <-> int/float or str to Audio for example + pa_table = table_cast(pa_table, schema) + return pa_table + + def _generate_tables(self): + chunksize = self.config.chunksize + sql_reader = pd.read_sql( + self.config.sql, self.config.con, chunksize=chunksize, **self.config.pd_read_sql_kwargs + ) + sql_reader = [sql_reader] if chunksize is None else sql_reader + for chunk_idx, df in enumerate(sql_reader): + pa_table = pa.Table.from_pandas(df) + yield chunk_idx, self._cast_table(pa_table) diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/videofolder/__init__.py b/lib/python3.12/site-packages/datasets/packaged_modules/videofolder/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/videofolder/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/datasets/packaged_modules/videofolder/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8bca7ec79b8341a9b774cd64d638fb5928f2dcc2 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/packaged_modules/videofolder/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/videofolder/__pycache__/videofolder.cpython-312.pyc b/lib/python3.12/site-packages/datasets/packaged_modules/videofolder/__pycache__/videofolder.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7b13407de67e0c85f1383f9a0384ab53ee4b4910 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/packaged_modules/videofolder/__pycache__/videofolder.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/videofolder/videofolder.py b/lib/python3.12/site-packages/datasets/packaged_modules/videofolder/videofolder.py new file mode 100644 index 0000000000000000000000000000000000000000..099a17a8d09e981b34c4a40a3f5e8c948d0efa5d --- /dev/null +++ b/lib/python3.12/site-packages/datasets/packaged_modules/videofolder/videofolder.py @@ -0,0 +1,34 @@ +import datasets + +from ..folder_based_builder import folder_based_builder + + +logger = datasets.utils.logging.get_logger(__name__) + + +class VideoFolderConfig(folder_based_builder.FolderBasedBuilderConfig): + """BuilderConfig for ImageFolder.""" + + drop_labels: bool = None + drop_metadata: bool = None + + def __post_init__(self): + super().__post_init__() + + +class VideoFolder(folder_based_builder.FolderBasedBuilder): + BASE_FEATURE = datasets.Video + BASE_COLUMN_NAME = "video" + BUILDER_CONFIG_CLASS = VideoFolderConfig + EXTENSIONS: list[str] # definition at the bottom of the script + + +# TODO: initial list, we should check the compatibility of other formats +VIDEO_EXTENSIONS = [ + ".mkv", + ".mp4", + ".avi", + ".mpeg", + ".mov", +] +VideoFolder.EXTENSIONS = VIDEO_EXTENSIONS diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/xml/__init__.py b/lib/python3.12/site-packages/datasets/packaged_modules/xml/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/xml/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/datasets/packaged_modules/xml/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0daadacd7adaaadc9397bb4ad5c5483dd9119b32 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/packaged_modules/xml/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/xml/__pycache__/xml.cpython-312.pyc b/lib/python3.12/site-packages/datasets/packaged_modules/xml/__pycache__/xml.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7f398ee9514d61531cea07d2bd1180bdaf5f4427 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/packaged_modules/xml/__pycache__/xml.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/xml/xml.py b/lib/python3.12/site-packages/datasets/packaged_modules/xml/xml.py new file mode 100644 index 0000000000000000000000000000000000000000..d5009b4dd6ad3d7936eb7b96c4c0ecaf207e5915 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/packaged_modules/xml/xml.py @@ -0,0 +1,68 @@ +import itertools +from dataclasses import dataclass +from typing import Optional + +import pyarrow as pa + +import datasets +from datasets.features.features import require_storage_cast +from datasets.table import table_cast + + +logger = datasets.utils.logging.get_logger(__name__) + + +@dataclass +class XmlConfig(datasets.BuilderConfig): + """BuilderConfig for xml files.""" + + features: Optional[datasets.Features] = None + encoding: str = "utf-8" + encoding_errors: Optional[str] = None + + +class Xml(datasets.ArrowBasedBuilder): + BUILDER_CONFIG_CLASS = XmlConfig + + def _info(self): + return datasets.DatasetInfo(features=self.config.features) + + def _split_generators(self, dl_manager): + """The `data_files` kwarg in load_dataset() can be a str, List[str], Dict[str,str], or Dict[str,List[str]]. + + If str or List[str], then the dataset returns only the 'train' split. + If dict, then keys should be from the `datasets.Split` enum. + """ + if not self.config.data_files: + raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}") + dl_manager.download_config.extract_on_the_fly = True + data_files = dl_manager.download_and_extract(self.config.data_files) + splits = [] + for split_name, files in data_files.items(): + if isinstance(files, str): + files = [files] + files = [dl_manager.iter_files(file) for file in files] + splits.append(datasets.SplitGenerator(name=split_name, gen_kwargs={"files": files})) + return splits + + def _cast_table(self, pa_table: pa.Table) -> pa.Table: + if self.config.features is not None: + schema = self.config.features.arrow_schema + if all(not require_storage_cast(feature) for feature in self.config.features.values()): + # cheaper cast + pa_table = pa_table.cast(schema) + else: + # more expensive cast; allows str <-> int/float or str to Audio for example + pa_table = table_cast(pa_table, schema) + return pa_table + else: + return pa_table.cast(pa.schema({"xml": pa.string()})) + + def _generate_tables(self, files): + pa_table_names = list(self.config.features) if self.config.features is not None else ["xml"] + for file_idx, file in enumerate(itertools.chain.from_iterable(files)): + # open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n" + with open(file, encoding=self.config.encoding, errors=self.config.encoding_errors) as f: + xml = f.read() + pa_table = pa.Table.from_arrays([pa.array([xml])], names=pa_table_names) + yield file_idx, self._cast_table(pa_table) diff --git a/lib/python3.12/site-packages/datasets/search.py b/lib/python3.12/site-packages/datasets/search.py new file mode 100644 index 0000000000000000000000000000000000000000..07ec6c93bc13329616ed0600e497215dc9c3b0ef --- /dev/null +++ b/lib/python3.12/site-packages/datasets/search.py @@ -0,0 +1,785 @@ +import importlib.util +import os +import tempfile +from pathlib import PurePath +from typing import TYPE_CHECKING, NamedTuple, Optional, Union + +import fsspec +import numpy as np + +from .features import Sequence +from .utils import logging +from .utils import tqdm as hf_tqdm + + +if TYPE_CHECKING: + from .arrow_dataset import Dataset # noqa: F401 + + try: + from elasticsearch import Elasticsearch # noqa: F401 + + except ImportError: + pass + try: + import faiss # noqa: F401 + + except ImportError: + pass + +_has_elasticsearch = importlib.util.find_spec("elasticsearch") is not None +_has_faiss = importlib.util.find_spec("faiss") is not None + + +logger = logging.get_logger(__name__) + + +class MissingIndex(Exception): + pass + + +class SearchResults(NamedTuple): + scores: list[float] + indices: list[int] + + +class BatchedSearchResults(NamedTuple): + total_scores: list[list[float]] + total_indices: list[list[int]] + + +class NearestExamplesResults(NamedTuple): + scores: list[float] + examples: dict + + +class BatchedNearestExamplesResults(NamedTuple): + total_scores: list[list[float]] + total_examples: list[dict] + + +class BaseIndex: + """Base class for indexing""" + + def search(self, query, k: int = 10, **kwargs) -> SearchResults: + """ + To implement. + This method has to return the scores and the indices of the retrieved examples given a certain query. + """ + raise NotImplementedError + + def search_batch(self, queries, k: int = 10, **kwargs) -> BatchedSearchResults: + """Find the nearest examples indices to the query. + + Args: + queries (`Union[List[str], np.ndarray]`): The queries as a list of strings if `column` is a text index or as a numpy array if `column` is a vector index. + k (`int`): The number of examples to retrieve per query. + + Output: + total_scores (`List[List[float]`): The retrieval scores of the retrieved examples per query. + total_indices (`List[List[int]]`): The indices of the retrieved examples per query. + """ + total_scores, total_indices = [], [] + for query in queries: + scores, indices = self.search(query, k) + total_scores.append(scores) + total_indices.append(indices) + return BatchedSearchResults(total_scores, total_indices) + + def save(self, file: Union[str, PurePath]): + """Serialize the index on disk""" + raise NotImplementedError + + @classmethod + def load(cls, file: Union[str, PurePath]) -> "BaseIndex": + """Deserialize the index from disk""" + raise NotImplementedError + + +class ElasticSearchIndex(BaseIndex): + """ + Sparse index using Elasticsearch. It is used to index text and run queries based on BM25 similarity. + An Elasticsearch server needs to be accessible, and a python client is declared with + ``` + es_client = Elasticsearch([{'host': 'localhost', 'port': '9200'}]) + ``` + for example. + """ + + def __init__( + self, + host: Optional[str] = None, + port: Optional[int] = None, + es_client: Optional["Elasticsearch"] = None, + es_index_name: Optional[str] = None, + es_index_config: Optional[dict] = None, + ): + if not _has_elasticsearch: + raise ImportError( + "You must install ElasticSearch to use ElasticSearchIndex. To do so you can run `pip install elasticsearch==7.7.1 for example`" + ) + if es_client is not None and (host is not None or port is not None): + raise ValueError("Please specify either `es_client` or `(host, port)`, but not both.") + host = host or "localhost" + port = port or 9200 + + import elasticsearch.helpers # noqa: F401 - need this to properly load all the es features + from elasticsearch import Elasticsearch # noqa: F811 + + self.es_client = es_client if es_client is not None else Elasticsearch([{"host": host, "port": str(port)}]) + self.es_index_name = ( + es_index_name + if es_index_name is not None + else "huggingface_datasets_" + os.path.basename(tempfile.NamedTemporaryFile().name) + ) + self.es_index_config = ( + es_index_config + if es_index_config is not None + else { + "settings": { + "number_of_shards": 1, + "analysis": {"analyzer": {"stop_standard": {"type": "standard", " stopwords": "_english_"}}}, + }, + "mappings": {"properties": {"text": {"type": "text", "analyzer": "standard", "similarity": "BM25"}}}, + } + ) + + def add_documents(self, documents: Union[list[str], "Dataset"], column: Optional[str] = None): + """ + Add documents to the index. + If the documents are inside a certain column, you can specify it using the `column` argument. + """ + index_name = self.es_index_name + index_config = self.es_index_config + self.es_client.indices.create(index=index_name, body=index_config) + number_of_docs = len(documents) + progress = hf_tqdm(unit="docs", total=number_of_docs) + successes = 0 + + def passage_generator(): + if column is not None: + for i, example in enumerate(documents): + yield {"text": example[column], "_id": i} + else: + for i, example in enumerate(documents): + yield {"text": example, "_id": i} + + # create the ES index + import elasticsearch as es + + for ok, action in es.helpers.streaming_bulk( + client=self.es_client, + index=index_name, + actions=passage_generator(), + ): + progress.update(1) + successes += ok + if successes != len(documents): + logger.warning( + f"Some documents failed to be added to ElasticSearch. Failures: {len(documents) - successes}/{len(documents)}" + ) + logger.info(f"Indexed {successes:d} documents") + + def search(self, query: str, k=10, **kwargs) -> SearchResults: + """Find the nearest examples indices to the query. + + Args: + query (`str`): The query as a string. + k (`int`): The number of examples to retrieve. + + Output: + scores (`List[List[float]`): The retrieval scores of the retrieved examples. + indices (`List[List[int]]`): The indices of the retrieved examples. + """ + response = self.es_client.search( + index=self.es_index_name, + body={"query": {"multi_match": {"query": query, "fields": ["text"], "type": "cross_fields"}}, "size": k}, + **kwargs, + ) + hits = response["hits"]["hits"] + return SearchResults([hit["_score"] for hit in hits], [int(hit["_id"]) for hit in hits]) + + def search_batch(self, queries, k: int = 10, max_workers=10, **kwargs) -> BatchedSearchResults: + import concurrent.futures + + total_scores, total_indices = [None] * len(queries), [None] * len(queries) + with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: + future_to_index = {executor.submit(self.search, query, k, **kwargs): i for i, query in enumerate(queries)} + for future in concurrent.futures.as_completed(future_to_index): + index = future_to_index[future] + results: SearchResults = future.result() + total_scores[index] = results.scores + total_indices[index] = results.indices + return BatchedSearchResults(total_indices=total_indices, total_scores=total_scores) + + +class FaissIndex(BaseIndex): + """ + Dense index using Faiss. It is used to index vectors. + Faiss is a library for efficient similarity search and clustering of dense vectors. + It contains algorithms that search in sets of vectors of any size, up to ones that possibly do not fit in RAM. + You can find more information about Faiss here: + - For index types and the string factory: https://github.com/facebookresearch/faiss/wiki/The-index-factory + - For GPU settings: https://github.com/facebookresearch/faiss/wiki/Faiss-on-the-GPU + """ + + def __init__( + self, + device: Optional[Union[int, list[int]]] = None, + string_factory: Optional[str] = None, + metric_type: Optional[int] = None, + custom_index: Optional["faiss.Index"] = None, + ): + """ + Create a Dense index using Faiss. You can specify `device` if you want to run it on GPU (`device` must be the GPU index). + You can find more information about Faiss here: + - For `string factory`: https://github.com/facebookresearch/faiss/wiki/The-index-factory + """ + if string_factory is not None and custom_index is not None: + raise ValueError("Please specify either `string_factory` or `custom_index` but not both.") + if device is not None and custom_index is not None: + raise ValueError( + "Cannot pass both 'custom_index' and 'device'. " + "Pass 'custom_index' already transferred to the target device instead." + ) + self.device = device + self.string_factory = string_factory + self.metric_type = metric_type + self.faiss_index = custom_index + if not _has_faiss: + raise ImportError( + "You must install Faiss to use FaissIndex. To do so you can run `conda install -c pytorch faiss-cpu` or `conda install -c pytorch faiss-gpu`. " + "A community supported package is also available on pypi: `pip install faiss-cpu` or `pip install faiss-gpu`. " + "Note that pip may not have the latest version of FAISS, and thus, some of the latest features and bug fixes may not be available." + ) + + def add_vectors( + self, + vectors: Union[np.array, "Dataset"], + column: Optional[str] = None, + batch_size: int = 1000, + train_size: Optional[int] = None, + faiss_verbose: Optional[bool] = None, + ): + """ + Add vectors to the index. + If the arrays are inside a certain column, you can specify it using the `column` argument. + """ + import faiss # noqa: F811 + + if column and not isinstance(vectors.features[column], Sequence): + raise ValueError( + f"Wrong feature type for column '{column}'. Expected 1d array, got {vectors.features[column]}" + ) + + # Create index + if self.faiss_index is None: + size = len(vectors[0]) if column is None else len(vectors[0][column]) + if self.string_factory is not None: + if self.metric_type is None: + index = faiss.index_factory(size, self.string_factory) + else: + index = faiss.index_factory(size, self.string_factory, self.metric_type) + else: + if self.metric_type is None: + index = faiss.IndexFlat(size) + else: + index = faiss.IndexFlat(size, self.metric_type) + + self.faiss_index = self._faiss_index_to_device(index, self.device) + logger.info(f"Created faiss index of type {type(self.faiss_index)}") + + # Set verbosity level + if faiss_verbose is not None: + self.faiss_index.verbose = faiss_verbose + if hasattr(self.faiss_index, "index") and self.faiss_index.index is not None: + self.faiss_index.index.verbose = faiss_verbose + if hasattr(self.faiss_index, "quantizer") and self.faiss_index.quantizer is not None: + self.faiss_index.quantizer.verbose = faiss_verbose + if hasattr(self.faiss_index, "clustering_index") and self.faiss_index.clustering_index is not None: + self.faiss_index.clustering_index.verbose = faiss_verbose + + # Train + if train_size is not None: + train_vecs = vectors[:train_size] if column is None else vectors[:train_size][column] + logger.info(f"Training the index with the first {len(train_vecs)} vectors") + self.faiss_index.train(train_vecs) + else: + logger.info("Ignored the training step of the faiss index as `train_size` is None.") + + # Add vectors + logger.info(f"Adding {len(vectors)} vectors to the faiss index") + for i in hf_tqdm(range(0, len(vectors), batch_size)): + vecs = vectors[i : i + batch_size] if column is None else vectors[i : i + batch_size][column] + self.faiss_index.add(vecs) + + @staticmethod + def _faiss_index_to_device(index: "faiss.Index", device: Optional[Union[int, list[int]]] = None) -> "faiss.Index": + """ + Sends a faiss index to a device. + A device can either be a positive integer (GPU id), a negative integer (all GPUs), + or a list of positive integers (select GPUs to use), or `None` for CPU. + """ + + # If device is not specified, then it runs on CPU. + if device is None: + return index + + import faiss # noqa: F811 + + # If the device id is given as an integer + if isinstance(device, int): + # Positive integers are directly mapped to GPU ids + if device > -1: + faiss_res = faiss.StandardGpuResources() + index = faiss.index_cpu_to_gpu(faiss_res, device, index) + # And negative integers mean using all GPUs + else: + index = faiss.index_cpu_to_all_gpus(index) + # Device ids given as a list mean mapping to those devices specified. + elif isinstance(device, (list, tuple)): + index = faiss.index_cpu_to_gpus_list(index, gpus=list(device)) + else: + raise TypeError( + f"The argument type: {type(device)} is not expected. " + + "Please pass in either nothing, a positive int, a negative int, or a list of positive ints." + ) + + return index + + def search(self, query: np.array, k=10, **kwargs) -> SearchResults: + """Find the nearest examples indices to the query. + + Args: + query (`np.array`): The query as a numpy array. + k (`int`): The number of examples to retrieve. + + Output: + scores (`List[List[float]`): The retrieval scores of the retrieved examples. + indices (`List[List[int]]`): The indices of the retrieved examples. + """ + if len(query.shape) != 1 and (len(query.shape) != 2 or query.shape[0] != 1): + raise ValueError("Shape of query is incorrect, it has to be either a 1D array or 2D (1, N)") + + queries = query.reshape(1, -1) + if not queries.flags.c_contiguous: + queries = np.asarray(queries, order="C") + scores, indices = self.faiss_index.search(queries, k, **kwargs) + return SearchResults(scores[0], indices[0].astype(int)) + + def search_batch(self, queries: np.array, k=10, **kwargs) -> BatchedSearchResults: + """Find the nearest examples indices to the queries. + + Args: + queries (`np.array`): The queries as a numpy array. + k (`int`): The number of examples to retrieve. + + Output: + total_scores (`List[List[float]`): The retrieval scores of the retrieved examples per query. + total_indices (`List[List[int]]`): The indices of the retrieved examples per query. + """ + if len(queries.shape) != 2: + raise ValueError("Shape of query must be 2D") + if not queries.flags.c_contiguous: + queries = np.asarray(queries, order="C") + scores, indices = self.faiss_index.search(queries, k, **kwargs) + return BatchedSearchResults(scores, indices.astype(int)) + + def save(self, file: Union[str, PurePath], storage_options: Optional[dict] = None): + """Serialize the FaissIndex on disk""" + import faiss # noqa: F811 + + if self.device is not None and isinstance(self.device, (int, list, tuple)): + index = faiss.index_gpu_to_cpu(self.faiss_index) + else: + index = self.faiss_index + + with fsspec.open(str(file), "wb", **(storage_options or {})) as f: + faiss.write_index(index, faiss.BufferedIOWriter(faiss.PyCallbackIOWriter(f.write))) + + @classmethod + def load( + cls, + file: Union[str, PurePath], + device: Optional[Union[int, list[int]]] = None, + storage_options: Optional[dict] = None, + ) -> "FaissIndex": + """Deserialize the FaissIndex from disk""" + import faiss # noqa: F811 + + # Instances of FaissIndex is essentially just a wrapper for faiss indices. + faiss_index = cls(device=device) + with fsspec.open(str(file), "rb", **(storage_options or {})) as f: + index = faiss.read_index(faiss.BufferedIOReader(faiss.PyCallbackIOReader(f.read))) + faiss_index.faiss_index = faiss_index._faiss_index_to_device(index, faiss_index.device) + return faiss_index + + +class IndexableMixin: + """Add indexing features to `datasets.Dataset`""" + + def __init__(self): + self._indexes: dict[str, BaseIndex] = {} + + def __len__(self): + raise NotImplementedError + + def __getitem__(self, key): + raise NotImplementedError + + def is_index_initialized(self, index_name: str) -> bool: + return index_name in self._indexes + + def _check_index_is_initialized(self, index_name: str): + if not self.is_index_initialized(index_name): + raise MissingIndex( + f"Index with index_name '{index_name}' not initialized yet. Please make sure that you call `add_faiss_index` or `add_elasticsearch_index` first." + ) + + def list_indexes(self) -> list[str]: + """List the `colindex_nameumns`/identifiers of all the attached indexes.""" + return list(self._indexes) + + def get_index(self, index_name: str) -> BaseIndex: + """List the `index_name`/identifiers of all the attached indexes. + + Args: + index_name (`str`): Index name. + + Returns: + [`BaseIndex`] + """ + self._check_index_is_initialized(index_name) + return self._indexes[index_name] + + def add_faiss_index( + self, + column: str, + index_name: Optional[str] = None, + device: Optional[Union[int, list[int]]] = None, + string_factory: Optional[str] = None, + metric_type: Optional[int] = None, + custom_index: Optional["faiss.Index"] = None, + batch_size: int = 1000, + train_size: Optional[int] = None, + faiss_verbose: bool = False, + ): + """Add a dense index using Faiss for fast retrieval. + The index is created using the vectors of the specified column. + You can specify `device` if you want to run it on GPU (`device` must be the GPU index, see more below). + You can find more information about Faiss here: + - For `string factory`: https://github.com/facebookresearch/faiss/wiki/The-index-factory + + Args: + column (`str`): The column of the vectors to add to the index. + index_name (Optional `str`): The index_name/identifier of the index. This is the index_name that is used to call `.get_nearest` or `.search`. + By default it corresponds to `column`. + device (Optional `Union[int, List[int]]`): If positive integer, this is the index of the GPU to use. If negative integer, use all GPUs. + If a list of positive integers is passed in, run only on those GPUs. By default it uses the CPU. + string_factory (Optional `str`): This is passed to the index factory of Faiss to create the index. Default index class is IndexFlatIP. + metric_type (Optional `int`): Type of metric. Ex: `faiss.METRIC_INNER_PRODUCT` or `faiss.METRIC_L2`. + custom_index (Optional `faiss.Index`): Custom Faiss index that you already have instantiated and configured for your needs. + batch_size (Optional `int`): Size of the batch to use while adding vectors to the FaissIndex. Default value is 1000. + + train_size (Optional `int`): If the index needs a training step, specifies how many vectors will be used to train the index. + faiss_verbose (`bool`, defaults to False): Enable the verbosity of the Faiss index. + """ + index_name = index_name if index_name is not None else column + faiss_index = FaissIndex( + device=device, string_factory=string_factory, metric_type=metric_type, custom_index=custom_index + ) + faiss_index.add_vectors( + self, column=column, batch_size=batch_size, train_size=train_size, faiss_verbose=faiss_verbose + ) + self._indexes[index_name] = faiss_index + + def add_faiss_index_from_external_arrays( + self, + external_arrays: np.array, + index_name: str, + device: Optional[Union[int, list[int]]] = None, + string_factory: Optional[str] = None, + metric_type: Optional[int] = None, + custom_index: Optional["faiss.Index"] = None, + batch_size: int = 1000, + train_size: Optional[int] = None, + faiss_verbose: bool = False, + ): + """Add a dense index using Faiss for fast retrieval. + The index is created using the vectors of `external_arrays`. + You can specify `device` if you want to run it on GPU (`device` must be the GPU index). + You can find more information about Faiss here: + - For `string factory`: https://github.com/facebookresearch/faiss/wiki/The-index-factory + + Args: + external_arrays (`np.array`): If you want to use arrays from outside the lib for the index, you can set `external_arrays`. + It will use `external_arrays` to create the Faiss index instead of the arrays in the given `column`. + index_name (`str`): The index_name/identifier of the index. This is the index_name that is used to call `.get_nearest` or `.search`. + device (Optional `Union[int, List[int]]`): If positive integer, this is the index of the GPU to use. If negative integer, use all GPUs. + If a list of positive integers is passed in, run only on those GPUs. By default it uses the CPU. + string_factory (Optional `str`): This is passed to the index factory of Faiss to create the index. Default index class is IndexFlatIP. + metric_type (Optional `int`): Type of metric. Ex: `faiss.METRIC_INNER_PRODUCT` or `faiss.METRIC_L2`. + custom_index (Optional `faiss.Index`): Custom Faiss index that you already have instantiated and configured for your needs. + batch_size (Optional `int`): Size of the batch to use while adding vectors to the FaissIndex. Default value is 1000. + + train_size (Optional `int`): If the index needs a training step, specifies how many vectors will be used to train the index. + faiss_verbose (`bool`, defaults to False): Enable the verbosity of the Faiss index. + """ + faiss_index = FaissIndex( + device=device, string_factory=string_factory, metric_type=metric_type, custom_index=custom_index + ) + faiss_index.add_vectors( + external_arrays, column=None, batch_size=batch_size, train_size=train_size, faiss_verbose=faiss_verbose + ) + self._indexes[index_name] = faiss_index + + def save_faiss_index(self, index_name: str, file: Union[str, PurePath], storage_options: Optional[dict] = None): + """Save a FaissIndex on disk. + + Args: + index_name (`str`): The index_name/identifier of the index. This is the index_name that is used to call `.get_nearest` or `.search`. + file (`str`): The path to the serialized faiss index on disk or remote URI (e.g. `"s3://my-bucket/index.faiss"`). + storage_options (`dict`, *optional*): + Key/value pairs to be passed on to the file-system backend, if any. + + + + """ + index = self.get_index(index_name) + if not isinstance(index, FaissIndex): + raise ValueError(f"Index '{index_name}' is not a FaissIndex but a '{type(index)}'") + index.save(file, storage_options=storage_options) + logger.info(f"Saved FaissIndex {index_name} at {file}") + + def load_faiss_index( + self, + index_name: str, + file: Union[str, PurePath], + device: Optional[Union[int, list[int]]] = None, + storage_options: Optional[dict] = None, + ): + """Load a FaissIndex from disk. + + If you want to do additional configurations, you can have access to the faiss index object by doing + `.get_index(index_name).faiss_index` to make it fit your needs. + + Args: + index_name (`str`): The index_name/identifier of the index. This is the index_name that is used to + call `.get_nearest` or `.search`. + file (`str`): The path to the serialized faiss index on disk or remote URI (e.g. `"s3://my-bucket/index.faiss"`). + device (Optional `Union[int, List[int]]`): If positive integer, this is the index of the GPU to use. If negative integer, use all GPUs. + If a list of positive integers is passed in, run only on those GPUs. By default it uses the CPU. + storage_options (`dict`, *optional*): + Key/value pairs to be passed on to the file-system backend, if any. + + + + """ + index = FaissIndex.load(file, device=device, storage_options=storage_options) + if index.faiss_index.ntotal != len(self): + raise ValueError( + f"Index size should match Dataset size, but Index '{index_name}' at {file} has {index.faiss_index.ntotal} elements while the dataset has {len(self)} examples." + ) + self._indexes[index_name] = index + logger.info(f"Loaded FaissIndex {index_name} from {file}") + + def add_elasticsearch_index( + self, + column: str, + index_name: Optional[str] = None, + host: Optional[str] = None, + port: Optional[int] = None, + es_client: Optional["Elasticsearch"] = None, + es_index_name: Optional[str] = None, + es_index_config: Optional[dict] = None, + ): + """Add a text index using ElasticSearch for fast retrieval. + + Args: + column (`str`): The column of the documents to add to the index. + index_name (Optional `str`): The index_name/identifier of the index. This is the index name that is used to call `.get_nearest` or `.search`. + By default it corresponds to `column`. + host (Optional `str`, defaults to localhost): + host of where ElasticSearch is running + port (Optional `str`, defaults to 9200): + port of where ElasticSearch is running + es_client (Optional `elasticsearch.Elasticsearch`): + The elasticsearch client used to create the index if host and port are None. + es_index_name (Optional `str`): The elasticsearch index name used to create the index. + es_index_config (Optional `dict`): + The configuration of the elasticsearch index. + Default config is: + + Config:: + + { + "settings": { + "number_of_shards": 1, + "analysis": {"analyzer": {"stop_standard": {"type": "standard", " stopwords": "_english_"}}}, + }, + "mappings": { + "properties": { + "text": { + "type": "text", + "analyzer": "standard", + "similarity": "BM25" + }, + } + }, + } + """ + index_name = index_name if index_name is not None else column + es_index = ElasticSearchIndex( + host=host, port=port, es_client=es_client, es_index_name=es_index_name, es_index_config=es_index_config + ) + es_index.add_documents(self, column=column) + self._indexes[index_name] = es_index + + def load_elasticsearch_index( + self, + index_name: str, + es_index_name: str, + host: Optional[str] = None, + port: Optional[int] = None, + es_client: Optional["Elasticsearch"] = None, + es_index_config: Optional[dict] = None, + ): + """Load an existing text index using ElasticSearch for fast retrieval. + + Args: + index_name (`str`): + The `index_name`/identifier of the index. This is the index name that is used to call `get_nearest` or `search`. + es_index_name (`str`): + The name of elasticsearch index to load. + host (`str`, *optional*, defaults to `localhost`): + Host of where ElasticSearch is running. + port (`str`, *optional*, defaults to `9200`): + Port of where ElasticSearch is running. + es_client (`elasticsearch.Elasticsearch`, *optional*): + The elasticsearch client used to create the index if host and port are `None`. + es_index_config (`dict`, *optional*): + The configuration of the elasticsearch index. + Default config is: + ``` + { + "settings": { + "number_of_shards": 1, + "analysis": {"analyzer": {"stop_standard": {"type": "standard", " stopwords": "_english_"}}}, + }, + "mappings": { + "properties": { + "text": { + "type": "text", + "analyzer": "standard", + "similarity": "BM25" + }, + } + }, + } + ``` + """ + self._indexes[index_name] = ElasticSearchIndex( + host=host, port=port, es_client=es_client, es_index_name=es_index_name, es_index_config=es_index_config + ) + + def drop_index(self, index_name: str): + """Drop the index with the specified column. + + Args: + index_name (`str`): + The `index_name`/identifier of the index. + """ + del self._indexes[index_name] + + def search(self, index_name: str, query: Union[str, np.array], k: int = 10, **kwargs) -> SearchResults: + """Find the nearest examples indices in the dataset to the query. + + Args: + index_name (`str`): + The name/identifier of the index. + query (`Union[str, np.ndarray]`): + The query as a string if `index_name` is a text index or as a numpy array if `index_name` is a vector index. + k (`int`): + The number of examples to retrieve. + + Returns: + `(scores, indices)`: + A tuple of `(scores, indices)` where: + - **scores** (`List[List[float]`): the retrieval scores from either FAISS (`IndexFlatL2` by default) or ElasticSearch of the retrieved examples + - **indices** (`List[List[int]]`): the indices of the retrieved examples + """ + self._check_index_is_initialized(index_name) + return self._indexes[index_name].search(query, k, **kwargs) + + def search_batch( + self, index_name: str, queries: Union[list[str], np.array], k: int = 10, **kwargs + ) -> BatchedSearchResults: + """Find the nearest examples indices in the dataset to the query. + + Args: + index_name (`str`): + The `index_name`/identifier of the index. + queries (`Union[List[str], np.ndarray]`): + The queries as a list of strings if `index_name` is a text index or as a numpy array if `index_name` is a vector index. + k (`int`): + The number of examples to retrieve per query. + + Returns: + `(total_scores, total_indices)`: + A tuple of `(total_scores, total_indices)` where: + - **total_scores** (`List[List[float]`): the retrieval scores from either FAISS (`IndexFlatL2` by default) or ElasticSearch of the retrieved examples per query + - **total_indices** (`List[List[int]]`): the indices of the retrieved examples per query + """ + self._check_index_is_initialized(index_name) + return self._indexes[index_name].search_batch(queries, k, **kwargs) + + def get_nearest_examples( + self, index_name: str, query: Union[str, np.array], k: int = 10, **kwargs + ) -> NearestExamplesResults: + """Find the nearest examples in the dataset to the query. + + Args: + index_name (`str`): + The index_name/identifier of the index. + query (`Union[str, np.ndarray]`): + The query as a string if `index_name` is a text index or as a numpy array if `index_name` is a vector index. + k (`int`): + The number of examples to retrieve. + + Returns: + `(scores, examples)`: + A tuple of `(scores, examples)` where: + - **scores** (`List[float]`): the retrieval scores from either FAISS (`IndexFlatL2` by default) or ElasticSearch of the retrieved examples + - **examples** (`dict`): the retrieved examples + """ + self._check_index_is_initialized(index_name) + scores, indices = self.search(index_name, query, k, **kwargs) + top_indices = [i for i in indices if i >= 0] + return NearestExamplesResults(scores[: len(top_indices)], self[top_indices]) + + def get_nearest_examples_batch( + self, index_name: str, queries: Union[list[str], np.array], k: int = 10, **kwargs + ) -> BatchedNearestExamplesResults: + """Find the nearest examples in the dataset to the query. + + Args: + index_name (`str`): + The `index_name`/identifier of the index. + queries (`Union[List[str], np.ndarray]`): + The queries as a list of strings if `index_name` is a text index or as a numpy array if `index_name` is a vector index. + k (`int`): + The number of examples to retrieve per query. + + Returns: + `(total_scores, total_examples)`: + A tuple of `(total_scores, total_examples)` where: + - **total_scores** (`List[List[float]`): the retrieval scores from either FAISS (`IndexFlatL2` by default) or ElasticSearch of the retrieved examples per query + - **total_examples** (`List[dict]`): the retrieved examples per query + """ + self._check_index_is_initialized(index_name) + total_scores, total_indices = self.search_batch(index_name, queries, k, **kwargs) + total_scores = [ + scores_i[: len([i for i in indices_i if i >= 0])] + for scores_i, indices_i in zip(total_scores, total_indices) + ] + total_samples = [self[[i for i in indices if i >= 0]] for indices in total_indices] + return BatchedNearestExamplesResults(total_scores, total_samples) diff --git a/lib/python3.12/site-packages/datasets/splits.py b/lib/python3.12/site-packages/datasets/splits.py new file mode 100644 index 0000000000000000000000000000000000000000..02c18eef351ec3bd5270d8c6a42ca23710dd5f49 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/splits.py @@ -0,0 +1,635 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +"""Splits related API.""" + +import abc +import collections +import copy +import dataclasses +import re +from dataclasses import dataclass +from typing import Optional, Union + +from .arrow_reader import FileInstructions, make_file_instructions +from .naming import _split_re +from .utils.py_utils import NonMutableDict, asdict + + +@dataclass +class SplitInfo: + name: str = dataclasses.field(default="", metadata={"include_in_asdict_even_if_is_default": True}) + num_bytes: int = dataclasses.field(default=0, metadata={"include_in_asdict_even_if_is_default": True}) + num_examples: int = dataclasses.field(default=0, metadata={"include_in_asdict_even_if_is_default": True}) + shard_lengths: Optional[list[int]] = None + + # Deprecated + # For backward compatibility, this field needs to always be included in files like + # dataset_infos.json and dataset_info.json files + # To do so, we always include it in the output of datasets.utils.py_utils.asdict(split_info) + dataset_name: Optional[str] = dataclasses.field( + default=None, metadata={"include_in_asdict_even_if_is_default": True} + ) + + @property + def file_instructions(self): + """Returns the list of dict(filename, take, skip).""" + # `self.dataset_name` is assigned in `SplitDict.add()`. + instructions = make_file_instructions( + name=self.dataset_name, + split_infos=[self], + instruction=str(self.name), + ) + return instructions.file_instructions + + +@dataclass +class SubSplitInfo: + """Wrapper around a sub split info. + This class expose info on the subsplit: + ``` + ds, info = datasets.load_dataset(..., split='train[75%:]', with_info=True) + info.splits['train[75%:]'].num_examples + ``` + """ + + instructions: FileInstructions + + @property + def num_examples(self): + """Returns the number of example in the subsplit.""" + return self.instructions.num_examples + + @property + def file_instructions(self): + """Returns the list of dict(filename, take, skip).""" + return self.instructions.file_instructions + + +class SplitBase(metaclass=abc.ABCMeta): + # pylint: disable=line-too-long + """Abstract base class for Split compositionality. + + See the + [guide on splits](../loading#slice-splits) + for more information. + + There are three parts to the composition: + 1) The splits are composed (defined, merged, split,...) together before + calling the `.as_dataset()` function. This is done with the `__add__`, + `__getitem__`, which return a tree of `SplitBase` (whose leaf + are the `NamedSplit` objects) + + ``` + split = datasets.Split.TRAIN + datasets.Split.TEST.subsplit(datasets.percent[:50]) + ``` + + 2) The `SplitBase` is forwarded to the `.as_dataset()` function + to be resolved into actual read instruction. This is done by the + `.get_read_instruction()` method which takes the real dataset splits + (name, number of shards,...) and parse the tree to return a + `SplitReadInstruction()` object + + ``` + read_instruction = split.get_read_instruction(self.info.splits) + ``` + + 3) The `SplitReadInstruction` is then used in the `tf.data.Dataset` pipeline + to define which files to read and how to skip examples within file. + + """ + + # pylint: enable=line-too-long + + @abc.abstractmethod + def get_read_instruction(self, split_dict): + """Parse the descriptor tree and compile all read instructions together. + + Args: + split_dict: `dict`, The `dict[split_name, SplitInfo]` of the dataset + + Returns: + split_read_instruction: `SplitReadInstruction` + """ + raise NotImplementedError("Abstract method") + + def __eq__(self, other): + """Equality: datasets.Split.TRAIN == 'train'.""" + if isinstance(other, (NamedSplit, str)): + return False + raise NotImplementedError("Equality is not implemented between merged/sub splits.") + + def __ne__(self, other): + """InEquality: datasets.Split.TRAIN != 'test'.""" + return not self.__eq__(other) + + def __add__(self, other): + """Merging: datasets.Split.TRAIN + datasets.Split.TEST.""" + return _SplitMerged(self, other) + + def subsplit(self, arg=None, k=None, percent=None, weighted=None): # pylint: disable=redefined-outer-name + """Divides this split into subsplits. + + There are 3 ways to define subsplits, which correspond to the 3 + arguments `k` (get `k` even subsplits), `percent` (get a slice of the + dataset with `datasets.percent`), and `weighted` (get subsplits with proportions + specified by `weighted`). + + Example:: + + ``` + # 50% train, 50% test + train, test = split.subsplit(k=2) + # 50% train, 25% test, 25% validation + train, test, validation = split.subsplit(weighted=[2, 1, 1]) + # Extract last 20% + subsplit = split.subsplit(datasets.percent[-20:]) + ``` + + Warning: k and weighted will be converted into percent which mean that + values below the percent will be rounded up or down. The final split may be + bigger to deal with remainders. For instance: + + ``` + train, test, valid = split.subsplit(k=3) # 33%, 33%, 34% + s1, s2, s3, s4 = split.subsplit(weighted=[2, 2, 1, 1]) # 33%, 33%, 16%, 18% + ``` + + Args: + arg: If no kwargs are given, `arg` will be interpreted as one of + `k`, `percent`, or `weighted` depending on the type. + For example: + ``` + split.subsplit(10) # Equivalent to split.subsplit(k=10) + split.subsplit(datasets.percent[:-20]) # percent=datasets.percent[:-20] + split.subsplit([1, 1, 2]) # weighted=[1, 1, 2] + ``` + k: `int` If set, subdivide the split into `k` equal parts. + percent: `datasets.percent slice`, return a single subsplit corresponding to + a slice of the original split. For example: + `split.subsplit(datasets.percent[-20:]) # Last 20% of the dataset`. + weighted: `list[int]`, return a list of subsplits whose proportions match + the normalized sum of the list. For example: + `split.subsplit(weighted=[1, 1, 2]) # 25%, 25%, 50%`. + + Returns: + A subsplit or list of subsplits extracted from this split object. + """ + # Note that the percent kwargs redefine the outer name datasets.percent. This + # is done for consistency (.subsplit(percent=datasets.percent[:40])) + if sum(bool(x) for x in (arg, k, percent, weighted)) != 1: + raise ValueError("Only one argument of subsplit should be set.") + + # Auto deduce k + if isinstance(arg, int): + k = arg + elif isinstance(arg, slice): + percent = arg + elif isinstance(arg, list): + weighted = arg + + if not (k or percent or weighted): + raise ValueError( + f"Invalid split argument {arg}. Only list, slice and int supported. " + "One of k, weighted or percent should be set to a non empty value." + ) + + def assert_slices_coverage(slices): + # Ensure that the expended slices cover all percents. + assert sum((list(range(*s.indices(100))) for s in slices), []) == list(range(100)) + + if k: + if not 0 < k <= 100: + raise ValueError(f"Subsplit k should be between 0 and 100, got {k}") + shift = 100 // k + slices = [slice(i * shift, (i + 1) * shift) for i in range(k)] + # Round up last element to ensure all elements are taken + slices[-1] = slice(slices[-1].start, 100) + # Internal check to ensure full coverage + assert_slices_coverage(slices) + return tuple(_SubSplit(self, s) for s in slices) + elif percent: + return _SubSplit(self, percent) + elif weighted: + # Normalize the weighted sum + total = sum(weighted) + weighted = [100 * x // total for x in weighted] + # Create the slice for each of the elements + start = 0 + stop = 0 + slices = [] + for v in weighted: + stop += v + slices.append(slice(start, stop)) + start = stop + # Round up last element to ensure all elements are taken + slices[-1] = slice(slices[-1].start, 100) + # Internal check to ensure full coverage + assert_slices_coverage(slices) + return tuple(_SubSplit(self, s) for s in slices) + else: + # Should not be possible + raise ValueError("Could not determine the split") + + +# 2 requirements: +# 1. datasets.percent be sliceable +# 2. datasets.percent be documented +# +# Instances are not documented, so we want datasets.percent to be a class, but to +# have it be sliceable, we need this metaclass. +class PercentSliceMeta(type): + def __getitem__(cls, slice_value): + if not isinstance(slice_value, slice): + raise ValueError(f"datasets.percent should only be called with slice, not {slice_value}") + return slice_value + + +class PercentSlice(metaclass=PercentSliceMeta): + # pylint: disable=line-too-long + """Syntactic sugar for defining slice subsplits: `datasets.percent[75:-5]`. + + See the + [guide on splits](../loading#slice-splits) + for more information. + """ + + # pylint: enable=line-too-long + pass + + +percent = PercentSlice # pylint: disable=invalid-name + + +class _SplitMerged(SplitBase): + """Represent two split descriptors merged together.""" + + def __init__(self, split1, split2): + self._split1 = split1 + self._split2 = split2 + + def get_read_instruction(self, split_dict): + read_instruction1 = self._split1.get_read_instruction(split_dict) + read_instruction2 = self._split2.get_read_instruction(split_dict) + return read_instruction1 + read_instruction2 + + def __repr__(self): + return f"({repr(self._split1)} + {repr(self._split2)})" + + +class _SubSplit(SplitBase): + """Represent a sub split of a split descriptor.""" + + def __init__(self, split, slice_value): + self._split = split + self._slice_value = slice_value + + def get_read_instruction(self, split_dict): + return self._split.get_read_instruction(split_dict)[self._slice_value] + + def __repr__(self): + slice_str = "{start}:{stop}" + if self._slice_value.step is not None: + slice_str += ":{step}" + slice_str = slice_str.format( + start="" if self._slice_value.start is None else self._slice_value.start, + stop="" if self._slice_value.stop is None else self._slice_value.stop, + step=self._slice_value.step, + ) + return f"{repr(self._split)}(datasets.percent[{slice_str}])" + + +class NamedSplit(SplitBase): + """Descriptor corresponding to a named split (train, test, ...). + + Example: + Each descriptor can be composed with other using addition or slice: + + ```py + split = datasets.Split.TRAIN.subsplit(datasets.percent[0:25]) + datasets.Split.TEST + ``` + + The resulting split will correspond to 25% of the train split merged with + 100% of the test split. + + A split cannot be added twice, so the following will fail: + + ```py + split = ( + datasets.Split.TRAIN.subsplit(datasets.percent[:25]) + + datasets.Split.TRAIN.subsplit(datasets.percent[75:]) + ) # Error + split = datasets.Split.TEST + datasets.Split.ALL # Error + ``` + + The slices can be applied only one time. So the following are valid: + + ```py + split = ( + datasets.Split.TRAIN.subsplit(datasets.percent[:25]) + + datasets.Split.TEST.subsplit(datasets.percent[:50]) + ) + split = (datasets.Split.TRAIN + datasets.Split.TEST).subsplit(datasets.percent[:50]) + ``` + + But this is not valid: + + ```py + train = datasets.Split.TRAIN + test = datasets.Split.TEST + split = train.subsplit(datasets.percent[:25]).subsplit(datasets.percent[:25]) + split = (train.subsplit(datasets.percent[:25]) + test).subsplit(datasets.percent[:50]) + ``` + """ + + def __init__(self, name): + self._name = name + split_names_from_instruction = [split_instruction.split("[")[0] for split_instruction in name.split("+")] + for split_name in split_names_from_instruction: + if not re.match(_split_re, split_name): + raise ValueError(f"Split name should match '{_split_re}' but got '{split_name}'.") + + def __str__(self): + return self._name + + def __repr__(self): + return f"NamedSplit({self._name!r})" + + def __eq__(self, other): + """Equality: datasets.Split.TRAIN == 'train'.""" + if isinstance(other, NamedSplit): + return self._name == other._name # pylint: disable=protected-access + elif isinstance(other, SplitBase): + return False + elif isinstance(other, str): # Other should be string + return self._name == other + else: + return False + + def __lt__(self, other): + return self._name < other._name # pylint: disable=protected-access + + def __hash__(self): + return hash(self._name) + + def get_read_instruction(self, split_dict): + return SplitReadInstruction(split_dict[self._name]) + + +class NamedSplitAll(NamedSplit): + """Split corresponding to the union of all defined dataset splits.""" + + def __init__(self): + super().__init__("all") + + def __repr__(self): + return "NamedSplitAll()" + + def get_read_instruction(self, split_dict): + # Merge all dataset split together + read_instructions = [SplitReadInstruction(s) for s in split_dict.values()] + return sum(read_instructions, SplitReadInstruction()) + + +class Split: + # pylint: disable=line-too-long + """`Enum` for dataset splits. + + Datasets are typically split into different subsets to be used at various + stages of training and evaluation. + + - `TRAIN`: the training data. + - `VALIDATION`: the validation data. If present, this is typically used as + evaluation data while iterating on a model (e.g. changing hyperparameters, + model architecture, etc.). + - `TEST`: the testing data. This is the data to report metrics on. Typically + you do not want to use this during model iteration as you may overfit to it. + - `ALL`: the union of all defined dataset splits. + + All splits, including compositions inherit from `datasets.SplitBase`. + + See the [guide](../load_hub#splits) on splits for more information. + + Example: + + ```py + >>> datasets.SplitGenerator( + ... name=datasets.Split.TRAIN, + ... gen_kwargs={"split_key": "train", "files": dl_manager.download_and extract(url)}, + ... ), + ... datasets.SplitGenerator( + ... name=datasets.Split.VALIDATION, + ... gen_kwargs={"split_key": "validation", "files": dl_manager.download_and extract(url)}, + ... ), + ... datasets.SplitGenerator( + ... name=datasets.Split.TEST, + ... gen_kwargs={"split_key": "test", "files": dl_manager.download_and extract(url)}, + ... ) + ``` + """ + + # pylint: enable=line-too-long + TRAIN = NamedSplit("train") + TEST = NamedSplit("test") + VALIDATION = NamedSplit("validation") + ALL = NamedSplitAll() + + def __new__(cls, name): + """Create a custom split with datasets.Split('custom_name').""" + return NamedSplitAll() if name == "all" else NamedSplit(name) + + +# Similar to SplitInfo, but contain an additional slice info +SlicedSplitInfo = collections.namedtuple( + "SlicedSplitInfo", + [ + "split_info", + "slice_value", + ], +) # noqa: E231 + + +class SplitReadInstruction: + """Object containing the reading instruction for the dataset. + + Similarly to `SplitDescriptor` nodes, this object can be composed with itself, + but the resolution happens instantaneously, instead of keeping track of the + tree, such as all instructions are compiled and flattened in a single + SplitReadInstruction object containing the list of files and slice to use. + + Once resolved, the instructions can be accessed with: + + ``` + read_instructions.get_list_sliced_split_info() # List of splits to use + ``` + + """ + + def __init__(self, split_info=None): + self._splits = NonMutableDict(error_msg="Overlap between splits. Split {key} has been added with itself.") + + if split_info: + self.add(SlicedSplitInfo(split_info=split_info, slice_value=None)) + + def add(self, sliced_split): + """Add a SlicedSplitInfo the read instructions.""" + # TODO(epot): Check that the number of examples per shard % 100 == 0 + # Otherwise the slices value may be unbalanced and not exactly reflect the + # requested slice. + self._splits[sliced_split.split_info.name] = sliced_split + + def __add__(self, other): + """Merging split together.""" + # Will raise error if a split has already be added (NonMutableDict) + # TODO(epot): If a split is already added but there is no overlap between + # the slices, should merge the slices (ex: [:10] + [80:]) + split_instruction = SplitReadInstruction() + split_instruction._splits.update(self._splits) # pylint: disable=protected-access + split_instruction._splits.update(other._splits) # pylint: disable=protected-access + return split_instruction + + def __getitem__(self, slice_value): + """Sub-splits.""" + # Will raise an error if a split has already been sliced + split_instruction = SplitReadInstruction() + for v in self._splits.values(): + if v.slice_value is not None: + raise ValueError(f"Trying to slice Split {v.split_info.name} which has already been sliced") + v = v._asdict() + v["slice_value"] = slice_value + split_instruction.add(SlicedSplitInfo(**v)) + return split_instruction + + def get_list_sliced_split_info(self): + return list(self._splits.values()) + + +class SplitDict(dict): + """Split info object.""" + + def __init__(self, *args, dataset_name=None, **kwargs): + super().__init__(*args, **kwargs) + self.dataset_name = dataset_name + + def __getitem__(self, key: Union[SplitBase, str]): + # 1st case: The key exists: `info.splits['train']` + if str(key) in self: + return super().__getitem__(str(key)) + # 2nd case: Uses instructions: `info.splits['train[50%]']` + else: + instructions = make_file_instructions( + name=self.dataset_name, + split_infos=self.values(), + instruction=key, + ) + return SubSplitInfo(instructions) + + def __setitem__(self, key: Union[SplitBase, str], value: SplitInfo): + if key != value.name: + raise ValueError(f"Cannot add elem. (key mismatch: '{key}' != '{value.name}')") + super().__setitem__(key, value) + + def add(self, split_info: SplitInfo): + """Add the split info.""" + if split_info.name in self: + raise ValueError(f"Split {split_info.name} already present") + split_info.dataset_name = self.dataset_name + super().__setitem__(split_info.name, split_info) + + @property + def total_num_examples(self): + """Return the total number of examples.""" + return sum(s.num_examples for s in self.values()) + + @classmethod + def from_split_dict(cls, split_infos: Union[list, dict], dataset_name: Optional[str] = None): + """Returns a new SplitDict initialized from a Dict or List of `split_infos`.""" + if isinstance(split_infos, dict): + split_infos = list(split_infos.values()) + + if dataset_name is None: + dataset_name = split_infos[0].get("dataset_name") if split_infos else None + + split_dict = cls(dataset_name=dataset_name) + + for split_info in split_infos: + if isinstance(split_info, dict): + split_info = SplitInfo(**split_info) + split_dict.add(split_info) + + return split_dict + + def to_split_dict(self): + """Returns a list of SplitInfo protos that we have.""" + out = [] + for split_name, split_info in self.items(): + split_info = copy.deepcopy(split_info) + split_info.name = split_name + out.append(split_info) + return out + + def copy(self): + return SplitDict.from_split_dict(self.to_split_dict(), self.dataset_name) + + def _to_yaml_list(self) -> list: + out = [asdict(s) for s in self.to_split_dict()] + # we don't need the shard lengths in YAML, since it depends on max_shard_size and num_proc + for split_info_dict in out: + split_info_dict.pop("shard_lengths", None) + # we don't need the dataset_name attribute that is deprecated + for split_info_dict in out: + split_info_dict.pop("dataset_name", None) + return out + + @classmethod + def _from_yaml_list(cls, yaml_data: list) -> "SplitDict": + return cls.from_split_dict(yaml_data) + + +@dataclass +class SplitGenerator: + """Defines the split information for the generator. + + This should be used as returned value of + `GeneratorBasedBuilder._split_generators`. + See `GeneratorBasedBuilder._split_generators` for more info and example + of usage. + + Args: + name (`str`): + Name of the `Split` for which the generator will + create the examples. + **gen_kwargs (additional keyword arguments): + Keyword arguments to forward to the `DatasetBuilder._generate_examples` method + of the builder. + + Example: + + ```py + >>> datasets.SplitGenerator( + ... name=datasets.Split.TRAIN, + ... gen_kwargs={"split_key": "train", "files": dl_manager.download_and_extract(url)}, + ... ) + ``` + """ + + name: str + gen_kwargs: dict = dataclasses.field(default_factory=dict) + split_info: SplitInfo = dataclasses.field(init=False) + + def __post_init__(self): + self.name = str(self.name) # Make sure we convert NamedSplits in strings + NamedSplit(self.name) # check that it's a valid split name + self.split_info = SplitInfo(name=self.name) diff --git a/lib/python3.12/site-packages/datasets/streaming.py b/lib/python3.12/site-packages/datasets/streaming.py new file mode 100644 index 0000000000000000000000000000000000000000..81091417af70194d1a329b50a30fd5f9931b0ac9 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/streaming.py @@ -0,0 +1,142 @@ +import importlib +import inspect +from functools import wraps +from typing import TYPE_CHECKING, Optional + +from .download.download_config import DownloadConfig +from .utils.file_utils import ( + xbasename, + xdirname, + xet_parse, + xexists, + xgetsize, + xglob, + xgzip_open, + xisdir, + xisfile, + xjoin, + xlistdir, + xnumpy_load, + xopen, + xpandas_read_csv, + xpandas_read_excel, + xPath, + xpyarrow_parquet_read_table, + xrelpath, + xsio_loadmat, + xsplit, + xsplitext, + xwalk, + xxml_dom_minidom_parse, +) +from .utils.logging import get_logger +from .utils.patching import patch_submodule +from .utils.py_utils import get_imports, lock_importable_file + + +logger = get_logger(__name__) + + +if TYPE_CHECKING: + from .builder import DatasetBuilder + + +def extend_module_for_streaming(module_path, download_config: Optional[DownloadConfig] = None): + """Extend the module to support streaming. + + We patch some functions in the module to use `fsspec` to support data streaming: + - We use `fsspec.open` to open and read remote files. We patch the module function: + - `open` + - We use the "::" hop separator to join paths and navigate remote compressed/archive files. We patch the module + functions: + - `os.path.join` + - `pathlib.Path.joinpath` and `pathlib.Path.__truediv__` (called when using the "/" operator) + + The patched functions are replaced with custom functions defined to work with the + :class:`~download.streaming_download_manager.StreamingDownloadManager`. + + Args: + module_path: Path to the module to be extended. + download_config: Mainly use `token` or `storage_options` to support different platforms and auth types. + """ + + module = importlib.import_module(module_path) + + # TODO(QL): always update the module to add subsequent new authentication without removing old ones + if hasattr(module, "_patched_for_streaming") and module._patched_for_streaming: + if isinstance(module._patched_for_streaming, DownloadConfig): + module._patched_for_streaming.token = download_config.token + module._patched_for_streaming.storage_options = download_config.storage_options + return + + def wrap_auth(function): + @wraps(function) + def wrapper(*args, **kwargs): + return function(*args, download_config=download_config, **kwargs) + + wrapper._decorator_name_ = "wrap_auth" + return wrapper + + # open files in a streaming fashion + patch_submodule(module, "open", wrap_auth(xopen)).start() + patch_submodule(module, "os.listdir", wrap_auth(xlistdir)).start() + patch_submodule(module, "os.walk", wrap_auth(xwalk)).start() + patch_submodule(module, "glob.glob", wrap_auth(xglob)).start() + # allow to navigate in remote zip files + patch_submodule(module, "os.path.join", xjoin).start() + patch_submodule(module, "os.path.dirname", xdirname).start() + patch_submodule(module, "os.path.basename", xbasename).start() + patch_submodule(module, "os.path.relpath", xrelpath).start() + patch_submodule(module, "os.path.split", xsplit).start() + patch_submodule(module, "os.path.splitext", xsplitext).start() + # allow checks on paths + patch_submodule(module, "os.path.exists", wrap_auth(xexists)).start() + patch_submodule(module, "os.path.isdir", wrap_auth(xisdir)).start() + patch_submodule(module, "os.path.isfile", wrap_auth(xisfile)).start() + patch_submodule(module, "os.path.getsize", wrap_auth(xgetsize)).start() + patch_submodule(module, "pathlib.Path", xPath).start() + # file readers + patch_submodule(module, "gzip.open", wrap_auth(xgzip_open)).start() + patch_submodule(module, "numpy.load", wrap_auth(xnumpy_load)).start() + patch_submodule(module, "pandas.read_csv", wrap_auth(xpandas_read_csv), attrs=["__version__"]).start() + patch_submodule(module, "pandas.read_excel", wrap_auth(xpandas_read_excel), attrs=["__version__"]).start() + patch_submodule(module, "scipy.io.loadmat", wrap_auth(xsio_loadmat), attrs=["__version__"]).start() + patch_submodule(module, "xml.etree.ElementTree.parse", wrap_auth(xet_parse)).start() + patch_submodule(module, "xml.dom.minidom.parse", wrap_auth(xxml_dom_minidom_parse)).start() + # pyarrow: do not patch pyarrow attribute in packaged modules + if not module.__name__.startswith("datasets.packaged_modules."): + patch_submodule(module, "pyarrow.parquet.read_table", wrap_auth(xpyarrow_parquet_read_table)).start() + module._patched_for_streaming = download_config + + +def extend_dataset_builder_for_streaming(builder: "DatasetBuilder"): + """Extend the dataset builder module and the modules imported by it to support streaming. + + Args: + builder (:class:`DatasetBuilder`): Dataset builder instance. + """ + # this extends the open and os.path.join functions for data streaming + download_config = DownloadConfig(storage_options=builder.storage_options, token=builder.token) + extend_module_for_streaming(builder.__module__, download_config=download_config) + # if needed, we also have to extend additional internal imports (like wmt14 -> wmt_utils) + if not builder.__module__.startswith("datasets."): # check that it's not a packaged builder like csv + importable_file = inspect.getfile(builder.__class__) + with lock_importable_file(importable_file): + for imports in get_imports(importable_file): + if imports[0] == "internal": + internal_import_name = imports[1] + internal_module_name = ".".join(builder.__module__.split(".")[:-1] + [internal_import_name]) + extend_module_for_streaming(internal_module_name, download_config=download_config) + + # builders can inherit from other builders that might use streaming functionality + # (for example, ImageFolder and AudioFolder inherit from FolderBuilder which implements examples generation) + # but these parents builders are not patched automatically as they are not instantiated, so we patch them here + from .builder import DatasetBuilder + + parent_builder_modules = [ + cls.__module__ + for cls in type(builder).__mro__[1:] # make sure it's not the same module we've already patched + if issubclass(cls, DatasetBuilder) and cls.__module__ != DatasetBuilder.__module__ + ] # check it's not a standard builder from datasets.builder + for module in parent_builder_modules: + extend_module_for_streaming(module, download_config=download_config) diff --git a/lib/python3.12/site-packages/datasets/table.py b/lib/python3.12/site-packages/datasets/table.py new file mode 100644 index 0000000000000000000000000000000000000000..e57a0c54927ef4e23636fcab67088ad8e2dd7469 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/table.py @@ -0,0 +1,2413 @@ +import copy +import os +from collections.abc import Iterator +from functools import partial +from itertools import groupby +from typing import TYPE_CHECKING, Any, Callable, Optional, TypeVar, Union + +import numpy as np +import pyarrow as pa +import pyarrow.compute as pc +import pyarrow.types + +from .utils.logging import get_logger + + +if TYPE_CHECKING: + from .features.features import Features, FeatureType + + +logger = get_logger(__name__) + + +def inject_arrow_table_documentation(arrow_table_method): + def wrapper(fn): + fn.__doc__ = arrow_table_method.__doc__ + (fn.__doc__ if fn.__doc__ is not None else "") + fn.__doc__ = fn.__doc__.replace("pyarrow.Table", "Table") + if hasattr(arrow_table_method, "__annotations__"): + fn.__annotations__ = arrow_table_method.__annotations__ + return fn + + return wrapper + + +def _in_memory_arrow_table_from_file(filename: str) -> pa.Table: + in_memory_stream = pa.input_stream(filename) + opened_stream = pa.ipc.open_stream(in_memory_stream) + pa_table = opened_stream.read_all() + return pa_table + + +def _in_memory_arrow_table_from_buffer(buffer: pa.Buffer) -> pa.Table: + stream = pa.BufferReader(buffer) + opened_stream = pa.ipc.open_stream(stream) + table = opened_stream.read_all() + return table + + +def _memory_mapped_record_batch_reader_from_file(filename: str) -> pa.RecordBatchStreamReader: + memory_mapped_stream = pa.memory_map(filename) + return pa.ipc.open_stream(memory_mapped_stream) + + +def read_schema_from_file(filename: str) -> pa.Schema: + """ + Infer arrow table schema from file without loading whole file into memory. + Useful especially while having very big files. + """ + with pa.memory_map(filename) as memory_mapped_stream: + schema = pa.ipc.open_stream(memory_mapped_stream).schema + return schema + + +def _memory_mapped_arrow_table_from_file(filename: str) -> pa.Table: + opened_stream = _memory_mapped_record_batch_reader_from_file(filename) + pa_table = opened_stream.read_all() + return pa_table + + +def _deepcopy(x, memo: dict): + """deepcopy a regular class instance""" + cls = x.__class__ + result = cls.__new__(cls) + memo[id(x)] = result + for k, v in x.__dict__.items(): + setattr(result, k, copy.deepcopy(v, memo)) + return result + + +def _interpolation_search(arr: list[int], x: int) -> int: + """ + Return the position i of a sorted array so that arr[i] <= x < arr[i+1] + + Args: + arr (`List[int]`): non-empty sorted list of integers + x (`int`): query + + Returns: + `int`: the position i so that arr[i] <= x < arr[i+1] + + Raises: + `IndexError`: if the array is empty or if the query is outside the array values + """ + i, j = 0, len(arr) - 1 + while i < j and arr[i] <= x < arr[j]: + k = i + ((j - i) * (x - arr[i]) // (arr[j] - arr[i])) + if arr[k] <= x < arr[k + 1]: + return k + elif arr[k] < x: + i, j = k + 1, j + else: + i, j = i, k + raise IndexError(f"Invalid query '{x}' for size {arr[-1] if len(arr) else 'none'}.") + + +class IndexedTableMixin: + def __init__(self, table: pa.Table): + self._schema: pa.Schema = table.schema + self._batches: list[pa.RecordBatch] = [ + recordbatch for recordbatch in table.to_batches() if len(recordbatch) > 0 + ] + self._offsets: np.ndarray = np.cumsum([0] + [len(b) for b in self._batches], dtype=np.int64) + + def fast_gather(self, indices: Union[list[int], np.ndarray]) -> pa.Table: + """ + Create a pa.Table by gathering the records at the records at the specified indices. Should be faster + than pa.concat_tables(table.fast_slice(int(i) % table.num_rows, 1) for i in indices) since NumPy can compute + the binary searches in parallel, highly optimized C + """ + if not len(indices): + raise ValueError("Indices must be non-empty") + batch_indices = np.searchsorted(self._offsets, indices, side="right") - 1 + return pa.Table.from_batches( + [ + self._batches[batch_idx].slice(i - self._offsets[batch_idx], 1) + for batch_idx, i in zip(batch_indices, indices) + ], + schema=self._schema, + ) + + def fast_slice(self, offset=0, length=None) -> pa.Table: + """ + Slice the Table using interpolation search. + The behavior is the same as `pyarrow.Table.slice` but it's significantly faster. + + Interpolation search is used to find the start and end indexes of the batches we want to keep. + The batches to keep are then concatenated to form the sliced Table. + """ + if offset < 0: + raise IndexError("Offset must be non-negative") + elif offset >= self._offsets[-1] or (length is not None and length <= 0): + return pa.Table.from_batches([], schema=self._schema) + i = _interpolation_search(self._offsets, offset) + if length is None or length + offset >= self._offsets[-1]: + batches = self._batches[i:] + batches[0] = batches[0].slice(offset - self._offsets[i]) + else: + j = _interpolation_search(self._offsets, offset + length - 1) + batches = self._batches[i : j + 1] + batches[-1] = batches[-1].slice(0, offset + length - self._offsets[j]) + batches[0] = batches[0].slice(offset - self._offsets[i]) + return pa.Table.from_batches(batches, schema=self._schema) + + +class Table(IndexedTableMixin): + """ + Wraps a pyarrow Table by using composition. + This is the base class for `InMemoryTable`, `MemoryMappedTable` and `ConcatenationTable`. + + It implements all the basic attributes/methods of the pyarrow Table class except + the Table transforms: `slice, filter, flatten, combine_chunks, cast, add_column, + append_column, remove_column, set_column, rename_columns` and `drop`. + + The implementation of these methods differs for the subclasses. + """ + + def __init__(self, table: pa.Table): + super().__init__(table) + self.table = table + + def __deepcopy__(self, memo: dict): + # arrow tables are immutable, so there's no need to copy self.table + # moreover calling deepcopy on a pyarrow table seems to make pa.total_allocated_bytes() decrease for some reason + # by adding it to the memo, self.table won't be copied + memo[id(self.table)] = self.table + # same for the recordbatches used by the index + memo[id(self._batches)] = list(self._batches) + return _deepcopy(self, memo) + + def validate(self, *args, **kwargs): + """ + Perform validation checks. An exception is raised if validation fails. + + By default only cheap validation checks are run. Pass `full=True` + for thorough validation checks (potentially `O(n)`). + + Args: + full (`bool`, defaults to `False`): + If `True`, run expensive checks, otherwise cheap checks only. + + Raises: + `pa.lib.ArrowInvalid`: if validation fails + """ + return self.table.validate(*args, **kwargs) + + def equals(self, *args, **kwargs): + """ + Check if contents of two tables are equal. + + Args: + other ([`~datasets.table.Table`]): + Table to compare against. + check_metadata `bool`, defaults to `False`): + Whether schema metadata equality should be checked as well. + + Returns: + `bool` + """ + args = tuple(arg.table if isinstance(arg, Table) else arg for arg in args) + kwargs = {k: v.table if isinstance(v, Table) else v for k, v in kwargs} + return self.table.equals(*args, **kwargs) + + def to_batches(self, *args, **kwargs): + """ + Convert Table to list of (contiguous) `RecordBatch` objects. + + Args: + max_chunksize (`int`, defaults to `None`): + Maximum size for `RecordBatch` chunks. Individual chunks may be + smaller depending on the chunk layout of individual columns. + + Returns: + `List[pyarrow.RecordBatch]` + """ + return self.table.to_batches(*args, **kwargs) + + def to_pydict(self, *args, **kwargs): + """ + Convert the Table to a `dict` or `OrderedDict`. + + Returns: + `dict` + """ + return self.table.to_pydict(*args, **kwargs) + + def to_pylist(self, *args, **kwargs): + """ + Convert the Table to a list + + Returns: + `list` + """ + return self.table.to_pylist(*args, **kwargs) + + def to_pandas(self, *args, **kwargs): + """ + Convert to a pandas-compatible NumPy array or DataFrame, as appropriate. + + Args: + memory_pool (`MemoryPool`, defaults to `None`): + Arrow MemoryPool to use for allocations. Uses the default memory + pool is not passed. + strings_to_categorical (`bool`, defaults to `False`): + Encode string (UTF8) and binary types to `pandas.Categorical`. + categories (`list`, defaults to `empty`): + List of fields that should be returned as `pandas.Categorical`. Only + applies to table-like data structures. + zero_copy_only (`bool`, defaults to `False`): + Raise an `ArrowException` if this function call would require copying + the underlying data. + integer_object_nulls (`bool`, defaults to `False`): + Cast integers with nulls to objects. + date_as_object (`bool`, defaults to `True`): + Cast dates to objects. If `False`, convert to `datetime64[ns]` dtype. + timestamp_as_object (`bool`, defaults to `False`): + Cast non-nanosecond timestamps (`np.datetime64`) to objects. This is + useful if you have timestamps that don't fit in the normal date + range of nanosecond timestamps (1678 CE-2262 CE). + If `False`, all timestamps are converted to `datetime64[ns]` dtype. + use_threads (`bool`, defaults to `True`): + Whether to parallelize the conversion using multiple threads. + deduplicate_objects (`bool`, defaults to `False`): + Do not create multiple copies Python objects when created, to save + on memory use. Conversion will be slower. + ignore_metadata (`bool`, defaults to `False`): + If `True`, do not use the 'pandas' metadata to reconstruct the + DataFrame index, if present. + safe (`bool`, defaults to `True`): + For certain data types, a cast is needed in order to store the + data in a pandas DataFrame or Series (e.g. timestamps are always + stored as nanoseconds in pandas). This option controls whether it + is a safe cast or not. + split_blocks (`bool`, defaults to `False`): + If `True`, generate one internal "block" for each column when + creating a pandas.DataFrame from a `RecordBatch` or `Table`. While this + can temporarily reduce memory note that various pandas operations + can trigger "consolidation" which may balloon memory use. + self_destruct (`bool`, defaults to `False`): + EXPERIMENTAL: If `True`, attempt to deallocate the originating Arrow + memory while converting the Arrow object to pandas. If you use the + object after calling `to_pandas` with this option it will crash your + program. + types_mapper (`function`, defaults to `None`): + A function mapping a pyarrow DataType to a pandas `ExtensionDtype`. + This can be used to override the default pandas type for conversion + of built-in pyarrow types or in absence of `pandas_metadata` in the + Table schema. The function receives a pyarrow DataType and is + expected to return a pandas `ExtensionDtype` or `None` if the + default conversion should be used for that type. If you have + a dictionary mapping, you can pass `dict.get` as function. + + Returns: + `pandas.Series` or `pandas.DataFrame`: `pandas.Series` or `pandas.DataFrame` depending on type of object + """ + return self.table.to_pandas(*args, **kwargs) + + def to_string(self, *args, **kwargs): + return self.table.to_string(*args, **kwargs) + + def to_reader(self, max_chunksize: Optional[int] = None): + """ + Convert the Table to a RecordBatchReader. + + Note that this method is zero-copy, it merely exposes the same data under a different API. + + Args: + max_chunksize (`int`, defaults to `None`) + Maximum size for RecordBatch chunks. Individual chunks may be smaller depending + on the chunk layout of individual columns. + + Returns: + `pyarrow.RecordBatchReader` + """ + return self.table.to_reader(max_chunksize=max_chunksize) + + def field(self, *args, **kwargs): + """ + Select a schema field by its column name or numeric index. + + Args: + i (`Union[int, str]`): + The index or name of the field to retrieve. + + Returns: + `pyarrow.Field` + """ + return self.table.field(*args, **kwargs) + + def column(self, *args, **kwargs): + """ + Select a column by its column name, or numeric index. + + Args: + i (`Union[int, str]`): + The index or name of the column to retrieve. + + Returns: + `pyarrow.ChunkedArray` + """ + return self.table.column(*args, **kwargs) + + def itercolumns(self, *args, **kwargs): + """ + Iterator over all columns in their numerical order. + + Yields: + `pyarrow.ChunkedArray` + """ + return self.table.itercolumns(*args, **kwargs) + + @property + def schema(self): + """ + Schema of the table and its columns. + + Returns: + `pyarrow.Schema` + """ + return self.table.schema + + @property + def columns(self): + """ + List of all columns in numerical order. + + Returns: + `List[pa.ChunkedArray]` + """ + return self.table.columns + + @property + def num_columns(self): + """ + Number of columns in this table. + + Returns: + int + """ + return self.table.num_columns + + @property + def num_rows(self): + """ + Number of rows in this table. + + Due to the definition of a table, all columns have the same number of + rows. + + Returns: + int + """ + return self.table.num_rows + + @property + def shape(self): + """ + Dimensions of the table: (#rows, #columns). + + Returns: + `(int, int)`: Number of rows and number of columns. + """ + return self.table.shape + + @property + def nbytes(self): + """ + Total number of bytes consumed by the elements of the table. + """ + return self.table.nbytes + + @property + def column_names(self): + """ + Names of the table's columns. + """ + return self.table.column_names + + def __eq__(self, other): + return self.equals(other) + + def __getitem__(self, i): + return self.table[i] + + def __len__(self): + return len(self.table) + + def __repr__(self): + return self.table.__repr__().replace("pyarrow.Table", self.__class__.__name__) + + def __str__(self): + return self.table.__str__().replace("pyarrow.Table", self.__class__.__name__) + + def slice(self, *args, **kwargs): + """ + Compute zero-copy slice of this Table. + + Args: + offset (`int`, defaults to `0`): + Offset from start of table to slice. + length (`int`, defaults to `None`): + Length of slice (default is until end of table starting from + offset). + + Returns: + `datasets.table.Table` + """ + raise NotImplementedError() + + def filter(self, *args, **kwargs): + """ + Select records from a Table. See `pyarrow.compute.filter` for full usage. + """ + raise NotImplementedError() + + def flatten(self, *args, **kwargs): + """ + Flatten this Table. Each column with a struct type is flattened + into one column per struct field. Other columns are left unchanged. + + Args: + memory_pool (`MemoryPool`, defaults to `None`): + For memory allocations, if required, otherwise use default pool. + + Returns: + `datasets.table.Table` + """ + raise NotImplementedError() + + def combine_chunks(self, *args, **kwargs): + """ + Make a new table by combining the chunks this table has. + + All the underlying chunks in the `ChunkedArray` of each column are + concatenated into zero or one chunk. + + Args: + memory_pool (`MemoryPool`, defaults to `None`): + For memory allocations, if required, otherwise use default pool. + + Returns: + `datasets.table.Table` + """ + raise NotImplementedError() + + def cast(self, *args, **kwargs): + """ + Cast table values to another schema. + + Args: + target_schema (`Schema`): + Schema to cast to, the names and order of fields must match. + safe (`bool`, defaults to `True`): + Check for overflows or other unsafe conversions. + + Returns: + `datasets.table.Table` + """ + raise NotImplementedError() + + def replace_schema_metadata(self, *args, **kwargs): + """ + EXPERIMENTAL: Create shallow copy of table by replacing schema + key-value metadata with the indicated new metadata (which may be None, + which deletes any existing metadata + + Args: + metadata (`dict`, defaults to `None`): + + Returns: + `datasets.table.Table`: shallow_copy + """ + raise NotImplementedError() + + def add_column(self, *args, **kwargs): + """ + Add column to Table at position. + + A new table is returned with the column added, the original table + object is left unchanged. + + Args: + i (`int`): + Index to place the column at. + field_ (`Union[str, pyarrow.Field]`): + If a string is passed then the type is deduced from the column + data. + column (`Union[pyarrow.Array, List[pyarrow.Array]]`): + Column data. + + Returns: + `datasets.table.Table`: New table with the passed column added. + """ + raise NotImplementedError() + + def append_column(self, *args, **kwargs): + """ + Append column at end of columns. + + Args: + field_ (`Union[str, pyarrow.Field]`): + If a string is passed then the type is deduced from the column + data. + column (`Union[pyarrow.Array, List[pyarrow.Array]]`): + Column data. + + Returns: + `datasets.table.Table`: New table with the passed column added. + """ + raise NotImplementedError() + + def remove_column(self, *args, **kwargs): + """ + Create new Table with the indicated column removed. + + Args: + i (`int`): + Index of column to remove. + + Returns: + `datasets.table.Table`: New table without the column. + """ + raise NotImplementedError() + + def set_column(self, *args, **kwargs): + """ + Replace column in Table at position. + + Args: + i (`int`): + Index to place the column at. + field_ (`Union[str, pyarrow.Field]`): + If a string is passed then the type is deduced from the column + data. + column (`Union[pyarrow.Array, List[pyarrow.Array]]`): + Column data. + + Returns: + `datasets.table.Table`: New table with the passed column set. + """ + raise NotImplementedError() + + def rename_columns(self, *args, **kwargs): + """ + Create new table with columns renamed to provided names. + """ + raise NotImplementedError() + + def drop(self, *args, **kwargs): + """ + Drop one or more columns and return a new table. + + Args: + columns (`List[str]`): + List of field names referencing existing columns. + + Raises: + `KeyError` : if any of the passed columns name are not existing. + + Returns: + `datasets.table.Table`: New table without the columns. + """ + raise NotImplementedError() + + def select(self, *args, **kwargs): + """ + Select columns of the table. + + Returns a new table with the specified columns, and metadata preserved. + + Args: + columns (:obj:`Union[List[str], List[int]]`): + The column names or integer indices to select. + + Returns: + `datasets.table.Table`: table with only a subset of the columns + """ + raise NotImplementedError() + + +class TableBlock(Table): + """ + `TableBlock` is the allowed class inside a `ConcanetationTable`. + Only `MemoryMappedTable` and `InMemoryTable` are `TableBlock`. + This is because we don't want a `ConcanetationTable` made out of other `ConcanetationTables`. + """ + + pass + + +class InMemoryTable(TableBlock): + """ + The table is said in-memory when it is loaded into the user's RAM. + + Pickling it does copy all the data using memory. + Its implementation is simple and uses the underlying pyarrow Table methods directly. + + This is different from the `MemoryMapped` table, for which pickling doesn't copy all the + data in memory. For a `MemoryMapped`, unpickling instead reloads the table from the disk. + + `InMemoryTable` must be used when data fit in memory, while `MemoryMapped` are reserved for + data bigger than memory or when you want the memory footprint of your application to + stay low. + """ + + @classmethod + def from_file(cls, filename: str): + table = _in_memory_arrow_table_from_file(filename) + return cls(table) + + @classmethod + def from_buffer(cls, buffer: pa.Buffer): + table = _in_memory_arrow_table_from_buffer(buffer) + return cls(table) + + @classmethod + def from_pandas(cls, *args, **kwargs): + """ + Convert pandas.DataFrame to an Arrow Table. + + The column types in the resulting Arrow Table are inferred from the + dtypes of the pandas.Series in the DataFrame. In the case of non-object + Series, the NumPy dtype is translated to its Arrow equivalent. In the + case of `object`, we need to guess the datatype by looking at the + Python objects in this Series. + + Be aware that Series of the `object` dtype don't carry enough + information to always lead to a meaningful Arrow type. In the case that + we cannot infer a type, e.g. because the DataFrame is of length 0 or + the Series only contains `None/nan` objects, the type is set to + null. This behavior can be avoided by constructing an explicit schema + and passing it to this function. + + Args: + df (`pandas.DataFrame`): + schema (`pyarrow.Schema`, *optional*): + The expected schema of the Arrow Table. This can be used to + indicate the type of columns if we cannot infer it automatically. + If passed, the output will have exactly this schema. Columns + specified in the schema that are not found in the DataFrame columns + or its index will raise an error. Additional columns or index + levels in the DataFrame which are not specified in the schema will + be ignored. + preserve_index (`bool`, *optional*): + Whether to store the index as an additional column in the resulting + `Table`. The default of None will store the index as a column, + except for RangeIndex which is stored as metadata only. Use + `preserve_index=True` to force it to be stored as a column. + nthreads (`int`, defaults to `None` (may use up to system CPU count threads)) + If greater than 1, convert columns to Arrow in parallel using + indicated number of threads. + columns (`List[str]`, *optional*): + List of column to be converted. If `None`, use all columns. + safe (`bool`, defaults to `True`): + Check for overflows or other unsafe conversions, + + Returns: + `datasets.table.Table`: + + Examples: + ```python + >>> import pandas as pd + >>> import pyarrow as pa + >>> df = pd.DataFrame({ + ... 'int': [1, 2], + ... 'str': ['a', 'b'] + ... }) + >>> pa.Table.from_pandas(df) + + ``` + """ + return cls(pa.Table.from_pandas(*args, **kwargs)) + + @classmethod + def from_arrays(cls, *args, **kwargs): + """ + Construct a Table from Arrow arrays. + + Args: + arrays (`List[Union[pyarrow.Array, pyarrow.ChunkedArray]]`): + Equal-length arrays that should form the table. + names (`List[str]`, *optional*): + Names for the table columns. If not passed, schema must be passed. + schema (`Schema`, defaults to `None`): + Schema for the created table. If not passed, names must be passed. + metadata (`Union[dict, Mapping]`, defaults to `None`): + Optional metadata for the schema (if inferred). + + Returns: + `datasets.table.Table` + """ + return cls(pa.Table.from_arrays(*args, **kwargs)) + + @classmethod + def from_pydict(cls, *args, **kwargs): + """ + Construct a Table from Arrow arrays or columns. + + Args: + mapping (`Union[dict, Mapping]`): + A mapping of strings to Arrays or Python lists. + schema (`Schema`, defaults to `None`): + If not passed, will be inferred from the Mapping values + metadata (`Union[dict, Mapping]`, defaults to `None`): + Optional metadata for the schema (if inferred). + + Returns: + `datasets.table.Table` + """ + return cls(pa.Table.from_pydict(*args, **kwargs)) + + @classmethod + def from_pylist(cls, mapping, *args, **kwargs): + """ + Construct a Table from list of rows / dictionaries. + + Args: + mapping (`List[dict]`): + A mapping of strings to row values. + schema (`Schema`, defaults to `None`): + If not passed, will be inferred from the Mapping values + metadata (`Union[dict, Mapping]`, defaults to `None`): + Optional metadata for the schema (if inferred). + + Returns: + `datasets.table.Table` + """ + return cls(pa.Table.from_pylist(mapping, *args, **kwargs)) + + @classmethod + def from_batches(cls, *args, **kwargs): + """ + Construct a Table from a sequence or iterator of Arrow `RecordBatches`. + + Args: + batches (`Union[Sequence[pyarrow.RecordBatch], Iterator[pyarrow.RecordBatch]]`): + Sequence of `RecordBatch` to be converted, all schemas must be equal. + schema (`Schema`, defaults to `None`): + If not passed, will be inferred from the first `RecordBatch`. + + Returns: + `datasets.table.Table`: + """ + return cls(pa.Table.from_batches(*args, **kwargs)) + + def slice(self, offset=0, length=None): + """ + Compute zero-copy slice of this Table. + + Args: + offset (`int`, defaults to `0`): + Offset from start of table to slice. + length (`int`, defaults to `None`): + Length of slice (default is until end of table starting from + offset). + + Returns: + `datasets.table.Table` + """ + # Use fast slicing here + return InMemoryTable(self.fast_slice(offset=offset, length=length)) + + def filter(self, *args, **kwargs): + """ + Select records from a Table. See `pyarrow.compute.filter` for full usage. + """ + return InMemoryTable(self.table.filter(*args, **kwargs)) + + def flatten(self, *args, **kwargs): + """ + Flatten this Table. Each column with a struct type is flattened + into one column per struct field. Other columns are left unchanged. + + Args: + memory_pool (`MemoryPool`, defaults to `None`): + For memory allocations, if required, otherwise use default pool. + + Returns: + `datasets.table.Table` + """ + return InMemoryTable(table_flatten(self.table, *args, **kwargs)) + + def combine_chunks(self, *args, **kwargs): + """ + Make a new table by combining the chunks this table has. + + All the underlying chunks in the `ChunkedArray` of each column are + concatenated into zero or one chunk. + + Args: + memory_pool (`MemoryPool`, defaults to `None`): + For memory allocations, if required, otherwise use default pool. + + Returns: + `datasets.table.Table` + """ + return InMemoryTable(self.table.combine_chunks(*args, **kwargs)) + + def cast(self, *args, **kwargs): + """ + Cast table values to another schema. + + Args: + target_schema (`Schema`): + Schema to cast to, the names and order of fields must match. + safe (`bool`, defaults to `True`): + Check for overflows or other unsafe conversions. + + Returns: + `datasets.table.Table` + """ + return InMemoryTable(table_cast(self.table, *args, **kwargs)) + + def replace_schema_metadata(self, *args, **kwargs): + """ + EXPERIMENTAL: Create shallow copy of table by replacing schema + key-value metadata with the indicated new metadata (which may be `None`, + which deletes any existing metadata). + + Args: + metadata (`dict`, defaults to `None`): + + Returns: + `datasets.table.Table`: shallow_copy + """ + return InMemoryTable(self.table.replace_schema_metadata(*args, **kwargs)) + + def add_column(self, *args, **kwargs): + """ + Add column to Table at position. + + A new table is returned with the column added, the original table + object is left unchanged. + + Args: + i (`int`): + Index to place the column at. + field_ (`Union[str, pyarrow.Field]`): + If a string is passed then the type is deduced from the column + data. + column (`Union[pyarrow.Array, List[pyarrow.Array]]`): + Column data. + + Returns: + `datasets.table.Table`: New table with the passed column added. + """ + return InMemoryTable(self.table.add_column(*args, **kwargs)) + + def append_column(self, *args, **kwargs): + """ + Append column at end of columns. + + Args: + field_ (`Union[str, pyarrow.Field]`): + If a string is passed then the type is deduced from the column + data. + column (`Union[pyarrow.Array, List[pyarrow.Array]]`): + Column data. + + Returns: + `datasets.table.Table`: + New table with the passed column added. + """ + return InMemoryTable(self.table.append_column(*args, **kwargs)) + + def remove_column(self, *args, **kwargs): + """ + Create new Table with the indicated column removed. + + Args: + i (`int`): + Index of column to remove. + + Returns: + `datasets.table.Table`: + New table without the column. + """ + return InMemoryTable(self.table.remove_column(*args, **kwargs)) + + def set_column(self, *args, **kwargs): + """ + Replace column in Table at position. + + Args: + i (`int`): + Index to place the column at. + field_ (`Union[str, pyarrow.Field]`): + If a string is passed then the type is deduced from the column + data. + column (`Union[pyarrow.Array, List[pyarrow.Array]]`): + Column data. + + Returns: + `datasets.table.Table`: + New table with the passed column set. + """ + return InMemoryTable(self.table.set_column(*args, **kwargs)) + + def rename_columns(self, *args, **kwargs): + """ + Create new table with columns renamed to provided names. + """ + return InMemoryTable(self.table.rename_columns(*args, **kwargs)) + + def drop(self, *args, **kwargs): + """ + Drop one or more columns and return a new table. + + Args: + columns (`List[str]`): + List of field names referencing existing columns. + + Raises: + `KeyError` : if any of the passed columns name are not existing. + + Returns: + `datasets.table.Table`: + New table without the columns. + """ + return InMemoryTable(self.table.drop(*args, **kwargs)) + + def select(self, *args, **kwargs): + """ + Select columns of the table. + + Returns a new table with the specified columns, and metadata preserved. + + Args: + columns (:obj:`Union[List[str], List[int]]`): + The column names or integer indices to select. + + Returns: + :class:`datasets.table.Table`: New table with the specified columns, and metadata preserved. + """ + return InMemoryTable(self.table.select(*args, **kwargs)) + + +# The MemoryMappedTable needs replays to properly reload tables from the disk +Replay = tuple[str, tuple, dict] + + +class MemoryMappedTable(TableBlock): + """ + The table is said memory mapped when it doesn't use the user's RAM but loads the data + from the disk instead. + + Pickling it doesn't copy the data into memory. + Instead, only the path to the memory mapped arrow file is pickled, as well as the list + of transforms to "replay" when reloading the table from the disk. + + Its implementation requires to store an history of all the transforms that were applied + to the underlying pyarrow Table, so that they can be "replayed" when reloading the Table + from the disk. + + This is different from the `InMemoryTable` table, for which pickling does copy all the + data in memory. + + `InMemoryTable` must be used when data fit in memory, while `MemoryMapped` are reserved for + data bigger than memory or when you want the memory footprint of your application to + stay low. + """ + + def __init__(self, table: pa.Table, path: str, replays: Optional[list[Replay]] = None): + super().__init__(table) + self.path = os.path.abspath(path) + self.replays: list[Replay] = replays if replays is not None else [] + + @classmethod + def from_file(cls, filename: str, replays=None): + table = _memory_mapped_arrow_table_from_file(filename) + table = cls._apply_replays(table, replays) + return cls(table, filename, replays) + + def __getstate__(self): + return {"path": self.path, "replays": self.replays} + + def __setstate__(self, state): + path = state["path"] + replays = state["replays"] + table = _memory_mapped_arrow_table_from_file(path) + table = self._apply_replays(table, replays) + MemoryMappedTable.__init__(self, table, path=path, replays=replays) + + @staticmethod + def _apply_replays(table: pa.Table, replays: Optional[list[Replay]] = None) -> pa.Table: + if replays is not None: + for name, args, kwargs in replays: + if name == "cast": + table = table_cast(table, *args, **kwargs) + elif name == "flatten": + table = table_flatten(table, *args, **kwargs) + else: + table = getattr(table, name)(*args, **kwargs) + return table + + def _append_replay(self, replay: Replay) -> list[Replay]: + replays = copy.deepcopy(self.replays) + replays.append(replay) + return replays + + def slice(self, offset=0, length=None): + """ + Compute zero-copy slice of this Table. + + Args: + offset (`int`, defaults to `0`): + Offset from start of table to slice. + length (`int`, defaults to `None`): + Length of slice (default is until end of table starting from + offset). + + Returns: + `datasets.table.Table` + """ + replay = ("slice", (offset, length), {}) + replays = self._append_replay(replay) + # Use fast slicing here + return MemoryMappedTable(self.fast_slice(offset=offset, length=length), self.path, replays) + + def filter(self, *args, **kwargs): + """ + Select records from a Table. See `pyarrow.compute.filter` for full usage. + """ + replay = ("filter", copy.deepcopy(args), copy.deepcopy(kwargs)) + replays = self._append_replay(replay) + return MemoryMappedTable(self.table.filter(*args, **kwargs), self.path, replays) + + def flatten(self, *args, **kwargs): + """ + Flatten this Table. Each column with a struct type is flattened + into one column per struct field. Other columns are left unchanged. + + Args: + memory_pool (`MemoryPool`, defaults to `None`): + For memory allocations, if required, otherwise use default pool. + + Returns: + `datasets.table.Table` + """ + replay = ("flatten", copy.deepcopy(args), copy.deepcopy(kwargs)) + replays = self._append_replay(replay) + return MemoryMappedTable(table_flatten(self.table, *args, **kwargs), self.path, replays) + + def combine_chunks(self, *args, **kwargs): + """ + Make a new table by combining the chunks this table has. + + All the underlying chunks in the ChunkedArray of each column are + concatenated into zero or one chunk. + + Args: + memory_pool (`MemoryPool`, defaults to `None`): + For memory allocations, if required, otherwise use default pool. + + Returns: + `datasets.table.Table` + """ + replay = ("combine_chunks", copy.deepcopy(args), copy.deepcopy(kwargs)) + replays = self._append_replay(replay) + return MemoryMappedTable(self.table.combine_chunks(*args, **kwargs), self.path, replays) + + def cast(self, *args, **kwargs): + """ + Cast table values to another schema + + Args: + target_schema (`Schema`): + Schema to cast to, the names and order of fields must match. + safe (`bool`, defaults to `True`): + Check for overflows or other unsafe conversions. + + Returns: + `datasets.table.Table` + """ + replay = ("cast", copy.deepcopy(args), copy.deepcopy(kwargs)) + replays = self._append_replay(replay) + return MemoryMappedTable(table_cast(self.table, *args, **kwargs), self.path, replays) + + def replace_schema_metadata(self, *args, **kwargs): + """ + EXPERIMENTAL: Create shallow copy of table by replacing schema + key-value metadata with the indicated new metadata (which may be None, + which deletes any existing metadata. + + Args: + metadata (`dict`, defaults to `None`): + + Returns: + `datasets.table.Table`: shallow_copy + """ + replay = ("replace_schema_metadata", copy.deepcopy(args), copy.deepcopy(kwargs)) + replays = self._append_replay(replay) + return MemoryMappedTable(self.table.replace_schema_metadata(*args, **kwargs), self.path, replays) + + def add_column(self, *args, **kwargs): + """ + Add column to Table at position. + + A new table is returned with the column added, the original table + object is left unchanged. + + Args: + i (`int`): + Index to place the column at. + field_ (`Union[str, pyarrow.Field]`): + If a string is passed then the type is deduced from the column + data. + column (`Union[pyarrow.Array, List[pyarrow.Array]]`): + Column data. + + Returns: + `datasets.table.Table`: New table with the passed column added. + """ + replay = ("add_column", copy.deepcopy(args), copy.deepcopy(kwargs)) + replays = self._append_replay(replay) + return MemoryMappedTable(self.table.add_column(*args, **kwargs), self.path, replays) + + def append_column(self, *args, **kwargs): + """ + Append column at end of columns. + + Args: + field_ (`Union[str, pyarrow.Field]`): + If a string is passed then the type is deduced from the column + data. + column (`Union[pyarrow.Array, List[pyarrow.Array]]`): + Column data. + + Returns: + `datasets.table.Table`: + New table with the passed column added. + """ + replay = ("append_column", copy.deepcopy(args), copy.deepcopy(kwargs)) + replays = self._append_replay(replay) + return MemoryMappedTable(self.table.append_column(*args, **kwargs), self.path, replays) + + def remove_column(self, *args, **kwargs): + """ + Create new Table with the indicated column removed. + + Args: + i (`int`): + Index of column to remove. + + Returns: + `datasets.table.Table`: + New table without the column. + """ + replay = ("remove_column", copy.deepcopy(args), copy.deepcopy(kwargs)) + replays = self._append_replay(replay) + return MemoryMappedTable(self.table.remove_column(*args, **kwargs), self.path, replays) + + def set_column(self, *args, **kwargs): + """ + Replace column in Table at position. + + Args: + i (`int`): + Index to place the column at. + field_ (`Union[str, pyarrow.Field]`): + If a string is passed then the type is deduced from the column + data. + column (`Union[pyarrow.Array, List[pyarrow.Array]]`): + Column data. + + Returns: + `datasets.table.Table`: + New table with the passed column set. + """ + replay = ("set_column", copy.deepcopy(args), copy.deepcopy(kwargs)) + replays = self._append_replay(replay) + return MemoryMappedTable(self.table.set_column(*args, **kwargs), self.path, replays) + + def rename_columns(self, *args, **kwargs): + """ + Create new table with columns renamed to provided names. + """ + replay = ("rename_columns", copy.deepcopy(args), copy.deepcopy(kwargs)) + replays = self._append_replay(replay) + return MemoryMappedTable(self.table.rename_columns(*args, **kwargs), self.path, replays) + + def drop(self, *args, **kwargs): + """ + Drop one or more columns and return a new table. + + Args: + columns (`List[str]`): + List of field names referencing existing columns. + + Raises: + `KeyError` : if any of the passed columns name are not existing. + + Returns: + `datasets.table.Table`: + New table without the columns. + """ + replay = ("drop", copy.deepcopy(args), copy.deepcopy(kwargs)) + replays = self._append_replay(replay) + return MemoryMappedTable(self.table.drop(*args, **kwargs), self.path, replays) + + def select(self, *args, **kwargs): + """ + Select columns of the table. + + Returns a new table with the specified columns, and metadata preserved. + + Args: + columns (:obj:`Union[List[str], List[int]]`): + The column names or integer indices to select. + + Returns: + :class:`datasets.table.Table`: New table with the specified columns, and metadata preserved. + """ + replay = ("select", copy.deepcopy(args), copy.deepcopy(kwargs)) + replays = self._append_replay(replay) + return MemoryMappedTable(self.table.select(*args, **kwargs), self.path, replays) + + +# A ConcatenationTable is the concatenation of several tables. +# The ``blocks`` attributes stores a list of list of blocks. +# The first axis concatenates the tables along the axis 0 (it appends rows), +# while the second axis concatenates tables along the axis 1 (it appends columns). +TableBlockContainer = TypeVar("TableBlockContainer", TableBlock, list[TableBlock], list[list[TableBlock]]) + + +class ConcatenationTable(Table): + """ + The table comes from the concatenation of several tables called blocks. + It enables concatenation on both axis 0 (append rows) and axis 1 (append columns). + + The underlying tables are called "blocks" and can be either `InMemoryTable` + or `MemoryMappedTable` objects. + This allows to combine tables that come from memory or that are memory mapped. + When a `ConcatenationTable` is pickled, then each block is pickled: + - the `InMemoryTable` objects are pickled by copying all the data in memory. + - the MemoryMappedTable objects are pickled without copying the data into memory. + Instead, only the path to the memory mapped arrow file is pickled, as well as the list + of transforms to "replays" when reloading the table from the disk. + + Its implementation requires to store each block separately. + The `blocks` attributes stores a list of list of blocks. + The first axis concatenates the tables along the axis 0 (it appends rows), + while the second axis concatenates tables along the axis 1 (it appends columns). + + If some columns are missing when concatenating on axis 0, they are filled with null values. + This is done using `pyarrow.concat_tables(tables, promote=True)`. + + You can access the fully combined table by accessing the `ConcatenationTable.table` attribute, + and the blocks by accessing the `ConcatenationTable.blocks` attribute. + """ + + def __init__(self, table: pa.Table, blocks: list[list[TableBlock]]): + super().__init__(table) + self.blocks = blocks + # Check that all the blocks have the right type. + # Only InMemoryTable and MemoryMappedTable are allowed. + for subtables in blocks: + for subtable in subtables: + if not isinstance(subtable, TableBlock): + raise TypeError( + "The blocks of a ConcatenationTable must be InMemoryTable or MemoryMappedTable objects" + f", but got {_short_str(subtable)}." + ) + + def __getstate__(self): + return {"blocks": self.blocks, "schema": self.table.schema} + + def __setstate__(self, state): + blocks = state["blocks"] + schema = state["schema"] + table = self._concat_blocks_horizontally_and_vertically(blocks) + if schema is not None and table.schema != schema: + # We fix the columns by concatenating with an empty table with the right columns + empty_table = pa.Table.from_batches([], schema=schema) + # We set promote_options="default" to fill missing columns with null values + table = pa.concat_tables([table, empty_table], promote_options="default") + ConcatenationTable.__init__(self, table, blocks=blocks) + + @staticmethod + def _concat_blocks(blocks: list[Union[TableBlock, pa.Table]], axis: int = 0) -> pa.Table: + pa_tables = [table.table if hasattr(table, "table") else table for table in blocks] + if axis == 0: + # We set promote_options="default" to fill missing columns with null values + return pa.concat_tables(pa_tables, promote_options="default") + elif axis == 1: + for i, table in enumerate(pa_tables): + if i == 0: + pa_table = table + else: + for name, col in zip(table.column_names, table.columns): + pa_table = pa_table.append_column(name, col) + return pa_table + else: + raise ValueError("'axis' must be either 0 or 1") + + @classmethod + def _concat_blocks_horizontally_and_vertically(cls, blocks: list[list[TableBlock]]) -> pa.Table: + pa_tables_to_concat_vertically = [] + for i, tables in enumerate(blocks): + if not tables: + continue + pa_table_horizontally_concatenated = cls._concat_blocks(tables, axis=1) + pa_tables_to_concat_vertically.append(pa_table_horizontally_concatenated) + return cls._concat_blocks(pa_tables_to_concat_vertically, axis=0) + + @classmethod + def _merge_blocks(cls, blocks: TableBlockContainer, axis: Optional[int] = None) -> TableBlockContainer: + if axis is not None: + merged_blocks = [] + for is_in_memory, block_group in groupby(blocks, key=lambda x: isinstance(x, InMemoryTable)): + if is_in_memory: + block_group = [InMemoryTable(cls._concat_blocks(list(block_group), axis=axis))] + merged_blocks += list(block_group) + else: # both + merged_blocks = [cls._merge_blocks(row_block, axis=1) for row_block in blocks] + if all(len(row_block) == 1 for row_block in merged_blocks): + merged_blocks = cls._merge_blocks( + [block for row_block in merged_blocks for block in row_block], axis=0 + ) + return merged_blocks + + @classmethod + def _consolidate_blocks(cls, blocks: TableBlockContainer) -> TableBlockContainer: + if isinstance(blocks, TableBlock): + return blocks + elif isinstance(blocks[0], TableBlock): + return cls._merge_blocks(blocks, axis=0) + else: + return cls._merge_blocks(blocks) + + @classmethod + def from_blocks(cls, blocks: TableBlockContainer) -> "ConcatenationTable": + blocks = cls._consolidate_blocks(blocks) + if isinstance(blocks, TableBlock): + table = blocks + return cls(table.table, [[table]]) + elif isinstance(blocks[0], TableBlock): + table = cls._concat_blocks(blocks, axis=0) + blocks = [[t] for t in blocks] + return cls(table, blocks) + else: + table = cls._concat_blocks_horizontally_and_vertically(blocks) + return cls(table, blocks) + + @classmethod + def from_tables(cls, tables: list[Union[pa.Table, Table]], axis: int = 0) -> "ConcatenationTable": + """Create `ConcatenationTable` from list of tables. + + Args: + tables (list of `Table` or list of `pyarrow.Table`): + List of tables. + axis (`{0, 1}`, defaults to `0`, meaning over rows): + Axis to concatenate over, where `0` means over rows (vertically) and `1` means over columns + (horizontally). + + + """ + + def to_blocks(table: Union[pa.Table, Table]) -> list[list[TableBlock]]: + if isinstance(table, pa.Table): + return [[InMemoryTable(table)]] + elif isinstance(table, ConcatenationTable): + return copy.deepcopy(table.blocks) + else: + return [[table]] + + def _slice_row_block(row_block: list[TableBlock], length: int) -> tuple[list[TableBlock], list[TableBlock]]: + sliced = [table.slice(0, length) for table in row_block] + remainder = [table.slice(length, len(row_block[0]) - length) for table in row_block] + return sliced, remainder + + def _split_both_like( + result: list[list[TableBlock]], blocks: list[list[TableBlock]] + ) -> tuple[list[list[TableBlock]], list[list[TableBlock]]]: + """ + Make sure each row_block contain the same num_rows to be able to concatenate them on axis=1. + + To do so, we modify both blocks sets to have the same row_blocks boundaries. + For example, if `result` has 2 row_blocks of 3 rows and `blocks` has 3 row_blocks of 2 rows, + we modify both to have 4 row_blocks of size 2, 1, 1 and 2: + + [ x x x | x x x ] + + [ y y | y y | y y ] + ----------------------------- + = [ x x | x | x | x x ] + [ y y | y | y | y y ] + + """ + result, blocks = list(result), list(blocks) + new_result, new_blocks = [], [] + while result and blocks: + # we slice the longest row block to save two row blocks of same length + # and we replace the long row block by its remainder if necessary + if len(result[0][0]) > len(blocks[0][0]): + new_blocks.append(blocks[0]) + sliced, result[0] = _slice_row_block(result[0], len(blocks.pop(0)[0])) + new_result.append(sliced) + elif len(result[0][0]) < len(blocks[0][0]): + new_result.append(result[0]) + sliced, blocks[0] = _slice_row_block(blocks[0], len(result.pop(0)[0])) + new_blocks.append(sliced) + else: + new_result.append(result.pop(0)) + new_blocks.append(blocks.pop(0)) + if result or blocks: + raise ValueError("Failed to concatenate on axis=1 because tables don't have the same number of rows") + return new_result, new_blocks + + def _extend_blocks( + result: list[list[TableBlock]], blocks: list[list[TableBlock]], axis: int = 0 + ) -> list[list[TableBlock]]: + if axis == 0: + result.extend(blocks) + elif axis == 1: + # We make sure each row_block have the same num_rows + result, blocks = _split_both_like(result, blocks) + for i, row_block in enumerate(blocks): + result[i].extend(row_block) + return result + + blocks = to_blocks(tables[0]) + for table in tables[1:]: + table_blocks = to_blocks(table) + blocks = _extend_blocks(blocks, table_blocks, axis=axis) + return cls.from_blocks(blocks) + + @property + def _slices(self): + offset = 0 + for tables in self.blocks: + length = len(tables[0]) + yield (offset, length) + offset += length + + def slice(self, offset=0, length=None): + """ + Compute zero-copy slice of this Table. + + Args: + offset (`int`, defaults to `0`): + Offset from start of table to slice. + length (`int`, defaults to `None`): + Length of slice (default is until end of table starting from + offset). + + Returns: + `datasets.table.Table` + """ + table = self.table.slice(offset, length=length) + length = length if length is not None else self.num_rows - offset + blocks = [] + for tables in self.blocks: + n_rows = len(tables[0]) + if length == 0: + break + elif n_rows <= offset: + offset = offset - n_rows + elif n_rows <= offset + length: + blocks.append([t.slice(offset) for t in tables]) + length, offset = length + offset - n_rows, 0 + else: + blocks.append([t.slice(offset, length) for t in tables]) + length, offset = 0, 0 + return ConcatenationTable(table, blocks) + + def filter(self, mask, *args, **kwargs): + """ + Select records from a Table. See `pyarrow.compute.filter` for full usage. + """ + table = self.table.filter(mask, *args, **kwargs) + blocks = [] + for (offset, length), tables in zip(self._slices, self.blocks): + submask = mask.slice(offset, length) + blocks.append([t.filter(submask, *args, **kwargs) for t in tables]) + return ConcatenationTable(table, blocks) + + def flatten(self, *args, **kwargs): + """ + Flatten this Table. Each column with a struct type is flattened + into one column per struct field. Other columns are left unchanged. + + Args: + memory_pool (`MemoryPool`, defaults to `None`): + For memory allocations, if required, otherwise use default pool. + + Returns: + `datasets.table.Table` + """ + table = table_flatten(self.table, *args, **kwargs) + blocks = [] + for tables in self.blocks: + blocks.append([t.flatten(*args, **kwargs) for t in tables]) + return ConcatenationTable(table, blocks) + + def combine_chunks(self, *args, **kwargs): + """ + Make a new table by combining the chunks this table has. + + All the underlying chunks in the `ChunkedArray` of each column are + concatenated into zero or one chunk. + + Args: + memory_pool (`MemoryPool`, defaults to `None`): + For memory allocations, if required, otherwise use default pool. + + Returns: + `datasets.table.Table` + """ + table = self.table.combine_chunks(*args, **kwargs) + blocks = [] + for tables in self.blocks: + blocks.append([t.combine_chunks(*args, **kwargs) for t in tables]) + return ConcatenationTable(table, blocks) + + def cast(self, target_schema, *args, **kwargs): + """ + Cast table values to another schema. + + Args: + target_schema (`Schema`): + Schema to cast to, the names and order of fields must match. + safe (`bool`, defaults to `True`): + Check for overflows or other unsafe conversions. + + Returns: + `datasets.table.Table` + """ + from .features import Features + + table = table_cast(self.table, target_schema, *args, **kwargs) + target_features = Features.from_arrow_schema(target_schema) + blocks = [] + for subtables in self.blocks: + new_tables = [] + fields = list(target_schema) + for subtable in subtables: + subfields = [] + for name in subtable.column_names: + subfields.append(fields.pop(next(i for i, field in enumerate(fields) if field.name == name))) + subfeatures = Features({subfield.name: target_features[subfield.name] for subfield in subfields}) + subschema = subfeatures.arrow_schema + new_tables.append(subtable.cast(subschema, *args, **kwargs)) + blocks.append(new_tables) + return ConcatenationTable(table, blocks) + + def replace_schema_metadata(self, *args, **kwargs): + """ + EXPERIMENTAL: Create shallow copy of table by replacing schema + key-value metadata with the indicated new metadata (which may be `None`, + which deletes any existing metadata). + + Args: + metadata (`dict`, defaults to `None`): + + Returns: + `datasets.table.Table`: shallow_copy + """ + table = self.table.replace_schema_metadata(*args, **kwargs) + blocks = [] + for tables in self.blocks: + blocks.append([t.replace_schema_metadata(*args, **kwargs) for t in tables]) + return ConcatenationTable(table, self.blocks) + + def add_column(self, *args, **kwargs): + """ + Add column to Table at position. + + A new table is returned with the column added, the original table + object is left unchanged. + + Args: + i (`int`): + Index to place the column at. + field_ (`Union[str, pyarrow.Field]`): + If a string is passed then the type is deduced from the column + data. + column (`Union[pyarrow.Array, List[pyarrow.Array]]`): + Column data. + + Returns: + `datasets.table.Table`: New table with the passed column added. + """ + raise NotImplementedError() + + def append_column(self, *args, **kwargs): + """ + Append column at end of columns. + + Args: + field_ (`Union[str, pyarrow.Field]`): + If a string is passed then the type is deduced from the column + data. + column (`Union[pyarrow.Array, List[pyarrow.Array]]`): + Column data. + + Returns: + `datasets.table.Table`: + New table with the passed column added. + """ + raise NotImplementedError() + + def remove_column(self, i, *args, **kwargs): + """ + Create new Table with the indicated column removed. + + Args: + i (`int`): + Index of column to remove. + + Returns: + `datasets.table.Table`: + New table without the column. + """ + table = self.table.remove_column(i, *args, **kwargs) + name = self.table.column_names[i] + blocks = [] + for tables in self.blocks: + blocks.append( + [ + t.remove_column(t.column_names.index(name), *args, **kwargs) if name in t.column_names else t + for t in tables + ] + ) + return ConcatenationTable(table, blocks) + + def set_column(self, *args, **kwargs): + """ + Replace column in Table at position. + + Args: + i (`int`): + Index to place the column at. + field_ (`Union[str, pyarrow.Field]`): + If a string is passed then the type is deduced from the column + data. + column (`Union[pyarrow.Array, List[pyarrow.Array]]`): + Column data. + + Returns: + `datasets.table.Table`: + New table with the passed column set. + """ + raise NotImplementedError() + + def rename_columns(self, names, *args, **kwargs): + """ + Create new table with columns renamed to provided names. + """ + table = self.table.rename_columns(names, *args, **kwargs) + names = dict(zip(self.table.column_names, names)) + blocks = [] + for tables in self.blocks: + blocks.append( + [t.rename_columns([names[name] for name in t.column_names], *args, **kwargs) for t in tables] + ) + return ConcatenationTable(table, blocks) + + def drop(self, columns, *args, **kwargs): + """ + Drop one or more columns and return a new table. + + Args: + columns (`List[str]`): + List of field names referencing existing columns. + + Raises: + `KeyError` : if any of the passed columns name are not existing. + + Returns: + `datasets.table.Table`: + New table without the columns. + """ + table = self.table.drop(columns, *args, **kwargs) + blocks = [] + for tables in self.blocks: + blocks.append([t.drop([c for c in columns if c in t.column_names], *args, **kwargs) for t in tables]) + return ConcatenationTable(table, blocks) + + def select(self, columns, *args, **kwargs): + """ + Select columns of the table. + + Returns a new table with the specified columns, and metadata preserved. + + Args: + columns (:obj:`Union[List[str], List[int]]`): + The column names or integer indices to select. + + Returns: + :class:`datasets.table.Table`: New table with the specified columns, and metadata preserved. + """ + table = self.table.select(columns, *args, **kwargs) + blocks = [] + for tables in self.blocks: + blocks.append([t.select([c for c in columns if c in t.column_names], *args, **kwargs) for t in tables]) + return ConcatenationTable(table, blocks) + + +def concat_tables(tables: list[Table], axis: int = 0) -> Table: + """ + Concatenate tables. + + Args: + tables (list of `Table`): + List of tables to be concatenated. + axis (`{0, 1}`, defaults to `0`, meaning over rows): + Axis to concatenate over, where `0` means over rows (vertically) and `1` means over columns + (horizontally). + + + Returns: + `datasets.table.Table`: + If the number of input tables is > 1, then the returned table is a `datasets.table.ConcatenationTable`. + Otherwise if there's only one table, it is returned as is. + """ + tables = list(tables) + if len(tables) == 1: + return tables[0] + return ConcatenationTable.from_tables(tables, axis=axis) + + +def list_table_cache_files(table: Table) -> list[str]: + """ + Get the cache files that are loaded by the table. + Cache file are used when parts of the table come from the disk via memory mapping. + + Returns: + `List[str]`: + A list of paths to the cache files loaded by the table. + """ + if isinstance(table, ConcatenationTable): + cache_files = [] + for subtables in table.blocks: + for subtable in subtables: + cache_files += list_table_cache_files(subtable) + return cache_files + elif isinstance(table, MemoryMappedTable): + return [table.path] + else: + return [] + + +def _wrap_for_chunked_arrays(func): + """Apply the function on each chunk of a `pyarrow.ChunkedArray`, or on the array directly""" + + def wrapper(array, *args, **kwargs): + if isinstance(array, pa.ChunkedArray): + return pa.chunked_array([func(chunk, *args, **kwargs) for chunk in array.chunks]) + else: + return func(array, *args, **kwargs) + + return wrapper + + +def _are_list_values_of_length(array: pa.ListArray, length: int) -> bool: + """Check if all the sub-lists of a `pa.ListArray` have the specified length.""" + return pc.all(pc.equal(array.value_lengths(), length)).as_py() or array.null_count == len(array) + + +def _combine_list_array_offsets_with_mask(array: pa.ListArray) -> pa.Array: + """Add the null bitmap to the offsets of a `pa.ListArray`.""" + offsets = array.offsets + if array.null_count > 0: + offsets = pa.concat_arrays( + [ + pc.replace_with_mask(offsets[:-1], array.is_null(), pa.nulls(len(array), pa.int32())), + offsets[-1:], + ] + ) + return offsets + + +def _storage_type(type: pa.DataType) -> pa.DataType: + """Convert a (possibly nested) `pa.ExtensionType` to its storage type.""" + if isinstance(type, pa.ExtensionType): + return _storage_type(type.storage_type) + elif isinstance(type, pa.StructType): + return pa.struct([pa.field(field.name, _storage_type(field.type)) for field in type]) + elif isinstance(type, pa.ListType): + return pa.list_(_storage_type(type.value_type)) + elif isinstance(type, pa.FixedSizeListType): + return pa.list_(_storage_type(type.value_type), type.list_size) + return type + + +def _short_str(value: Any) -> str: + out = str(value) + if len(out) > 3000: + out = out[:1500] + "\n...\n" + out[-1500:] + return out + + +@_wrap_for_chunked_arrays +def array_cast( + array: pa.Array, pa_type: pa.DataType, allow_primitive_to_str: bool = True, allow_decimal_to_str: bool = True +) -> Union[pa.Array, pa.FixedSizeListArray, pa.ListArray, pa.StructArray, pa.ExtensionArray]: + """Improved version of `pa.Array.cast` + + It supports casting `pa.StructArray` objects to re-order the fields. + It also let you control certain aspects of the casting, e.g. whether + to disable casting primitives (`booleans`, `floats` or `ints`) or + disable casting decimals to strings. + + Args: + array (`pa.Array`): + PyArrow array to cast + pa_type (`pa.DataType`): + Target PyArrow type + allow_primitive_to_str (`bool`, defaults to `True`): + Whether to allow casting primitives to strings. + Defaults to `True`. + allow_decimal_to_str (`bool`, defaults to `True`): + Whether to allow casting decimals to strings. + Defaults to `True`. + + Raises: + `pa.ArrowInvalidError`: if the arrow data casting fails + `TypeError`: if the target type is not supported according, e.g. + + - if a field is missing + - if casting from primitives to strings and `allow_primitive_to_str` is `False` + - if casting from decimals to strings and `allow_decimal_to_str` is `False` + + Returns: + `List[pyarrow.Array]`: the casted array + """ + _c = partial(array_cast, allow_primitive_to_str=allow_primitive_to_str, allow_decimal_to_str=allow_decimal_to_str) + if isinstance(array, pa.ExtensionArray): + array = array.storage + if isinstance(pa_type, pa.ExtensionType): + return pa_type.wrap_array(_c(array, pa_type.storage_type)) + elif array.type == pa_type: + return array + elif pa.types.is_struct(array.type): + if pa.types.is_struct(pa_type) and ({field.name for field in pa_type} == {field.name for field in array.type}): + if array.type.num_fields == 0: + return array + arrays = [_c(array.field(field.name), field.type) for field in pa_type] + return pa.StructArray.from_arrays(arrays, fields=list(pa_type), mask=array.is_null()) + elif pa.types.is_list(array.type) or pa.types.is_large_list(array.type): + if pa.types.is_fixed_size_list(pa_type): + if _are_list_values_of_length(array, pa_type.list_size): + if array.null_count > 0: + # Ensure each null value in the array translates to [null] * pa_type.list_size in the array's values array + array_type = array.type + storage_type = _storage_type(array_type) + if array_type != storage_type: + # Temporarily convert to the storage type to support extension types in the slice operation + array = _c(array, storage_type) + array = pc.list_slice(array, 0, pa_type.list_size, return_fixed_size_list=True) + array = _c(array, array_type) + else: + array = pc.list_slice(array, 0, pa_type.list_size, return_fixed_size_list=True) + array_values = array.values + return pa.FixedSizeListArray.from_arrays( + _c(array_values, pa_type.value_type), pa_type.list_size, mask=array.is_null() + ) + else: + array_values = array.values[ + array.offset * pa_type.list_size : (array.offset + len(array)) * pa_type.list_size + ] + return pa.FixedSizeListArray.from_arrays(_c(array_values, pa_type.value_type), pa_type.list_size) + elif pa.types.is_list(pa_type): + # Merge offsets with the null bitmap to avoid the "Null bitmap with offsets slice not supported" ArrowNotImplementedError + array_offsets = _combine_list_array_offsets_with_mask(array) + return pa.ListArray.from_arrays(array_offsets, _c(array.values, pa_type.value_type)) + elif pa.types.is_large_list(pa_type): + # Merge offsets with the null bitmap to avoid the "Null bitmap with offsets slice not supported" ArrowNotImplementedError + array_offsets = _combine_list_array_offsets_with_mask(array) + return pa.LargeListArray.from_arrays(array_offsets, _c(array.values, pa_type.value_type)) + elif pa.types.is_fixed_size_list(array.type): + if pa.types.is_fixed_size_list(pa_type): + if pa_type.list_size == array.type.list_size: + array_values = array.values[ + array.offset * array.type.list_size : (array.offset + len(array)) * array.type.list_size + ] + return pa.FixedSizeListArray.from_arrays( + _c(array_values, pa_type.value_type), pa_type.list_size, mask=array.is_null() + ) + elif pa.types.is_list(pa_type): + array_offsets = (np.arange(len(array) + 1) + array.offset) * array.type.list_size + return pa.ListArray.from_arrays(array_offsets, _c(array.values, pa_type.value_type), mask=array.is_null()) + elif pa.types.is_large_list(pa_type): + array_offsets = (np.arange(len(array) + 1) + array.offset) * array.type.list_size + return pa.LargeListArray.from_arrays( + array_offsets, _c(array.values, pa_type.value_type), mask=array.is_null() + ) + else: + if pa.types.is_string(pa_type): + if not allow_primitive_to_str and pa.types.is_primitive(array.type): + raise TypeError( + f"Couldn't cast array of type {_short_str(array.type)} to {_short_str(pa_type)} " + f"since allow_primitive_to_str is set to {allow_primitive_to_str} " + ) + if not allow_decimal_to_str and pa.types.is_decimal(array.type): + raise TypeError( + f"Couldn't cast array of type {_short_str(array.type)} to {_short_str(pa_type)} " + f"and allow_decimal_to_str is set to {allow_decimal_to_str}" + ) + if pa.types.is_null(pa_type) and not pa.types.is_null(array.type): + raise TypeError(f"Couldn't cast array of type {_short_str(array.type)} to {_short_str(pa_type)}") + return array.cast(pa_type) + raise TypeError(f"Couldn't cast array of type {_short_str(array.type)} to {_short_str(pa_type)}") + + +@_wrap_for_chunked_arrays +def cast_array_to_feature( + array: pa.Array, feature: "FeatureType", allow_primitive_to_str: bool = True, allow_decimal_to_str: bool = True +) -> pa.Array: + """Cast an array to the arrow type that corresponds to the requested feature type. + For custom features like [`Audio`] or [`Image`], it takes into account the "cast_storage" methods + they defined to enable casting from other arrow types. + + Args: + array (`pa.Array`): + The PyArrow array to cast. + feature (`datasets.features.FeatureType`): + The target feature type. + allow_primitive_to_str (`bool`, defaults to `True`): + Whether to allow casting primitives to strings. + Defaults to `True`. + allow_decimal_to_str (`bool`, defaults to `True`): + Whether to allow casting decimals to strings. + Defaults to `True`. + + Raises: + `pa.ArrowInvalidError`: if the arrow data casting fails + `TypeError`: if the target type is not supported according, e.g. + + - if a field is missing + - if casting from primitives and `allow_primitive_to_str` is `False` + - if casting from decimals and `allow_decimal_to_str` is `False` + + Returns: + array (`pyarrow.Array`): the casted array + """ + from .features.features import LargeList, Sequence, get_nested_type + + _c = partial( + cast_array_to_feature, + allow_primitive_to_str=allow_primitive_to_str, + allow_decimal_to_str=allow_decimal_to_str, + ) + + if isinstance(array, pa.ExtensionArray): + array = array.storage + if hasattr(feature, "cast_storage"): + return feature.cast_storage(array) + + elif pa.types.is_struct(array.type): + # feature must be a dict or Sequence(subfeatures_dict) + if isinstance(feature, Sequence) and isinstance(feature.feature, dict): + sequence_kwargs = vars(feature).copy() + feature = sequence_kwargs.pop("feature") + feature = {name: Sequence(subfeature, **sequence_kwargs) for name, subfeature in feature.items()} + if isinstance(feature, dict) and (array_fields := {field.name for field in array.type}) <= set(feature): + null_array = pa.array([None] * len(array)) + arrays = [ + _c(array.field(name) if name in array_fields else null_array, subfeature) + for name, subfeature in feature.items() + ] + return pa.StructArray.from_arrays(arrays, names=list(feature), mask=array.is_null()) + elif pa.types.is_list(array.type) or pa.types.is_large_list(array.type): + # feature must be either [subfeature] or LargeList(subfeature) or Sequence(subfeature) + if isinstance(feature, list): + casted_array_values = _c(array.values, feature[0]) + if pa.types.is_list(array.type) and casted_array_values.type == array.values.type: + # Both array and feature have equal list type and values (within the list) type + return array + else: + # Merge offsets with the null bitmap to avoid the "Null bitmap with offsets slice not supported" ArrowNotImplementedError + array_offsets = _combine_list_array_offsets_with_mask(array) + return pa.ListArray.from_arrays(array_offsets, casted_array_values) + elif isinstance(feature, LargeList): + casted_array_values = _c(array.values, feature.feature) + if pa.types.is_large_list(array.type) and casted_array_values.type == array.values.type: + # Both array and feature have equal large_list type and values (within the list) type + return array + else: + # Merge offsets with the null bitmap to avoid the "Null bitmap with offsets slice not supported" ArrowNotImplementedError + array_offsets = _combine_list_array_offsets_with_mask(array) + return pa.LargeListArray.from_arrays(array_offsets, casted_array_values) + elif isinstance(feature, Sequence): + if feature.length > -1: + if _are_list_values_of_length(array, feature.length): + if array.null_count > 0: + # Ensure each null value in the array translates to [null] * pa_type.list_size in the array's values array + array_type = array.type + storage_type = _storage_type(array_type) + if array_type != storage_type: + # Temporarily convert to the storage type to support extension types in the slice operation + array = array_cast( + array, + storage_type, + allow_primitive_to_str=allow_primitive_to_str, + allow_decimal_to_str=allow_decimal_to_str, + ) + array = pc.list_slice(array, 0, feature.length, return_fixed_size_list=True) + array = array_cast( + array, + array_type, + allow_primitive_to_str=allow_primitive_to_str, + allow_decimal_to_str=allow_decimal_to_str, + ) + else: + array = pc.list_slice(array, 0, feature.length, return_fixed_size_list=True) + array_values = array.values + casted_array_values = _c(array_values, feature.feature) + return pa.FixedSizeListArray.from_arrays( + casted_array_values, feature.length, mask=array.is_null() + ) + else: + array_values = array.values[ + array.offset * feature.length : (array.offset + len(array)) * feature.length + ] + return pa.FixedSizeListArray.from_arrays(_c(array_values, feature.feature), feature.length) + else: + casted_array_values = _c(array.values, feature.feature) + if pa.types.is_list(array.type) and casted_array_values.type == array.values.type: + # Both array and feature have equal list type and values (within the list) type + return array + else: + # Merge offsets with the null bitmap to avoid the "Null bitmap with offsets slice not supported" ArrowNotImplementedError + array_offsets = _combine_list_array_offsets_with_mask(array) + return pa.ListArray.from_arrays(array_offsets, casted_array_values) + elif pa.types.is_fixed_size_list(array.type): + # feature must be either [subfeature] or Sequence(subfeature) + if isinstance(feature, list): + array_offsets = (np.arange(len(array) + 1) + array.offset) * array.type.list_size + return pa.ListArray.from_arrays(array_offsets, _c(array.values, feature[0]), mask=array.is_null()) + elif isinstance(feature, LargeList): + array_offsets = (np.arange(len(array) + 1) + array.offset) * array.type.list_size + return pa.LargeListArray.from_arrays( + array_offsets, _c(array.values, feature.feature), mask=array.is_null() + ) + elif isinstance(feature, Sequence): + if feature.length > -1: + if feature.length == array.type.list_size: + array_values = array.values[ + array.offset * array.type.list_size : (array.offset + len(array)) * array.type.list_size + ] + casted_array_values = _c(array_values, feature.feature) + return pa.FixedSizeListArray.from_arrays(casted_array_values, feature.length, mask=array.is_null()) + else: + array_offsets = (np.arange(len(array) + 1) + array.offset) * array.type.list_size + return pa.ListArray.from_arrays(array_offsets, _c(array.values, feature.feature), mask=array.is_null()) + if pa.types.is_null(array.type): + return array_cast( + array, + get_nested_type(feature), + allow_primitive_to_str=allow_primitive_to_str, + allow_decimal_to_str=allow_decimal_to_str, + ) + elif not isinstance(feature, (Sequence, dict, list, tuple)): + return array_cast( + array, + feature(), + allow_primitive_to_str=allow_primitive_to_str, + allow_decimal_to_str=allow_decimal_to_str, + ) + raise TypeError(f"Couldn't cast array of type\n{_short_str(array.type)}\nto\n{_short_str(feature)}") + + +@_wrap_for_chunked_arrays +def embed_array_storage(array: pa.Array, feature: "FeatureType"): + """Embed data into an arrays's storage. + For custom features like Audio or Image, it takes into account the "embed_storage" methods + they define to embed external data (e.g. an image file) into an array. + + + + Args: + array (`pa.Array`): + The PyArrow array in which to embed data. + feature (`datasets.features.FeatureType`): + Array features. + + Raises: + `TypeError`: if the target type is not supported according, e.g. + + - if a field is missing + + Returns: + array (`pyarrow.Array`): the casted array + """ + from .features import Sequence + + _e = embed_array_storage + + if isinstance(array, pa.ExtensionArray): + array = array.storage + if hasattr(feature, "embed_storage"): + return feature.embed_storage(array) + elif pa.types.is_struct(array.type): + # feature must be a dict or Sequence(subfeatures_dict) + if isinstance(feature, Sequence) and isinstance(feature.feature, dict): + feature = { + name: Sequence(subfeature, length=feature.length) for name, subfeature in feature.feature.items() + } + if isinstance(feature, dict): + arrays = [_e(array.field(name), subfeature) for name, subfeature in feature.items()] + return pa.StructArray.from_arrays(arrays, names=list(feature), mask=array.is_null()) + elif pa.types.is_list(array.type): + # feature must be either [subfeature] or Sequence(subfeature) + # Merge offsets with the null bitmap to avoid the "Null bitmap with offsets slice not supported" ArrowNotImplementedError + array_offsets = _combine_list_array_offsets_with_mask(array) + if isinstance(feature, list): + return pa.ListArray.from_arrays(array_offsets, _e(array.values, feature[0])) + if isinstance(feature, Sequence) and feature.length == -1: + return pa.ListArray.from_arrays(array_offsets, _e(array.values, feature.feature)) + elif pa.types.is_large_list(array.type): + # feature must be LargeList(subfeature) + # Merge offsets with the null bitmap to avoid the "Null bitmap with offsets slice not supported" ArrowNotImplementedError + array_offsets = _combine_list_array_offsets_with_mask(array) + return pa.LargeListArray.from_arrays(array_offsets, _e(array.values, feature.feature)) + elif pa.types.is_fixed_size_list(array.type): + # feature must be Sequence(subfeature) + if isinstance(feature, Sequence) and feature.length > -1: + array_values = array.values[ + array.offset * array.type.list_size : (array.offset + len(array)) * array.type.list_size + ] + embedded_array_values = _e(array_values, feature.feature) + return pa.FixedSizeListArray.from_arrays(embedded_array_values, feature.length, mask=array.is_null()) + if not isinstance(feature, (Sequence, dict, list, tuple)): + return array + raise TypeError(f"Couldn't embed array of type\n{_short_str(array.type)}\nwith\n{_short_str(feature)}") + + +class CastError(ValueError): + """When it's not possible to cast an Arrow table to a specific schema or set of features""" + + def __init__(self, *args, table_column_names: list[str], requested_column_names: list[str]) -> None: + super().__init__(*args) + self.table_column_names = table_column_names + self.requested_column_names = requested_column_names + + def __reduce__(self): + # Fix unpickling: TypeError: __init__() missing 2 required keyword-only arguments: 'table_column_names' and 'requested_column_names' + return partial( + CastError, table_column_names=self.table_column_names, requested_column_names=self.requested_column_names + ), () + + def details(self): + new_columns = set(self.table_column_names) - set(self.requested_column_names) + missing_columns = set(self.requested_column_names) - set(self.table_column_names) + if new_columns and missing_columns: + return f"there are {len(new_columns)} new columns ({_short_str(new_columns)}) and {len(missing_columns)} missing columns ({_short_str(missing_columns)})." + elif new_columns: + return f"there are {len(new_columns)} new columns ({_short_str(new_columns)})" + else: + return f"there are {len(missing_columns)} missing columns ({_short_str(missing_columns)})" + + +def cast_table_to_features(table: pa.Table, features: "Features"): + """Cast a table to the arrow schema that corresponds to the requested features. + + Args: + table (`pyarrow.Table`): + PyArrow table to cast. + features ([`Features`]): + Target features. + + Returns: + table (`pyarrow.Table`): the casted table + """ + if sorted(table.column_names) != sorted(features): + raise CastError( + f"Couldn't cast\n{_short_str(table.schema)}\nto\n{_short_str(features)}\nbecause column names don't match", + table_column_names=table.column_names, + requested_column_names=list(features), + ) + arrays = [cast_array_to_feature(table[name], feature) for name, feature in features.items()] + return pa.Table.from_arrays(arrays, schema=features.arrow_schema) + + +def cast_table_to_schema(table: pa.Table, schema: pa.Schema): + """Cast a table to the arrow schema. Different from `cast_table_to_features`, this method can preserve nullability. + + Args: + table (`pa.Table`): + PyArrow table to cast. + features ([`Features`]): + Target features. + + Returns: + `pa.Table`: the casted table + """ + from .features import Features + + features = Features.from_arrow_schema(schema) + table_column_names = set(table.column_names) + if not table_column_names <= set(schema.names): + raise CastError( + f"Couldn't cast\n{_short_str(table.schema)}\nto\n{_short_str(features)}\nbecause column names don't match", + table_column_names=table.column_names, + requested_column_names=list(features), + ) + arrays = [ + cast_array_to_feature( + table[name] if name in table_column_names else pa.array([None] * len(table), type=schema.field(name).type), + feature, + ) + for name, feature in features.items() + ] + return pa.Table.from_arrays(arrays, schema=schema) + + +def embed_table_storage(table: pa.Table): + """Embed external data into a table's storage. + + + + Args: + table (`pyarrow.Table`): + PyArrow table in which to embed data. + + Returns: + table (`pyarrow.Table`): the table with embedded data + """ + from .features.features import Features, require_storage_embed + + features = Features.from_arrow_schema(table.schema) + arrays = [ + embed_array_storage(table[name], feature) if require_storage_embed(feature) else table[name] + for name, feature in features.items() + ] + return pa.Table.from_arrays(arrays, schema=features.arrow_schema) + + +def table_cast(table: pa.Table, schema: pa.Schema): + """Improved version of `pa.Table.cast`. + + It supports casting to feature types stored in the schema metadata. + + Args: + table (`pyarrow.Table`): + PyArrow table to cast. + schema (`pyarrow.Schema`): + Target PyArrow schema. + + Returns: + table (`pyarrow.Table`): the casted table + """ + if table.schema != schema: + return cast_table_to_schema(table, schema) + elif table.schema.metadata != schema.metadata: + return table.replace_schema_metadata(schema.metadata) + else: + return table + + +def table_flatten(table: pa.Table): + """Improved version of `pa.Table.flatten`. + + It behaves as `pa.Table.flatten` in a sense it does 1-step flatten of the columns with a struct type into one column per struct field, + but updates the metadata and skips decodable features unless the `decode` attribute of these features is set to False. + + Args: + table (`pa.Table`): + PyArrow table to flatten. + + Returns: + `Table`: the flattened table + """ + from .features import Features + + features = Features.from_arrow_schema(table.schema) + if any(hasattr(subfeature, "flatten") and subfeature.flatten() == subfeature for subfeature in features.values()): + flat_arrays = [] + flat_column_names = [] + for field in table.schema: + array = table.column(field.name) + subfeature = features[field.name] + if pa.types.is_struct(field.type) and ( + not hasattr(subfeature, "flatten") or subfeature.flatten() != subfeature + ): + flat_arrays.extend(array.flatten()) + flat_column_names.extend([f"{field.name}.{subfield.name}" for subfield in field.type]) + else: + flat_arrays.append(array) + flat_column_names.append(field.name) + flat_table = pa.Table.from_arrays( + flat_arrays, + names=flat_column_names, + ) + else: + flat_table = table.flatten() + # Preserve complex types in the metadata + flat_features = features.flatten(max_depth=2) + flat_features = Features({column_name: flat_features[column_name] for column_name in flat_table.column_names}) + return flat_table.replace_schema_metadata(flat_features.arrow_schema.metadata) + + +def table_visitor(table: pa.Table, function: Callable[[pa.Array], None]): + """Visit all arrays in a table and apply a function to them. + + Args: + table (`pyarrow.Table`): + PyArrow table to visit. + function (`Callable[[pa.Array], None]`): + Function to apply to each array. + """ + from .features import Features, Sequence + + features = Features.from_arrow_schema(table.schema) + + def _visit(array, feature): + if isinstance(array, pa.ChunkedArray): + for chunk in array.chunks: + _visit(chunk, feature) + else: + if isinstance(array, pa.ExtensionArray): + array = array.storage + function(array, feature) + if pa.types.is_struct(array.type) and not hasattr(feature, "cast_storage"): + if isinstance(feature, Sequence) and isinstance(feature.feature, dict): + feature = { + name: Sequence(subfeature, length=feature.length) + for name, subfeature in feature.feature.items() + } + for name, subfeature in feature.items(): + _visit(array.field(name), subfeature) + elif pa.types.is_list(array.type): + if isinstance(feature, list): + _visit(array.values, feature[0]) + elif isinstance(feature, Sequence): + _visit(array.values, feature.feature) + + for name, feature in features.items(): + _visit(table[name], feature) + + +def table_iter(table: Table, batch_size: int, drop_last_batch=False) -> Iterator[pa.Table]: + """Iterate over sub-tables of size `batch_size`. + + Args: + table (`pyarrow.Table`): + PyArrow table to iterate over. + batch_size (`int`): + Size of each sub-table to yield. + drop_last_batch (`bool`, defaults to `False`): + Drop the last batch if it is smaller than `batch_size`. + """ + chunks_buffer = [] + chunks_buffer_size = 0 + for chunk in table.to_reader(max_chunksize=batch_size): + if len(chunk) == 0: + continue + elif chunks_buffer_size + len(chunk) < batch_size: + chunks_buffer.append(chunk) + chunks_buffer_size += len(chunk) + continue + elif chunks_buffer_size + len(chunk) == batch_size: + chunks_buffer.append(chunk) + yield pa.Table.from_batches(chunks_buffer) + chunks_buffer = [] + chunks_buffer_size = 0 + else: + cropped_chunk_length = batch_size - chunks_buffer_size + chunks_buffer.append(chunk.slice(0, cropped_chunk_length)) + yield pa.Table.from_batches(chunks_buffer) + chunks_buffer = [chunk.slice(cropped_chunk_length, len(chunk) - cropped_chunk_length)] + chunks_buffer_size = len(chunk) - cropped_chunk_length + if not drop_last_batch and chunks_buffer: + yield pa.Table.from_batches(chunks_buffer) diff --git a/lib/python3.12/site-packages/datasets/utils/__init__.py b/lib/python3.12/site-packages/datasets/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a1c3d2935994956e230a441f3f2d65695ebaaf2d --- /dev/null +++ b/lib/python3.12/site-packages/datasets/utils/__init__.py @@ -0,0 +1,25 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from . import tqdm as _tqdm # _tqdm is the module +from .experimental import experimental +from .info_utils import VerificationMode +from .logging import disable_progress_bar, enable_progress_bar, is_progress_bar_enabled +from .tqdm import ( + are_progress_bars_disabled, + disable_progress_bars, + enable_progress_bars, + tqdm, +) +from .version import Version diff --git a/lib/python3.12/site-packages/datasets/utils/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/datasets/utils/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9d3352f09fb5ba2c184502f2c123f757c94fc9ea Binary files /dev/null and b/lib/python3.12/site-packages/datasets/utils/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/utils/__pycache__/_dataset_viewer.cpython-312.pyc b/lib/python3.12/site-packages/datasets/utils/__pycache__/_dataset_viewer.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0055b19c727279b2501ba53e1c7f137ccd27291f Binary files /dev/null and b/lib/python3.12/site-packages/datasets/utils/__pycache__/_dataset_viewer.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/utils/__pycache__/_dill.cpython-312.pyc b/lib/python3.12/site-packages/datasets/utils/__pycache__/_dill.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..060350930fbcfccabd4f31d761f3330645123970 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/utils/__pycache__/_dill.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/utils/__pycache__/_filelock.cpython-312.pyc b/lib/python3.12/site-packages/datasets/utils/__pycache__/_filelock.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e36a66d3f5e32ab6f516cb4a843a69c3e58e042a Binary files /dev/null and b/lib/python3.12/site-packages/datasets/utils/__pycache__/_filelock.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/utils/__pycache__/deprecation_utils.cpython-312.pyc b/lib/python3.12/site-packages/datasets/utils/__pycache__/deprecation_utils.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..df023c6d13228c72706e334dab579c0fe944339b Binary files /dev/null and b/lib/python3.12/site-packages/datasets/utils/__pycache__/deprecation_utils.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/utils/__pycache__/doc_utils.cpython-312.pyc b/lib/python3.12/site-packages/datasets/utils/__pycache__/doc_utils.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f36ec5b4853d03e6b059ee10b48559b2e10ae10d Binary files /dev/null and b/lib/python3.12/site-packages/datasets/utils/__pycache__/doc_utils.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/utils/__pycache__/experimental.cpython-312.pyc b/lib/python3.12/site-packages/datasets/utils/__pycache__/experimental.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0798c21ef598eb7d22cafd572abf222500a3068e Binary files /dev/null and b/lib/python3.12/site-packages/datasets/utils/__pycache__/experimental.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/utils/__pycache__/extract.cpython-312.pyc b/lib/python3.12/site-packages/datasets/utils/__pycache__/extract.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7e7c8eaaeaef3b9d42a1eff3da5e00eab371054b Binary files /dev/null and b/lib/python3.12/site-packages/datasets/utils/__pycache__/extract.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/utils/__pycache__/file_utils.cpython-312.pyc b/lib/python3.12/site-packages/datasets/utils/__pycache__/file_utils.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2f5cd0c59d99f032da88992662cfe2c878ea2f43 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/utils/__pycache__/file_utils.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/utils/__pycache__/filelock.cpython-312.pyc b/lib/python3.12/site-packages/datasets/utils/__pycache__/filelock.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..002f53288fff910be1b7430aece8647011d798ca Binary files /dev/null and b/lib/python3.12/site-packages/datasets/utils/__pycache__/filelock.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/utils/__pycache__/hub.cpython-312.pyc b/lib/python3.12/site-packages/datasets/utils/__pycache__/hub.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8a596b68533da0a89c28a0eaaa750fa184161a60 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/utils/__pycache__/hub.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/utils/__pycache__/info_utils.cpython-312.pyc b/lib/python3.12/site-packages/datasets/utils/__pycache__/info_utils.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..62242bdbb563361d2864fc774600256ba63f8930 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/utils/__pycache__/info_utils.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/utils/__pycache__/logging.cpython-312.pyc b/lib/python3.12/site-packages/datasets/utils/__pycache__/logging.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..91807a3f0b08deb88b289063da96260c924deaf4 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/utils/__pycache__/logging.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/utils/__pycache__/metadata.cpython-312.pyc b/lib/python3.12/site-packages/datasets/utils/__pycache__/metadata.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..be87c2025ad7658eff9e98f8a2fa49aa37ed4ea1 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/utils/__pycache__/metadata.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/utils/__pycache__/patching.cpython-312.pyc b/lib/python3.12/site-packages/datasets/utils/__pycache__/patching.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0fe65c3cdeb731ece01abb9746f54b2ca47098e9 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/utils/__pycache__/patching.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/utils/__pycache__/py_utils.cpython-312.pyc b/lib/python3.12/site-packages/datasets/utils/__pycache__/py_utils.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b6cdb0e174b216f982eb97d521a0ff27dde991d7 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/utils/__pycache__/py_utils.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/utils/__pycache__/sharding.cpython-312.pyc b/lib/python3.12/site-packages/datasets/utils/__pycache__/sharding.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d8a4565dfb02dabaf1658aeafa597ac8046f943f Binary files /dev/null and b/lib/python3.12/site-packages/datasets/utils/__pycache__/sharding.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/utils/__pycache__/stratify.cpython-312.pyc b/lib/python3.12/site-packages/datasets/utils/__pycache__/stratify.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b8aee0cb473b8fe008a8ef3522681139eab8d013 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/utils/__pycache__/stratify.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/utils/__pycache__/tf_utils.cpython-312.pyc b/lib/python3.12/site-packages/datasets/utils/__pycache__/tf_utils.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..39f529f3bae47652854167b0dad35a52037772ed Binary files /dev/null and b/lib/python3.12/site-packages/datasets/utils/__pycache__/tf_utils.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/utils/__pycache__/tqdm.cpython-312.pyc b/lib/python3.12/site-packages/datasets/utils/__pycache__/tqdm.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..67298c4e30b87966f8afe2ecace6a8f0c018cedb Binary files /dev/null and b/lib/python3.12/site-packages/datasets/utils/__pycache__/tqdm.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/utils/__pycache__/track.cpython-312.pyc b/lib/python3.12/site-packages/datasets/utils/__pycache__/track.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..da5b4354cd87b1d4e45532baa7b0f96b27a65448 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/utils/__pycache__/track.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/utils/__pycache__/typing.cpython-312.pyc b/lib/python3.12/site-packages/datasets/utils/__pycache__/typing.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..510b50476089487fbe80bff82ecfe1440c0eed60 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/utils/__pycache__/typing.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/utils/__pycache__/version.cpython-312.pyc b/lib/python3.12/site-packages/datasets/utils/__pycache__/version.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0bb10d4c822881acdf78a5d96e1f4884c8785f98 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/utils/__pycache__/version.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/utils/_dataset_viewer.py b/lib/python3.12/site-packages/datasets/utils/_dataset_viewer.py new file mode 100644 index 0000000000000000000000000000000000000000..0f4c298646c5534a93ecfd217b84987b4786cec6 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/utils/_dataset_viewer.py @@ -0,0 +1,95 @@ +from typing import Any, Optional, Union + +from huggingface_hub.utils import get_session + +from .. import config +from ..exceptions import DatasetsError +from .file_utils import ( + get_authentication_headers_for_url, +) +from .logging import get_logger + + +logger = get_logger(__name__) + + +class DatasetViewerError(DatasetsError): + """Dataset viewer error. + + Raised when trying to use the dataset viewer HTTP API and when trying to access: + - a missing dataset, or + - a private/gated dataset and the user is not authenticated. + - unavailable /parquet or /info responses + """ + + +def get_exported_parquet_files( + dataset: str, commit_hash: str, token: Optional[Union[str, bool]] +) -> list[dict[str, Any]]: + """ + Get the dataset exported parquet files + Docs: https://huggingface.co/docs/datasets-server/parquet + """ + dataset_viewer_parquet_url = config.HF_ENDPOINT.replace("://", "://datasets-server.") + "/parquet?dataset=" + try: + parquet_data_files_response = get_session().get( + url=dataset_viewer_parquet_url + dataset, + headers=get_authentication_headers_for_url(config.HF_ENDPOINT + f"datasets/{dataset}", token=token), + timeout=100.0, + ) + parquet_data_files_response.raise_for_status() + if "X-Revision" in parquet_data_files_response.headers: + if parquet_data_files_response.headers["X-Revision"] == commit_hash or commit_hash is None: + parquet_data_files_response_json = parquet_data_files_response.json() + if ( + parquet_data_files_response_json.get("partial") is False + and not parquet_data_files_response_json.get("pending", True) + and not parquet_data_files_response_json.get("failed", True) + and "parquet_files" in parquet_data_files_response_json + ): + return parquet_data_files_response_json["parquet_files"] + else: + logger.debug(f"Parquet export for {dataset} is not completely ready yet.") + else: + logger.debug( + f"Parquet export for {dataset} is available but outdated (commit_hash='{parquet_data_files_response.headers['X-Revision']}')" + ) + except Exception as e: # noqa catch any exception of the dataset viewer API and consider the parquet export doesn't exist + logger.debug(f"No parquet export for {dataset} available ({type(e).__name__}: {e})") + raise DatasetViewerError("No exported Parquet files available.") + + +def get_exported_dataset_infos( + dataset: str, commit_hash: str, token: Optional[Union[str, bool]] +) -> dict[str, dict[str, Any]]: + """ + Get the dataset information, can be useful to get e.g. the dataset features. + Docs: https://huggingface.co/docs/datasets-server/info + """ + dataset_viewer_info_url = config.HF_ENDPOINT.replace("://", "://datasets-server.") + "/info?dataset=" + try: + info_response = get_session().get( + url=dataset_viewer_info_url + dataset, + headers=get_authentication_headers_for_url(config.HF_ENDPOINT + f"datasets/{dataset}", token=token), + timeout=100.0, + ) + info_response.raise_for_status() + if "X-Revision" in info_response.headers: + if info_response.headers["X-Revision"] == commit_hash or commit_hash is None: + info_response = info_response.json() + if ( + info_response.get("partial") is False + and not info_response.get("pending", True) + and not info_response.get("failed", True) + and "dataset_info" in info_response + ): + return info_response["dataset_info"] + else: + logger.debug(f"Dataset info for {dataset} is not completely ready yet.") + else: + logger.debug( + f"Dataset info for {dataset} is available but outdated (commit_hash='{info_response.headers['X-Revision']}')" + ) + except Exception as e: # noqa catch any exception of the dataset viewer API and consider the dataset info doesn't exist + logger.debug(f"No dataset info for {dataset} available ({type(e).__name__}: {e})") + raise DatasetViewerError("No exported dataset infos available.") diff --git a/lib/python3.12/site-packages/datasets/utils/_dill.py b/lib/python3.12/site-packages/datasets/utils/_dill.py new file mode 100644 index 0000000000000000000000000000000000000000..2a414459266809933b9e63f1195b4c86cb7ea44b --- /dev/null +++ b/lib/python3.12/site-packages/datasets/utils/_dill.py @@ -0,0 +1,465 @@ +# Copyright 2023 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Extends `dill` to support pickling more types and produce more consistent dumps.""" + +import os +import sys +from io import BytesIO +from types import CodeType, FunctionType + +import dill +from packaging import version + +from .. import config + + +class Pickler(dill.Pickler): + dispatch = dill._dill.MetaCatchingDict(dill.Pickler.dispatch.copy()) + _legacy_no_dict_keys_sorting = False + + def save(self, obj, save_persistent_id=True): + obj_type = type(obj) + if obj_type not in self.dispatch: + if "regex" in sys.modules: + import regex # type: ignore + + if obj_type is regex.Pattern: + pklregister(obj_type)(_save_regexPattern) + if "spacy" in sys.modules: + import spacy # type: ignore + + if issubclass(obj_type, spacy.Language): + pklregister(obj_type)(_save_spacyLanguage) + if "tiktoken" in sys.modules: + import tiktoken # type: ignore + + if obj_type is tiktoken.Encoding: + pklregister(obj_type)(_save_tiktokenEncoding) + if "torch" in sys.modules: + import torch # type: ignore + + if issubclass(obj_type, torch.Tensor): + pklregister(obj_type)(_save_torchTensor) + + if obj_type is torch.Generator: + pklregister(obj_type)(_save_torchGenerator) + + # Unwrap `torch.compile`-ed modules + if issubclass(obj_type, torch.nn.Module): + obj = getattr(obj, "_orig_mod", obj) + if "transformers" in sys.modules: + import transformers # type: ignore + + if issubclass(obj_type, transformers.PreTrainedTokenizerBase): + pklregister(obj_type)(_save_transformersPreTrainedTokenizerBase) + + # Unwrap `torch.compile`-ed functions + if obj_type is FunctionType: + obj = getattr(obj, "_torchdynamo_orig_callable", obj) + dill.Pickler.save(self, obj, save_persistent_id=save_persistent_id) + + def _batch_setitems(self, items): + if self._legacy_no_dict_keys_sorting: + return super()._batch_setitems(items) + # Ignore the order of keys in a dict + try: + # Faster, but fails for unorderable elements + items = sorted(items) + except Exception: # TypeError, decimal.InvalidOperation, etc. + from datasets.fingerprint import Hasher + + items = sorted(items, key=lambda x: Hasher.hash(x[0])) + dill.Pickler._batch_setitems(self, items) + + def memoize(self, obj): + # Don't memoize strings since two identical strings can have different Python ids + if type(obj) is not str: # noqa: E721 + dill.Pickler.memoize(self, obj) + + +def pklregister(t): + """Register a custom reducer for the type.""" + + def proxy(func): + Pickler.dispatch[t] = func + return func + + return proxy + + +def dump(obj, file): + """Pickle an object to a file.""" + Pickler(file, recurse=True).dump(obj) + + +def dumps(obj): + """Pickle an object to a string.""" + file = BytesIO() + dump(obj, file) + return file.getvalue() + + +if config.DILL_VERSION < version.parse("0.3.6"): + + def log(pickler, msg): + dill._dill.log.info(msg) + +elif config.DILL_VERSION.release[:3] in [ + version.parse("0.3.6").release, + version.parse("0.3.7").release, + version.parse("0.3.8").release, +]: + + def log(pickler, msg): + dill._dill.logger.trace(pickler, msg) + + +@pklregister(set) +def _save_set(pickler, obj): + log(pickler, f"Se: {obj}") + try: + # Faster, but fails for unorderable elements + args = (sorted(obj),) + except Exception: # TypeError, decimal.InvalidOperation, etc. + from datasets.fingerprint import Hasher + + args = (sorted(obj, key=Hasher.hash),) + + pickler.save_reduce(set, args, obj=obj) + log(pickler, "# Se") + + +def _save_regexPattern(pickler, obj): + import regex # type: ignore + + log(pickler, f"Re: {obj}") + args = (obj.pattern, obj.flags) + pickler.save_reduce(regex.compile, args, obj=obj) + log(pickler, "# Re") + + +def _save_tiktokenEncoding(pickler, obj): + import tiktoken # type: ignore + + log(pickler, f"Enc: {obj}") + args = (obj.name, obj._pat_str, obj._mergeable_ranks, obj._special_tokens) + pickler.save_reduce(tiktoken.Encoding, args, obj=obj) + log(pickler, "# Enc") + + +def _save_torchTensor(pickler, obj): + import torch # type: ignore + + # `torch.from_numpy` is not picklable in `torch>=1.11.0` + def create_torchTensor(np_array, dtype=None): + tensor = torch.from_numpy(np_array) + if dtype: + tensor = tensor.type(dtype) + return tensor + + log(pickler, f"To: {obj}") + if obj.dtype == torch.bfloat16: + args = (obj.detach().to(torch.float).cpu().numpy(), torch.bfloat16) + else: + args = (obj.detach().cpu().numpy(),) + pickler.save_reduce(create_torchTensor, args, obj=obj) + log(pickler, "# To") + + +def _save_torchGenerator(pickler, obj): + import torch # type: ignore + + def create_torchGenerator(state): + generator = torch.Generator() + generator.set_state(state) + return generator + + log(pickler, f"Ge: {obj}") + args = (obj.get_state(),) + pickler.save_reduce(create_torchGenerator, args, obj=obj) + log(pickler, "# Ge") + + +def _save_spacyLanguage(pickler, obj): + import spacy # type: ignore + + def create_spacyLanguage(config, bytes): + lang_cls = spacy.util.get_lang_class(config["nlp"]["lang"]) + lang_inst = lang_cls.from_config(config) + return lang_inst.from_bytes(bytes) + + log(pickler, f"Sp: {obj}") + args = (obj.config, obj.to_bytes()) + pickler.save_reduce(create_spacyLanguage, args, obj=obj) + log(pickler, "# Sp") + + +def _save_transformersPreTrainedTokenizerBase(pickler, obj): + log(pickler, f"Tok: {obj}") + # Ignore the `cache` attribute + state = obj.__dict__ + if "cache" in state and isinstance(state["cache"], dict): + state["cache"] = {} + pickler.save_reduce(type(obj), (), state=state, obj=obj) + log(pickler, "# Tok") + + +if config.DILL_VERSION < version.parse("0.3.6"): + + @pklregister(CodeType) + def _save_code(pickler, obj): + """ + From dill._dill.save_code + This is a modified version that removes the origin (filename + line no.) + of functions created in notebooks or shells for example. + """ + dill._dill.log.info(f"Co: {obj}") + # The filename of a function is the .py file where it is defined. + # Filenames of functions created in notebooks or shells start with '<' + # ex: for ipython, and for shell + # Filenames of functions created in ipykernel the filename + # look like f"{tempdir}/ipykernel_{id1}/{id2}.py" + # Moreover lambda functions have a special name: '' + # ex: (lambda x: x).__code__.co_name == "" # True + # + # For the hashing mechanism we ignore where the function has been defined + # More specifically: + # - we ignore the filename of special functions (filename starts with '<') + # - we always ignore the line number + # - we only use the base name of the file instead of the whole path, + # to be robust in case a script is moved for example. + # + # Only those two lines are different from the original implementation: + co_filename = ( + "" + if obj.co_filename.startswith("<") + or ( + len(obj.co_filename.split(os.path.sep)) > 1 + and obj.co_filename.split(os.path.sep)[-2].startswith("ipykernel_") + ) + or obj.co_name == "" + else os.path.basename(obj.co_filename) + ) + co_firstlineno = 1 + # The rest is the same as in the original dill implementation + if dill._dill.PY3: + if hasattr(obj, "co_posonlyargcount"): + args = ( + obj.co_argcount, + obj.co_posonlyargcount, + obj.co_kwonlyargcount, + obj.co_nlocals, + obj.co_stacksize, + obj.co_flags, + obj.co_code, + obj.co_consts, + obj.co_names, + obj.co_varnames, + co_filename, + obj.co_name, + co_firstlineno, + obj.co_lnotab, + obj.co_freevars, + obj.co_cellvars, + ) + else: + args = ( + obj.co_argcount, + obj.co_kwonlyargcount, + obj.co_nlocals, + obj.co_stacksize, + obj.co_flags, + obj.co_code, + obj.co_consts, + obj.co_names, + obj.co_varnames, + co_filename, + obj.co_name, + co_firstlineno, + obj.co_lnotab, + obj.co_freevars, + obj.co_cellvars, + ) + else: + args = ( + obj.co_argcount, + obj.co_nlocals, + obj.co_stacksize, + obj.co_flags, + obj.co_code, + obj.co_consts, + obj.co_names, + obj.co_varnames, + co_filename, + obj.co_name, + co_firstlineno, + obj.co_lnotab, + obj.co_freevars, + obj.co_cellvars, + ) + pickler.save_reduce(CodeType, args, obj=obj) + dill._dill.log.info("# Co") + return + +elif config.DILL_VERSION.release[:3] in [ + version.parse("0.3.6").release, + version.parse("0.3.7").release, + version.parse("0.3.8").release, +]: + # From: https://github.com/uqfoundation/dill/blob/dill-0.3.6/dill/_dill.py#L1104 + @pklregister(CodeType) + def save_code(pickler, obj): + dill._dill.logger.trace(pickler, "Co: %s", obj) + + ############################################################################################################ + # Modification here for huggingface/datasets + # The filename of a function is the .py file where it is defined. + # Filenames of functions created in notebooks or shells start with '<' + # ex: for ipython, and for shell + # Filenames of functions created in ipykernel the filename + # look like f"{tempdir}/ipykernel_{id1}/{id2}.py" + # Moreover lambda functions have a special name: '' + # ex: (lambda x: x).__code__.co_name == "" # True + # + # For the hashing mechanism we ignore where the function has been defined + # More specifically: + # - we ignore the filename of special functions (filename starts with '<') + # - we always ignore the line number + # - we only use the base name of the file instead of the whole path, + # to be robust in case a script is moved for example. + # + # Only those two lines are different from the original implementation: + co_filename = ( + "" + if obj.co_filename.startswith("<") + or ( + len(obj.co_filename.split(os.path.sep)) > 1 + and obj.co_filename.split(os.path.sep)[-2].startswith("ipykernel_") + ) + or obj.co_name == "" + else os.path.basename(obj.co_filename) + ) + co_firstlineno = 1 + # The rest is the same as in the original dill implementation, except for the replacements: + # - obj.co_filename => co_filename + # - obj.co_firstlineno => co_firstlineno + ############################################################################################################ + + if hasattr(obj, "co_endlinetable"): # python 3.11a (20 args) + args = ( + obj.co_lnotab, # for < python 3.10 [not counted in args] + obj.co_argcount, + obj.co_posonlyargcount, + obj.co_kwonlyargcount, + obj.co_nlocals, + obj.co_stacksize, + obj.co_flags, + obj.co_code, + obj.co_consts, + obj.co_names, + obj.co_varnames, + co_filename, # Modification for huggingface/datasets ############################################ + obj.co_name, + obj.co_qualname, + co_firstlineno, # Modification for huggingface/datasets ######################################### + obj.co_linetable, + obj.co_endlinetable, + obj.co_columntable, + obj.co_exceptiontable, + obj.co_freevars, + obj.co_cellvars, + ) + elif hasattr(obj, "co_exceptiontable"): # python 3.11 (18 args) + args = ( + obj.co_lnotab, # for < python 3.10 [not counted in args] + obj.co_argcount, + obj.co_posonlyargcount, + obj.co_kwonlyargcount, + obj.co_nlocals, + obj.co_stacksize, + obj.co_flags, + obj.co_code, + obj.co_consts, + obj.co_names, + obj.co_varnames, + co_filename, # Modification for huggingface/datasets ############################################ + obj.co_name, + obj.co_qualname, + co_firstlineno, # Modification for huggingface/datasets ######################################### + obj.co_linetable, + obj.co_exceptiontable, + obj.co_freevars, + obj.co_cellvars, + ) + elif hasattr(obj, "co_linetable"): # python 3.10 (16 args) + args = ( + obj.co_lnotab, # for < python 3.10 [not counted in args] + obj.co_argcount, + obj.co_posonlyargcount, + obj.co_kwonlyargcount, + obj.co_nlocals, + obj.co_stacksize, + obj.co_flags, + obj.co_code, + obj.co_consts, + obj.co_names, + obj.co_varnames, + co_filename, # Modification for huggingface/datasets ############################################ + obj.co_name, + co_firstlineno, # Modification for huggingface/datasets ######################################### + obj.co_linetable, + obj.co_freevars, + obj.co_cellvars, + ) + elif hasattr(obj, "co_posonlyargcount"): # python 3.8 (16 args) + args = ( + obj.co_argcount, + obj.co_posonlyargcount, + obj.co_kwonlyargcount, + obj.co_nlocals, + obj.co_stacksize, + obj.co_flags, + obj.co_code, + obj.co_consts, + obj.co_names, + obj.co_varnames, + co_filename, # Modification for huggingface/datasets ############################################ + obj.co_name, + co_firstlineno, # Modification for huggingface/datasets ######################################### + obj.co_lnotab, + obj.co_freevars, + obj.co_cellvars, + ) + else: # python 3.7 (15 args) + args = ( + obj.co_argcount, + obj.co_kwonlyargcount, + obj.co_nlocals, + obj.co_stacksize, + obj.co_flags, + obj.co_code, + obj.co_consts, + obj.co_names, + obj.co_varnames, + co_filename, # Modification for huggingface/datasets ############################################ + obj.co_name, + co_firstlineno, # Modification for huggingface/datasets ######################################### + obj.co_lnotab, + obj.co_freevars, + obj.co_cellvars, + ) + + pickler.save_reduce(dill._dill._create_code, args, obj=obj) + dill._dill.logger.trace(pickler, "# Co") + return diff --git a/lib/python3.12/site-packages/datasets/utils/_filelock.py b/lib/python3.12/site-packages/datasets/utils/_filelock.py new file mode 100644 index 0000000000000000000000000000000000000000..803ce77f5a4168ac9023e1d4a4548db2bd154126 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/utils/_filelock.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +# Copyright 2023 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License +"""Utilities to handle file locking in `datasets`.""" + +import os + +from filelock import FileLock as FileLock_ +from filelock import UnixFileLock +from filelock import __version__ as _filelock_version +from packaging import version + + +class FileLock(FileLock_): + """ + A `filelock.FileLock` initializer that handles long paths. + It also uses the current umask for lock files. + """ + + MAX_FILENAME_LENGTH = 255 + + def __init__(self, lock_file, *args, **kwargs): + # The "mode" argument is required if we want to use the current umask in filelock >= 3.10 + # In previous previous it was already using the current umask. + if "mode" not in kwargs and version.parse(_filelock_version) >= version.parse("3.10.0"): + umask = os.umask(0o666) + os.umask(umask) + kwargs["mode"] = 0o666 & ~umask + lock_file = self.hash_filename_if_too_long(lock_file) + super().__init__(lock_file, *args, **kwargs) + + @classmethod + def hash_filename_if_too_long(cls, path: str) -> str: + path = os.path.abspath(os.path.expanduser(path)) + filename = os.path.basename(path) + max_filename_length = cls.MAX_FILENAME_LENGTH + if issubclass(cls, UnixFileLock): + max_filename_length = min(max_filename_length, os.statvfs(os.path.dirname(path)).f_namemax) + if len(filename) > max_filename_length: + dirname = os.path.dirname(path) + hashed_filename = str(hash(filename)) + new_filename = ( + filename[: max_filename_length - len(hashed_filename) - 8] + "..." + hashed_filename + ".lock" + ) + return os.path.join(dirname, new_filename) + else: + return path diff --git a/lib/python3.12/site-packages/datasets/utils/deprecation_utils.py b/lib/python3.12/site-packages/datasets/utils/deprecation_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..f05ecbeaa3eae5476e99c461dbede9ebfa111eb0 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/utils/deprecation_utils.py @@ -0,0 +1,105 @@ +import enum +import inspect +import warnings +from functools import wraps +from typing import Callable, Optional + +from .logging import get_logger + + +_emitted_deprecation_warnings = set() +logger = get_logger(__name__) + + +def deprecated(help_message: Optional[str] = None): + """Decorator to mark a class or a function as deprecated. + + Args: + help_message (:obj:`str`, optional): An optional message to guide the user on how to + switch to non-deprecated usage of the library. + """ + + def decorator(deprecated_class_or_function: Callable): + global _emitted_deprecation_warnings + + if inspect.isclass(deprecated_class_or_function): + deprecated_function = deprecated_class_or_function.__init__ + name = deprecated_class_or_function.__name__ + else: + deprecated_function = deprecated_class_or_function + name = deprecated_function.__name__ + # Support deprecating __init__ class method: class name instead + name = name if name != "__init__" else deprecated_function.__qualname__.split(".")[-2] + + warning_msg = ( + f"{name} is deprecated and will be removed in the next major version of datasets." + f" {help_message}" + if help_message + else "" + ) + + @wraps(deprecated_function) + def wrapper(*args, **kwargs): + func_hash = hash(deprecated_function) + if func_hash not in _emitted_deprecation_warnings: + warnings.warn(warning_msg, category=FutureWarning, stacklevel=2) + _emitted_deprecation_warnings.add(func_hash) + return deprecated_function(*args, **kwargs) + + wrapper._decorator_name_ = "deprecated" + + if inspect.isclass(deprecated_class_or_function): + deprecated_class_or_function.__init__ = wrapper + return deprecated_class_or_function + else: + return wrapper + + return decorator + + +class OnAccess(enum.EnumMeta): + """ + Enum metaclass that calls a user-specified function whenever a member is accessed. + """ + + def __getattribute__(cls, name): + obj = super().__getattribute__(name) + if isinstance(obj, enum.Enum) and obj._on_access: + obj._on_access() + return obj + + def __getitem__(cls, name): + member = super().__getitem__(name) + if member._on_access: + member._on_access() + return member + + def __call__(cls, value, names=None, *, module=None, qualname=None, type=None, start=1): + obj = super().__call__(value, names, module=module, qualname=qualname, type=type, start=start) + if isinstance(obj, enum.Enum) and obj._on_access: + obj._on_access() + return obj + + +class DeprecatedEnum(enum.Enum, metaclass=OnAccess): + """ + Enum class that calls `deprecate` method whenever a member is accessed. + """ + + def __new__(cls, value): + member = object.__new__(cls) + member._value_ = value + member._on_access = member.deprecate + return member + + @property + def help_message(self): + return "" + + def deprecate(self): + help_message = f" {self.help_message}" if self.help_message else "" + warnings.warn( + f"'{self.__objclass__.__name__}' is deprecated and will be removed in the next major version of datasets." + + help_message, + FutureWarning, + stacklevel=3, + ) diff --git a/lib/python3.12/site-packages/datasets/utils/doc_utils.py b/lib/python3.12/site-packages/datasets/utils/doc_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..6ef8bcb4e70725ad086cb817e0ec4551d1c0966e --- /dev/null +++ b/lib/python3.12/site-packages/datasets/utils/doc_utils.py @@ -0,0 +1,15 @@ +from typing import Callable + + +def is_documented_by(function_with_docstring: Callable): + """Decorator to share docstrings across common functions. + + Args: + function_with_docstring (`Callable`): Name of the function with the docstring. + """ + + def wrapper(target_function): + target_function.__doc__ = function_with_docstring.__doc__ + return target_function + + return wrapper diff --git a/lib/python3.12/site-packages/datasets/utils/experimental.py b/lib/python3.12/site-packages/datasets/utils/experimental.py new file mode 100644 index 0000000000000000000000000000000000000000..cc406154e9347f4df83b1f7b08c32a961d469f6a --- /dev/null +++ b/lib/python3.12/site-packages/datasets/utils/experimental.py @@ -0,0 +1,43 @@ +"""Contains utilities to flag a feature as "experimental" in datasets.""" + +import warnings +from functools import wraps +from typing import Callable + + +def experimental(fn: Callable) -> Callable: + """Decorator to flag a feature as experimental. + + An experimental feature trigger a warning when used as it might be subject to breaking changes in the future. + + Args: + fn (`Callable`): + The function to flag as experimental. + + Returns: + `Callable`: The decorated function. + + Example: + + ```python + >>> from datasets.utils import experimental + + >>> @experimental + ... def my_function(): + ... print("Hello world!") + + >>> my_function() + UserWarning: 'my_function' is experimental and might be subject to breaking changes in the future. + Hello world! + ``` + """ + + @wraps(fn) + def _inner_fn(*args, **kwargs): + warnings.warn( + (f"'{fn.__name__}' is experimental and might be subject to breaking changes in the future."), + UserWarning, + ) + return fn(*args, **kwargs) + + return _inner_fn diff --git a/lib/python3.12/site-packages/datasets/utils/extract.py b/lib/python3.12/site-packages/datasets/utils/extract.py new file mode 100644 index 0000000000000000000000000000000000000000..1e87c617217b2b93fd6273804c2ba553ce42eb68 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/utils/extract.py @@ -0,0 +1,332 @@ +import bz2 +import gzip +import lzma +import os +import shutil +import struct +import tarfile +import warnings +import zipfile +from abc import ABC, abstractmethod +from pathlib import Path +from typing import Optional, Union + +from .. import config +from ._filelock import FileLock +from .logging import get_logger + + +logger = get_logger(__name__) + + +class ExtractManager: + def __init__(self, cache_dir: Optional[str] = None): + self.extract_dir = ( + os.path.join(cache_dir, config.EXTRACTED_DATASETS_DIR) if cache_dir else config.EXTRACTED_DATASETS_PATH + ) + self.extractor = Extractor + + def _get_output_path(self, path: str) -> str: + from .file_utils import hash_url_to_filename + + # Path where we extract compressed archives + # We extract in the cache dir, and get the extracted path name by hashing the original path" + abs_path = os.path.abspath(path) + return os.path.join(self.extract_dir, hash_url_to_filename(abs_path)) + + def _do_extract(self, output_path: str, force_extract: bool) -> bool: + return force_extract or ( + not os.path.isfile(output_path) and not (os.path.isdir(output_path) and os.listdir(output_path)) + ) + + def extract(self, input_path: str, force_extract: bool = False) -> str: + extractor_format = self.extractor.infer_extractor_format(input_path) + if not extractor_format: + return input_path + output_path = self._get_output_path(input_path) + if self._do_extract(output_path, force_extract): + self.extractor.extract(input_path, output_path, extractor_format) + return output_path + + +class BaseExtractor(ABC): + @classmethod + @abstractmethod + def is_extractable(cls, path: Union[Path, str], **kwargs) -> bool: ... + + @staticmethod + @abstractmethod + def extract(input_path: Union[Path, str], output_path: Union[Path, str]) -> None: ... + + +class MagicNumberBaseExtractor(BaseExtractor, ABC): + magic_numbers: list[bytes] = [] + + @staticmethod + def read_magic_number(path: Union[Path, str], magic_number_length: int): + with open(path, "rb") as f: + return f.read(magic_number_length) + + @classmethod + def is_extractable(cls, path: Union[Path, str], magic_number: bytes = b"") -> bool: + if not magic_number: + magic_number_length = max(len(cls_magic_number) for cls_magic_number in cls.magic_numbers) + try: + magic_number = cls.read_magic_number(path, magic_number_length) + except OSError: + return False + return any(magic_number.startswith(cls_magic_number) for cls_magic_number in cls.magic_numbers) + + +class TarExtractor(BaseExtractor): + @classmethod + def is_extractable(cls, path: Union[Path, str], **kwargs) -> bool: + return tarfile.is_tarfile(path) + + @staticmethod + def safemembers(members, output_path): + """ + Fix for CVE-2007-4559 + Desc: + Directory traversal vulnerability in the (1) extract and (2) extractall functions in the tarfile + module in Python allows user-assisted remote attackers to overwrite arbitrary files via a .. (dot dot) + sequence in filenames in a TAR archive, a related issue to CVE-2001-1267. + See: https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2007-4559 + From: https://stackoverflow.com/a/10077309 + """ + + def resolved(path: str) -> str: + return os.path.realpath(os.path.abspath(path)) + + def badpath(path: str, base: str) -> bool: + # joinpath will ignore base if path is absolute + return not resolved(os.path.join(base, path)).startswith(base) + + def badlink(info, base: str) -> bool: + # Links are interpreted relative to the directory containing the link + tip = resolved(os.path.join(base, os.path.dirname(info.name))) + return badpath(info.linkname, base=tip) + + base = resolved(output_path) + + for finfo in members: + if badpath(finfo.name, base): + logger.error(f"Extraction of {finfo.name} is blocked (illegal path)") + elif finfo.issym() and badlink(finfo, base): + logger.error(f"Extraction of {finfo.name} is blocked: Symlink to {finfo.linkname}") + elif finfo.islnk() and badlink(finfo, base): + logger.error(f"Extraction of {finfo.name} is blocked: Hard link to {finfo.linkname}") + else: + yield finfo + + @staticmethod + def extract(input_path: Union[Path, str], output_path: Union[Path, str]) -> None: + os.makedirs(output_path, exist_ok=True) + tar_file = tarfile.open(input_path) + tar_file.extractall(output_path, members=TarExtractor.safemembers(tar_file, output_path)) + tar_file.close() + + +class GzipExtractor(MagicNumberBaseExtractor): + magic_numbers = [b"\x1f\x8b"] + + @staticmethod + def extract(input_path: Union[Path, str], output_path: Union[Path, str]) -> None: + with gzip.open(input_path, "rb") as gzip_file: + with open(output_path, "wb") as extracted_file: + shutil.copyfileobj(gzip_file, extracted_file) + + +class ZipExtractor(MagicNumberBaseExtractor): + magic_numbers = [ + b"PK\x03\x04", + b"PK\x05\x06", # empty archive + b"PK\x07\x08", # spanned archive + ] + + @classmethod + def is_extractable(cls, path: Union[Path, str], magic_number: bytes = b"") -> bool: + if super().is_extractable(path, magic_number=magic_number): + return True + try: + # Alternative version of zipfile.is_zipfile that has less false positives, but misses executable zip archives. + # From: https://github.com/python/cpython/pull/5053 + from zipfile import ( + _CD_SIGNATURE, + _ECD_DISK_NUMBER, + _ECD_DISK_START, + _ECD_ENTRIES_TOTAL, + _ECD_OFFSET, + _ECD_SIZE, + _EndRecData, + sizeCentralDir, + stringCentralDir, + structCentralDir, + ) + + with open(path, "rb") as fp: + endrec = _EndRecData(fp) + if endrec: + if endrec[_ECD_ENTRIES_TOTAL] == 0 and endrec[_ECD_SIZE] == 0 and endrec[_ECD_OFFSET] == 0: + return True # Empty zipfiles are still zipfiles + elif endrec[_ECD_DISK_NUMBER] == endrec[_ECD_DISK_START]: + fp.seek(endrec[_ECD_OFFSET]) # Central directory is on the same disk + if fp.tell() == endrec[_ECD_OFFSET] and endrec[_ECD_SIZE] >= sizeCentralDir: + data = fp.read(sizeCentralDir) # CD is where we expect it to be + if len(data) == sizeCentralDir: + centdir = struct.unpack(structCentralDir, data) # CD is the right size + if centdir[_CD_SIGNATURE] == stringCentralDir: + return True # First central directory entry has correct magic number + return False + except Exception: # catch all errors in case future python versions change the zipfile internals + return False + + @staticmethod + def extract(input_path: Union[Path, str], output_path: Union[Path, str]) -> None: + os.makedirs(output_path, exist_ok=True) + with zipfile.ZipFile(input_path, "r") as zip_file: + zip_file.extractall(output_path) + zip_file.close() + + +class XzExtractor(MagicNumberBaseExtractor): + magic_numbers = [b"\xfd\x37\x7a\x58\x5a\x00"] + + @staticmethod + def extract(input_path: Union[Path, str], output_path: Union[Path, str]) -> None: + with lzma.open(input_path) as compressed_file: + with open(output_path, "wb") as extracted_file: + shutil.copyfileobj(compressed_file, extracted_file) + + +class RarExtractor(MagicNumberBaseExtractor): + magic_numbers = [b"Rar!\x1a\x07\x00", b"Rar!\x1a\x07\x01\x00"] # RAR_ID # RAR5_ID + + @staticmethod + def extract(input_path: Union[Path, str], output_path: Union[Path, str]) -> None: + if not config.RARFILE_AVAILABLE: + raise ImportError("Please pip install rarfile") + import rarfile + + os.makedirs(output_path, exist_ok=True) + rf = rarfile.RarFile(input_path) + rf.extractall(output_path) + rf.close() + + +class ZstdExtractor(MagicNumberBaseExtractor): + magic_numbers = [b"\x28\xb5\x2f\xfd"] + + @staticmethod + def extract(input_path: Union[Path, str], output_path: Union[Path, str]) -> None: + if not config.ZSTANDARD_AVAILABLE: + raise ImportError("Please pip install zstandard") + import zstandard as zstd + + dctx = zstd.ZstdDecompressor() + with open(input_path, "rb") as ifh, open(output_path, "wb") as ofh: + dctx.copy_stream(ifh, ofh) + + +class Bzip2Extractor(MagicNumberBaseExtractor): + magic_numbers = [b"\x42\x5a\x68"] + + @staticmethod + def extract(input_path: Union[Path, str], output_path: Union[Path, str]) -> None: + with bz2.open(input_path, "rb") as compressed_file: + with open(output_path, "wb") as extracted_file: + shutil.copyfileobj(compressed_file, extracted_file) + + +class SevenZipExtractor(MagicNumberBaseExtractor): + magic_numbers = [b"\x37\x7a\xbc\xaf\x27\x1c"] + + @staticmethod + def extract(input_path: Union[Path, str], output_path: Union[Path, str]) -> None: + if not config.PY7ZR_AVAILABLE: + raise ImportError("Please pip install py7zr") + import py7zr + + os.makedirs(output_path, exist_ok=True) + with py7zr.SevenZipFile(input_path, "r") as archive: + archive.extractall(output_path) + + +class Lz4Extractor(MagicNumberBaseExtractor): + magic_numbers = [b"\x04\x22\x4d\x18"] + + @staticmethod + def extract(input_path: Union[Path, str], output_path: Union[Path, str]) -> None: + if not config.LZ4_AVAILABLE: + raise ImportError("Please pip install lz4") + import lz4.frame + + with lz4.frame.open(input_path, "rb") as compressed_file: + with open(output_path, "wb") as extracted_file: + shutil.copyfileobj(compressed_file, extracted_file) + + +class Extractor: + # Put zip file to the last, b/c it is possible wrongly detected as zip (I guess it means: as tar or gzip) + extractors: dict[str, type[BaseExtractor]] = { + "tar": TarExtractor, + "gzip": GzipExtractor, + "zip": ZipExtractor, + "xz": XzExtractor, + "rar": RarExtractor, + "zstd": ZstdExtractor, + "bz2": Bzip2Extractor, + "7z": SevenZipExtractor, # + "lz4": Lz4Extractor, # + } + + @classmethod + def _get_magic_number_max_length(cls): + return max( + len(extractor_magic_number) + for extractor in cls.extractors.values() + if issubclass(extractor, MagicNumberBaseExtractor) + for extractor_magic_number in extractor.magic_numbers + ) + + @staticmethod + def _read_magic_number(path: Union[Path, str], magic_number_length: int): + try: + return MagicNumberBaseExtractor.read_magic_number(path, magic_number_length=magic_number_length) + except OSError: + return b"" + + @classmethod + def is_extractable(cls, path: Union[Path, str], return_extractor: bool = False) -> bool: + warnings.warn( + "Method 'is_extractable' was deprecated in version 2.4.0 and will be removed in 3.0.0. " + "Use 'infer_extractor_format' instead.", + category=FutureWarning, + ) + extractor_format = cls.infer_extractor_format(path) + if extractor_format: + return True if not return_extractor else (True, cls.extractors[extractor_format]) + return False if not return_extractor else (False, None) + + @classmethod + def infer_extractor_format(cls, path: Union[Path, str]) -> Optional[str]: # + magic_number_max_length = cls._get_magic_number_max_length() + magic_number = cls._read_magic_number(path, magic_number_max_length) + for extractor_format, extractor in cls.extractors.items(): + if extractor.is_extractable(path, magic_number=magic_number): + return extractor_format + + @classmethod + def extract( + cls, + input_path: Union[Path, str], + output_path: Union[Path, str], + extractor_format: str, + ) -> None: + os.makedirs(os.path.dirname(output_path), exist_ok=True) + # Prevent parallel extractions + lock_path = str(Path(output_path).with_suffix(".lock")) + with FileLock(lock_path): + shutil.rmtree(output_path, ignore_errors=True) + extractor = cls.extractors[extractor_format] + return extractor.extract(input_path, output_path) diff --git a/lib/python3.12/site-packages/datasets/utils/file_utils.py b/lib/python3.12/site-packages/datasets/utils/file_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..462880d8eec928d2f6a3a7820d79123cb6dcb4cb --- /dev/null +++ b/lib/python3.12/site-packages/datasets/utils/file_utils.py @@ -0,0 +1,1383 @@ +""" +Utilities for working with the local dataset cache. +This file is adapted from the AllenNLP library at https://github.com/allenai/allennlp +Copyright by the AllenNLP authors. +""" + +import asyncio +import glob +import io +import json +import multiprocessing +import os +import posixpath +import re +import shutil +import sys +import tarfile +import time +import xml.dom.minidom +import zipfile +from collections.abc import Generator +from io import BytesIO +from itertools import chain +from pathlib import Path, PurePosixPath +from typing import Any, Optional, TypeVar, Union +from unittest.mock import patch +from urllib.parse import urlparse +from xml.etree import ElementTree as ET + +import fsspec +import huggingface_hub +import huggingface_hub.errors +import requests +from fsspec.core import strip_protocol, url_to_fs +from fsspec.utils import can_be_local +from huggingface_hub.utils import EntryNotFoundError, get_session, insecure_hashlib +from packaging import version + +from .. import __version__, config +from ..download.download_config import DownloadConfig +from ..filesystems import COMPRESSION_FILESYSTEMS +from . import _tqdm, logging +from ._filelock import FileLock +from .extract import ExtractManager +from .track import TrackedIterableFromGenerator + + +try: + from aiohttp.client_exceptions import ClientError as _AiohttpClientError +except ImportError: + # aiohttp is not available; synthesize an exception type + # that will never be raised by any actual code for use in the `except` + # clause only. + class _AiohttpClientError(Exception): + pass + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +INCOMPLETE_SUFFIX = ".incomplete" + +T = TypeVar("T", str, Path) + + +def init_hf_modules(hf_modules_cache: Optional[Union[Path, str]] = None) -> str: + """ + Add hf_modules_cache to the python path. + By default hf_modules_cache='~/.cache/huggingface/modules'. + It can also be set with the environment variable HF_MODULES_CACHE. + This is used to add modules such as `datasets_modules` + """ + hf_modules_cache = hf_modules_cache if hf_modules_cache is not None else config.HF_MODULES_CACHE + hf_modules_cache = str(hf_modules_cache) + if hf_modules_cache not in sys.path: + sys.path.append(hf_modules_cache) + + os.makedirs(hf_modules_cache, exist_ok=True) + if not os.path.exists(os.path.join(hf_modules_cache, "__init__.py")): + with open(os.path.join(hf_modules_cache, "__init__.py"), "w"): + pass + return hf_modules_cache + + +def is_remote_url(url_or_filename: str) -> bool: + return urlparse(url_or_filename).scheme != "" and not os.path.ismount(urlparse(url_or_filename).scheme + ":/") + + +def is_local_path(url_or_filename: str) -> bool: + # On unix the scheme of a local path is empty (for both absolute and relative), + # while on windows the scheme is the drive name (ex: "c") for absolute paths. + # for details on the windows behavior, see https://bugs.python.org/issue42215 + return urlparse(url_or_filename).scheme == "" or os.path.ismount(urlparse(url_or_filename).scheme + ":/") + + +def is_relative_path(url_or_filename: str) -> bool: + return urlparse(url_or_filename).scheme == "" and not os.path.isabs(url_or_filename) + + +def relative_to_absolute_path(path: T) -> T: + """Convert relative path to absolute path.""" + abs_path_str = os.path.abspath(os.path.expanduser(os.path.expandvars(str(path)))) + return Path(abs_path_str) if isinstance(path, Path) else abs_path_str + + +def url_or_path_join(base_name: str, *pathnames: str) -> str: + if is_remote_url(base_name): + return posixpath.join(base_name, *(str(pathname).replace(os.sep, "/").lstrip("/") for pathname in pathnames)) + else: + return Path(base_name, *pathnames).as_posix() + + +def url_or_path_parent(url_or_path: str) -> str: + if is_remote_url(url_or_path): + return url_or_path[: url_or_path.rindex("/")] + else: + return os.path.dirname(url_or_path) + + +def hash_url_to_filename(url, etag=None): + """ + Convert `url` into a hashed filename in a repeatable way. + If `etag` is specified, append its hash to the url's, delimited + by a period. + If the url ends with .h5 (Keras HDF5 weights) adds '.h5' to the name + so that TF 2.0 can identify it as a HDF5 file + (see https://github.com/tensorflow/tensorflow/blob/00fad90125b18b80fe054de1055770cfb8fe4ba3/tensorflow/python/keras/engine/network.py#L1380) + """ + url_bytes = url.encode("utf-8") + url_hash = insecure_hashlib.sha256(url_bytes) + filename = url_hash.hexdigest() + + if etag: + etag_bytes = etag.encode("utf-8") + etag_hash = insecure_hashlib.sha256(etag_bytes) + filename += "." + etag_hash.hexdigest() + + if url.endswith(".py"): + filename += ".py" + + return filename + + +def cached_path( + url_or_filename, + download_config=None, + **download_kwargs, +) -> str: + """ + Given something that might be a URL (or might be a local path), + determine which. If it's a URL, download the file and cache it, and + return the path to the cached file. If it's already a local path, + make sure the file exists and then return the path. + + Return: + Local path (string) + + Raises: + FileNotFoundError: in case of non-recoverable file + (non-existent or no cache on disk) + ConnectionError: in case of unreachable url + and no cache on disk + ValueError: if it couldn't parse the url or filename correctly + requests.exceptions.ConnectionError: in case of internet connection issue + """ + if download_config is None: + download_config = DownloadConfig(**download_kwargs) + + cache_dir = download_config.cache_dir or config.DOWNLOADED_DATASETS_PATH + if isinstance(cache_dir, Path): + cache_dir = str(cache_dir) + if isinstance(url_or_filename, Path): + url_or_filename = str(url_or_filename) + + # Convert fsspec URL in the format "file://local/path" to "local/path" + if can_be_local(url_or_filename): + url_or_filename = strip_protocol(url_or_filename) + + if is_remote_url(url_or_filename): + # URL, so get it from the cache (downloading if necessary) + url_or_filename, storage_options = _prepare_path_and_storage_options( + url_or_filename, download_config=download_config + ) + # Download files from Hugging Face. + # Note: no need to check for https://huggingface.co file URLs since _prepare_path_and_storage_options + # prepares Hugging Face HTTP URLs as hf:// paths already + if url_or_filename.startswith("hf://"): + resolved_path = huggingface_hub.HfFileSystem( + endpoint=config.HF_ENDPOINT, token=download_config.token + ).resolve_path(url_or_filename) + try: + output_path = huggingface_hub.HfApi( + endpoint=config.HF_ENDPOINT, + token=download_config.token, + library_name="datasets", + library_version=__version__, + user_agent=get_datasets_user_agent(download_config.user_agent), + ).hf_hub_download( + repo_id=resolved_path.repo_id, + repo_type=resolved_path.repo_type, + revision=resolved_path.revision, + filename=resolved_path.path_in_repo, + force_download=download_config.force_download, + proxies=download_config.proxies, + ) + except ( + huggingface_hub.utils.RepositoryNotFoundError, + huggingface_hub.utils.EntryNotFoundError, + huggingface_hub.utils.RevisionNotFoundError, + huggingface_hub.utils.GatedRepoError, + ) as e: + raise FileNotFoundError(str(e)) from e + # Download external files + else: + output_path = get_from_cache( + url_or_filename, + cache_dir=cache_dir, + force_download=download_config.force_download, + user_agent=download_config.user_agent, + use_etag=download_config.use_etag, + token=download_config.token, + storage_options=storage_options, + download_desc=download_config.download_desc, + disable_tqdm=download_config.disable_tqdm, + ) + elif os.path.exists(url_or_filename): + # File, and it exists. + output_path = url_or_filename + elif is_local_path(url_or_filename): + # File, but it doesn't exist. + raise FileNotFoundError(f"Local file {url_or_filename} doesn't exist") + else: + # Something unknown + raise ValueError(f"unable to parse {url_or_filename} as a URL or as a local path") + + if output_path is None: + return output_path + + if download_config.extract_compressed_file: + if download_config.extract_on_the_fly: + # Add a compression prefix to the compressed file so that it can be extracted + # as it's being read using xopen. + protocol = _get_extraction_protocol(output_path, download_config=download_config) + extension = _get_path_extension(url_or_filename.split("::")[0]) + if ( + protocol + and extension not in ["tgz", "tar"] + and not url_or_filename.split("::")[0].endswith((".tar.gz", ".tar.bz2", ".tar.xz")) + ): + output_path = relative_to_absolute_path(output_path) + if protocol in SINGLE_FILE_COMPRESSION_PROTOCOLS: + # there is one single file which is the uncompressed file + inner_file = os.path.basename(output_path) + inner_file = inner_file[: inner_file.rindex(".")] if "." in inner_file else inner_file + output_path = f"{protocol}://{inner_file}::{output_path}" + else: + output_path = f"{protocol}://::{output_path}" + return output_path + + # Eager extraction + output_path = ExtractManager(cache_dir=download_config.cache_dir).extract( + output_path, force_extract=download_config.force_extract + ) + return relative_to_absolute_path(output_path) + + +def get_datasets_user_agent(user_agent: Optional[Union[str, dict]] = None) -> str: + ua = f"datasets/{__version__}" + ua += f"; python/{config.PY_VERSION}" + ua += f"; huggingface_hub/{huggingface_hub.__version__}" + ua += f"; pyarrow/{config.PYARROW_VERSION}" + if config.TORCH_AVAILABLE: + ua += f"; torch/{config.TORCH_VERSION}" + if config.TF_AVAILABLE: + ua += f"; tensorflow/{config.TF_VERSION}" + if config.JAX_AVAILABLE: + ua += f"; jax/{config.JAX_VERSION}" + if isinstance(user_agent, dict): + ua += f"; {'; '.join(f'{k}/{v}' for k, v in user_agent.items())}" + elif isinstance(user_agent, str): + ua += "; " + user_agent + return ua + + +def get_authentication_headers_for_url(url: str, token: Optional[Union[str, bool]] = None) -> dict: + """Handle the HF authentication""" + if url.startswith(config.HF_ENDPOINT): + return huggingface_hub.utils.build_hf_headers( + token=token, library_name="datasets", library_version=__version__ + ) + else: + return {} + + +def _raise_if_offline_mode_is_enabled(msg: Optional[str] = None): + """Raise an OfflineModeIsEnabled error (subclass of ConnectionError) if HF_HUB_OFFLINE is True.""" + if config.HF_HUB_OFFLINE: + raise huggingface_hub.errors.OfflineModeIsEnabled( + "Offline mode is enabled." if msg is None else "Offline mode is enabled. " + str(msg) + ) + + +def fsspec_head(url, storage_options=None): + _raise_if_offline_mode_is_enabled(f"Tried to reach {url}") + fs, path = url_to_fs(url, **(storage_options or {})) + return fs.info(path) + + +def stack_multiprocessing_download_progress_bars(): + # Stack downloads progress bars automatically using HF_DATASETS_STACK_MULTIPROCESSING_DOWNLOAD_PROGRESS_BARS=1 + # We use environment variables since the download may happen in a subprocess + return patch.dict(os.environ, {"HF_DATASETS_STACK_MULTIPROCESSING_DOWNLOAD_PROGRESS_BARS": "1"}) + + +class TqdmCallback(fsspec.callbacks.TqdmCallback): + def __init__(self, tqdm_kwargs=None, *args, **kwargs): + if config.FSSPEC_VERSION < version.parse("2024.2.0"): + super().__init__(tqdm_kwargs, *args, **kwargs) + self._tqdm = _tqdm # replace tqdm module by datasets.utils.tqdm module + else: + kwargs["tqdm_cls"] = _tqdm.tqdm + super().__init__(tqdm_kwargs, *args, **kwargs) + + +def fsspec_get(url, temp_file, storage_options=None, desc=None, disable_tqdm=False): + _raise_if_offline_mode_is_enabled(f"Tried to reach {url}") + fs, path = url_to_fs(url, **(storage_options or {})) + callback = TqdmCallback( + tqdm_kwargs={ + "desc": desc or "Downloading", + "unit": "B", + "unit_scale": True, + "position": multiprocessing.current_process()._identity[-1] # contains the ranks of subprocesses + if os.environ.get("HF_DATASETS_STACK_MULTIPROCESSING_DOWNLOAD_PROGRESS_BARS") == "1" + and multiprocessing.current_process()._identity + else None, + "disable": disable_tqdm, + } + ) + fs.get_file(path, temp_file.name, callback=callback) + + +def get_from_cache( + url, + cache_dir=None, + force_download=False, + user_agent=None, + use_etag=True, + token=None, + storage_options=None, + download_desc=None, + disable_tqdm=False, +) -> str: + """ + Given a URL, look for the corresponding file in the local cache. + If it's not there, download it. Then return the path to the cached file. + + Return: + Local path (string) + + Raises: + FileNotFoundError: in case of non-recoverable file + (non-existent or no cache on disk) + ConnectionError: in case of unreachable url + and no cache on disk + """ + if storage_options is None: + storage_options = {} + if cache_dir is None: + cache_dir = config.HF_DATASETS_CACHE + if isinstance(cache_dir, Path): + cache_dir = str(cache_dir) + + os.makedirs(cache_dir, exist_ok=True) + + response = None + etag = None + + # Try a first time to file the file on the local file system without eTag (None) + # if we don't ask for 'force_download' then we spare a request + filename = hash_url_to_filename(url, etag=None) + cache_path = os.path.join(cache_dir, filename) + + if os.path.exists(cache_path) and not force_download and not use_etag: + return cache_path + + # Prepare headers for authentication + headers = get_authentication_headers_for_url(url, token=token) + if user_agent is not None: + headers["user-agent"] = user_agent + + response = fsspec_head(url, storage_options=storage_options) + etag = (response.get("ETag", None) or response.get("etag", None)) if use_etag else None + + # Try a second time + filename = hash_url_to_filename(url, etag) + cache_path = os.path.join(cache_dir, filename) + + if os.path.exists(cache_path) and not force_download: + return cache_path + + # Prevent parallel downloads of the same file with a lock. + lock_path = cache_path + ".lock" + with FileLock(lock_path): + # Retry in case previously locked processes just enter after the precedent process releases the lock + if os.path.exists(cache_path) and not force_download: + return cache_path + + incomplete_path = cache_path + ".incomplete" + + # Download to temporary file, then copy to cache path once finished. + # Otherwise, you get corrupt cache entries if the download gets interrupted. + with open(incomplete_path, "w+b") as temp_file: + logger.info(f"{url} not found in cache or force_download set to True, downloading to {temp_file.name}") + # GET file object + fsspec_get(url, temp_file, storage_options=storage_options, desc=download_desc, disable_tqdm=disable_tqdm) + + logger.info(f"storing {url} in cache at {cache_path}") + shutil.move(temp_file.name, cache_path) + + logger.info(f"creating metadata file for {cache_path}") + meta = {"url": url, "etag": etag} + meta_path = cache_path + ".json" + with open(meta_path, "w", encoding="utf-8") as meta_file: + json.dump(meta, meta_file) + + return cache_path + + +def add_start_docstrings(*docstr): + def docstring_decorator(fn): + fn.__doc__ = "".join(docstr) + "\n\n" + (fn.__doc__ if fn.__doc__ is not None else "") + return fn + + return docstring_decorator + + +def add_end_docstrings(*docstr): + def docstring_decorator(fn): + fn.__doc__ = (fn.__doc__ if fn.__doc__ is not None else "") + "\n\n" + "".join(docstr) + return fn + + return docstring_decorator + + +def estimate_dataset_size(paths): + return sum(path.stat().st_size for path in paths) + + +def readline(f: io.RawIOBase): + # From: https://github.com/python/cpython/blob/d27e2f4d118e7a9909b6a3e5da06c5ff95806a85/Lib/_pyio.py#L525 + res = bytearray() + while True: + b = f.read(1) + if not b: + break + res += b + if res.endswith(b"\n"): + break + return bytes(res) + + +####################### +# Streaming utilities # +####################### + +BASE_KNOWN_EXTENSIONS = [ + "txt", + "csv", + "json", + "jsonl", + "tsv", + "conll", + "conllu", + "orig", + "parquet", + "pkl", + "pickle", + "rel", + "xml", + "arrow", +] +COMPRESSION_EXTENSION_TO_PROTOCOL = { + # single file compression + **{fs_class.extension.lstrip("."): fs_class.protocol for fs_class in COMPRESSION_FILESYSTEMS}, + # archive compression + "zip": "zip", +} +SINGLE_FILE_COMPRESSION_EXTENSION_TO_PROTOCOL = { + fs_class.extension.lstrip("."): fs_class.protocol for fs_class in COMPRESSION_FILESYSTEMS +} +SINGLE_FILE_COMPRESSION_PROTOCOLS = {fs_class.protocol for fs_class in COMPRESSION_FILESYSTEMS} +SINGLE_SLASH_AFTER_PROTOCOL_PATTERN = re.compile(r"(? str: + # Get extension: https://foo.bar/train.json.gz -> gz + extension = path.split(".")[-1] + # Remove query params ("dl=1", "raw=true"): gz?dl=1 -> gz + # Remove shards infos (".txt_1", ".txt-00000-of-00100"): txt_1 -> txt + for symb in "?-_": + extension = extension.split(symb)[0] + return extension + + +def _get_extraction_protocol_with_magic_number(f) -> Optional[str]: + """read the magic number from a file-like object and return the compression protocol""" + # Check if the file object is seekable even before reading the magic number (to avoid https://bugs.python.org/issue26440) + try: + f.seek(0) + except (AttributeError, io.UnsupportedOperation): + return None + magic_number = f.read(MAGIC_NUMBER_MAX_LENGTH) + f.seek(0) + for i in range(MAGIC_NUMBER_MAX_LENGTH): + compression = MAGIC_NUMBER_TO_COMPRESSION_PROTOCOL.get(magic_number[: MAGIC_NUMBER_MAX_LENGTH - i]) + if compression is not None: + return compression + compression = MAGIC_NUMBER_TO_UNSUPPORTED_COMPRESSION_PROTOCOL.get(magic_number[: MAGIC_NUMBER_MAX_LENGTH - i]) + if compression is not None: + raise NotImplementedError(f"Compression protocol '{compression}' not implemented.") + + +def _get_extraction_protocol(urlpath: str, download_config: Optional[DownloadConfig] = None) -> Optional[str]: + # get inner file: zip://train-00000.json.gz::https://foo.bar/data.zip -> zip://train-00000.json.gz + urlpath = str(urlpath) + path = urlpath.split("::")[0] + extension = _get_path_extension(path) + if ( + extension in BASE_KNOWN_EXTENSIONS + or extension in ["tgz", "tar"] + or path.endswith((".tar.gz", ".tar.bz2", ".tar.xz")) + ): + return None + elif extension in COMPRESSION_EXTENSION_TO_PROTOCOL: + return COMPRESSION_EXTENSION_TO_PROTOCOL[extension] + urlpath, storage_options = _prepare_path_and_storage_options(urlpath, download_config=download_config) + try: + with fsspec.open(urlpath, **(storage_options or {})) as f: + return _get_extraction_protocol_with_magic_number(f) + except FileNotFoundError: + if urlpath.startswith(config.HF_ENDPOINT): + raise FileNotFoundError( + urlpath + "\nIf the repo is private or gated, make sure to log in with `huggingface-cli login`." + ) from None + else: + raise + + +def xjoin(a, *p): + """ + This function extends os.path.join to support the "::" hop separator. It supports both paths and urls. + + A shorthand, particularly useful where you have multiple hops, is to “chain” the URLs with the special separator "::". + This is used to access files inside a zip file over http for example. + + Let's say you have a zip file at https://host.com/archive.zip, and you want to access the file inside the zip file at /folder1/file.txt. + Then you can just chain the url this way: + + zip://folder1/file.txt::https://host.com/archive.zip + + The xjoin function allows you to apply the join on the first path of the chain. + + Example:: + + >>> xjoin("zip://folder1::https://host.com/archive.zip", "file.txt") + zip://folder1/file.txt::https://host.com/archive.zip + """ + a, *b = str(a).split("::") + if is_local_path(a): + return os.path.join(a, *p) + else: + a = posixpath.join(a, *p) + return "::".join([a] + b) + + +def xdirname(a): + """ + This function extends os.path.dirname to support the "::" hop separator. It supports both paths and urls. + + A shorthand, particularly useful where you have multiple hops, is to “chain” the URLs with the special separator "::". + This is used to access files inside a zip file over http for example. + + Let's say you have a zip file at https://host.com/archive.zip, and you want to access the file inside the zip file at /folder1/file.txt. + Then you can just chain the url this way: + + zip://folder1/file.txt::https://host.com/archive.zip + + The xdirname function allows you to apply the dirname on the first path of the chain. + + Example:: + + >>> xdirname("zip://folder1/file.txt::https://host.com/archive.zip") + zip://folder1::https://host.com/archive.zip + """ + a, *b = str(a).split("::") + if is_local_path(a): + a = os.path.dirname(Path(a).as_posix()) + else: + a = posixpath.dirname(a) + # if we end up at the root of the protocol, we get for example a = 'http:' + # so we have to fix it by adding the '//' that was removed: + if a.endswith(":"): + a += "//" + return "::".join([a] + b) + + +def xexists(urlpath: str, download_config: Optional[DownloadConfig] = None): + """Extend `os.path.exists` function to support both local and remote files. + + Args: + urlpath (`str`): URL path. + download_config : mainly use token or storage_options to support different platforms and auth types. + + Returns: + `bool` + """ + + main_hop, *rest_hops = _as_str(urlpath).split("::") + if is_local_path(main_hop): + return os.path.exists(main_hop) + else: + urlpath, storage_options = _prepare_path_and_storage_options(urlpath, download_config=download_config) + main_hop, *rest_hops = urlpath.split("::") + fs, *_ = url_to_fs(urlpath, **storage_options) + return fs.exists(main_hop) + + +def xbasename(a): + """ + This function extends os.path.basename to support the "::" hop separator. It supports both paths and urls. + + A shorthand, particularly useful where you have multiple hops, is to “chain” the URLs with the special separator "::". + This is used to access files inside a zip file over http for example. + + Let's say you have a zip file at https://host.com/archive.zip, and you want to access the file inside the zip file at /folder1/file.txt. + Then you can just chain the url this way: + + zip://folder1/file.txt::https://host.com/archive.zip + + The xbasename function allows you to apply the basename on the first path of the chain. + + Example:: + + >>> xbasename("zip://folder1/file.txt::https://host.com/archive.zip") + file.txt + """ + a, *b = str(a).split("::") + if is_local_path(a): + return os.path.basename(Path(a).as_posix()) + else: + return posixpath.basename(a) + + +def xsplit(a): + """ + This function extends os.path.split to support the "::" hop separator. It supports both paths and urls. + + A shorthand, particularly useful where you have multiple hops, is to “chain” the URLs with the special separator "::". + This is used to access files inside a zip file over http for example. + + Let's say you have a zip file at https://host.com/archive.zip, and you want to access the file inside the zip file at /folder1/file.txt. + Then you can just chain the url this way: + + zip://folder1/file.txt::https://host.com/archive.zip + + The xsplit function allows you to apply the xsplit on the first path of the chain. + + Example:: + + >>> xsplit("zip://folder1/file.txt::https://host.com/archive.zip") + ('zip://folder1::https://host.com/archive.zip', 'file.txt') + """ + a, *b = str(a).split("::") + if is_local_path(a): + return os.path.split(Path(a).as_posix()) + else: + a, tail = posixpath.split(a) + return "::".join([a + "//" if a.endswith(":") else a] + b), tail + + +def xsplitext(a): + """ + This function extends os.path.splitext to support the "::" hop separator. It supports both paths and urls. + + A shorthand, particularly useful where you have multiple hops, is to “chain” the URLs with the special separator "::". + This is used to access files inside a zip file over http for example. + + Let's say you have a zip file at https://host.com/archive.zip, and you want to access the file inside the zip file at /folder1/file.txt. + Then you can just chain the url this way: + + zip://folder1/file.txt::https://host.com/archive.zip + + The xsplitext function allows you to apply the splitext on the first path of the chain. + + Example:: + + >>> xsplitext("zip://folder1/file.txt::https://host.com/archive.zip") + ('zip://folder1/file::https://host.com/archive.zip', '.txt') + """ + a, *b = str(a).split("::") + if is_local_path(a): + return os.path.splitext(Path(a).as_posix()) + else: + a, ext = posixpath.splitext(a) + return "::".join([a] + b), ext + + +def xisfile(path, download_config: Optional[DownloadConfig] = None) -> bool: + """Extend `os.path.isfile` function to support remote files. + + Args: + path (`str`): URL path. + download_config : mainly use token or storage_options to support different platforms and auth types. + + Returns: + `bool` + """ + main_hop, *rest_hops = str(path).split("::") + if is_local_path(main_hop): + return os.path.isfile(path) + else: + path, storage_options = _prepare_path_and_storage_options(path, download_config=download_config) + main_hop, *rest_hops = path.split("::") + fs, *_ = url_to_fs(path, **storage_options) + return fs.isfile(main_hop) + + +def xgetsize(path, download_config: Optional[DownloadConfig] = None) -> int: + """Extend `os.path.getsize` function to support remote files. + + Args: + path (`str`): URL path. + download_config : mainly use token or storage_options to support different platforms and auth types. + + Returns: + `int`: optional + """ + main_hop, *rest_hops = str(path).split("::") + if is_local_path(main_hop): + return os.path.getsize(path) + else: + path, storage_options = _prepare_path_and_storage_options(path, download_config=download_config) + main_hop, *rest_hops = path.split("::") + fs, *_ = fs, *_ = url_to_fs(path, **storage_options) + try: + size = fs.size(main_hop) + except EntryNotFoundError: + raise FileNotFoundError(f"No such file: {path}") + if size is None: + # use xopen instead of fs.open to make data fetching more robust + with xopen(path, download_config=download_config) as f: + size = len(f.read()) + return size + + +def xisdir(path, download_config: Optional[DownloadConfig] = None) -> bool: + """Extend `os.path.isdir` function to support remote files. + + Args: + path (`str`): URL path. + download_config : mainly use token or storage_options to support different platforms and auth types. + + Returns: + `bool` + """ + main_hop, *rest_hops = str(path).split("::") + if is_local_path(main_hop): + return os.path.isdir(path) + else: + path, storage_options = _prepare_path_and_storage_options(path, download_config=download_config) + main_hop, *rest_hops = path.split("::") + fs, *_ = fs, *_ = url_to_fs(path, **storage_options) + inner_path = main_hop.split("://")[-1] + if not inner_path.strip("/"): + return True + return fs.isdir(inner_path) + + +def xrelpath(path, start=None): + """Extend `os.path.relpath` function to support remote files. + + Args: + path (`str`): URL path. + start (`str`): Start URL directory path. + + Returns: + `str` + """ + main_hop, *rest_hops = str(path).split("::") + if is_local_path(main_hop): + return os.path.relpath(main_hop, start=start) if start else os.path.relpath(main_hop) + else: + return posixpath.relpath(main_hop, start=str(start).split("::")[0]) if start else os.path.relpath(main_hop) + + +def _add_retries_to_file_obj_read_method(file_obj): + read = file_obj.read + max_retries = config.STREAMING_READ_MAX_RETRIES + + def read_with_retries(*args, **kwargs): + disconnect_err = None + for retry in range(1, max_retries + 1): + try: + out = read(*args, **kwargs) + break + except ( + _AiohttpClientError, + asyncio.TimeoutError, + requests.exceptions.ConnectionError, + requests.exceptions.Timeout, + ) as err: + disconnect_err = err + logger.warning( + f"Got disconnected from remote data host. Retrying in {config.STREAMING_READ_RETRY_INTERVAL}sec [{retry}/{max_retries}]" + ) + time.sleep(config.STREAMING_READ_RETRY_INTERVAL) + else: + raise ConnectionError("Server Disconnected") from disconnect_err + return out + + try: + file_obj.read = read_with_retries + except AttributeError: # read-only attribute + orig_file_obj = file_obj + file_obj = io.RawIOBase() + file_obj.read = read_with_retries + file_obj.__getattr__ = lambda _, attr: getattr(orig_file_obj, attr) + return file_obj + + +def _prepare_path_and_storage_options( + urlpath: str, download_config: Optional[DownloadConfig] = None +) -> tuple[str, dict[str, dict[str, Any]]]: + prepared_urlpath = [] + prepared_storage_options = {} + for hop in urlpath.split("::"): + hop, storage_options = _prepare_single_hop_path_and_storage_options(hop, download_config=download_config) + prepared_urlpath.append(hop) + prepared_storage_options.update(storage_options) + return "::".join(prepared_urlpath), storage_options + + +def _prepare_single_hop_path_and_storage_options( + urlpath: str, download_config: Optional[DownloadConfig] = None +) -> tuple[str, dict[str, dict[str, Any]]]: + """ + Prepare the URL and the kwargs that must be passed to the HttpFileSystem or HfFileSystem + + In particular it resolves google drive URLs + It also adds the authentication headers for the Hugging Face Hub, for both https:// and hf:// paths. + + Storage options are formatted in the form {protocol: storage_options_for_protocol} + """ + token = None if download_config is None else download_config.token + if urlpath.startswith(config.HF_ENDPOINT) and "/resolve/" in urlpath: + urlpath = "hf://" + urlpath[len(config.HF_ENDPOINT) + 1 :].replace("/resolve/", "@", 1) + protocol = urlpath.split("://")[0] if "://" in urlpath else "file" + if download_config is not None and protocol in download_config.storage_options: + storage_options = download_config.storage_options[protocol].copy() + elif download_config is not None and protocol not in download_config.storage_options: + storage_options = { + option_name: option_value + for option_name, option_value in download_config.storage_options.items() + if option_name not in fsspec.available_protocols() + } + else: + storage_options = {} + if protocol in {"http", "https"}: + client_kwargs = storage_options.pop("client_kwargs", {}) + storage_options["client_kwargs"] = {"trust_env": True, **client_kwargs} # Enable reading proxy env variables + if "drive.google.com" in urlpath: + response = get_session().head(urlpath, timeout=10) + for k, v in response.cookies.items(): + if k.startswith("download_warning"): + urlpath += "&confirm=" + v + cookies = response.cookies + storage_options = {"cookies": cookies, **storage_options} + # Fix Google Drive URL to avoid Virus scan warning + if "confirm=" not in urlpath: + urlpath += "&confirm=t" + if urlpath.startswith("https://raw.githubusercontent.com/"): + # Workaround for served data with gzip content-encoding: https://github.com/fsspec/filesystem_spec/issues/389 + headers = storage_options.pop("headers", {}) + storage_options["headers"] = {"Accept-Encoding": "identity", **headers} + elif protocol == "hf": + storage_options = { + "token": token, + "endpoint": config.HF_ENDPOINT, + **storage_options, + } + # streaming with block_size=0 is only implemented in 0.21 (see https://github.com/huggingface/huggingface_hub/pull/1967) + if config.HF_HUB_VERSION < version.parse("0.21.0"): + storage_options["block_size"] = "default" + if storage_options: + storage_options = {protocol: storage_options} + return urlpath, storage_options + + +def xopen(file: str, mode="r", *args, download_config: Optional[DownloadConfig] = None, **kwargs): + """Extend `open` function to support remote files using `fsspec`. + + It also has a retry mechanism in case connection fails. + The `args` and `kwargs` are passed to `fsspec.open`, except `token` which is used for queries to private repos on huggingface.co + + Args: + file (`str`): Path name of the file to be opened. + mode (`str`, *optional*, default "r"): Mode in which the file is opened. + *args: Arguments to be passed to `fsspec.open`. + download_config : mainly use token or storage_options to support different platforms and auth types. + **kwargs: Keyword arguments to be passed to `fsspec.open`. + + Returns: + file object + """ + # This works as well for `xopen(str(Path(...)))` + file_str = _as_str(file) + main_hop, *rest_hops = file_str.split("::") + if is_local_path(main_hop): + # ignore fsspec-specific kwargs + kwargs.pop("block_size", None) + return open(main_hop, mode, *args, **kwargs) + # add headers and cookies for authentication on the HF Hub and for Google Drive + file, storage_options = _prepare_path_and_storage_options(file_str, download_config=download_config) + kwargs = {**kwargs, **(storage_options or {})} + try: + file_obj = fsspec.open(file, mode=mode, *args, **kwargs).open() + except ValueError as e: + if str(e) == "Cannot seek streaming HTTP file": + raise NonStreamableDatasetError( + "Streaming is not possible for this dataset because data host server doesn't support HTTP range " + "requests. You can still load this dataset in non-streaming mode by passing `streaming=False` (default)" + ) from e + else: + raise + except FileNotFoundError: + if file.startswith(config.HF_ENDPOINT): + raise FileNotFoundError( + file + "\nIf the repo is private or gated, make sure to log in with `huggingface-cli login`." + ) from None + else: + raise + file_obj = _add_retries_to_file_obj_read_method(file_obj) + return file_obj + + +def xlistdir(path: str, download_config: Optional[DownloadConfig] = None) -> list[str]: + """Extend `os.listdir` function to support remote files. + + Args: + path (`str`): URL path. + download_config : mainly use token or storage_options to support different platforms and auth types. + + Returns: + `list` of `str` + """ + main_hop, *rest_hops = _as_str(path).split("::") + if is_local_path(main_hop): + return os.listdir(path) + else: + # globbing inside a zip in a private repo requires authentication + path, storage_options = _prepare_path_and_storage_options(path, download_config=download_config) + main_hop, *rest_hops = path.split("::") + fs, *_ = url_to_fs(path, **storage_options) + inner_path = main_hop.split("://")[-1] + if inner_path.strip("/") and not fs.isdir(inner_path): + raise FileNotFoundError(f"Directory doesn't exist: {path}") + paths = fs.listdir(inner_path, detail=False) + return [os.path.basename(path.rstrip("/")) for path in paths] + + +def xglob(urlpath, *, recursive=False, download_config: Optional[DownloadConfig] = None): + """Extend `glob.glob` function to support remote files. + + Args: + urlpath (`str`): URL path with shell-style wildcard patterns. + recursive (`bool`, default `False`): Whether to match the "**" pattern recursively to zero or more + directories or subdirectories. + download_config : mainly use token or storage_options to support different platforms and auth types. + + Returns: + `list` of `str` + """ + main_hop, *rest_hops = _as_str(urlpath).split("::") + if is_local_path(main_hop): + return glob.glob(main_hop, recursive=recursive) + else: + # globbing inside a zip in a private repo requires authentication + urlpath, storage_options = _prepare_path_and_storage_options(urlpath, download_config=download_config) + main_hop, *rest_hops = urlpath.split("::") + fs, *_ = url_to_fs(urlpath, **storage_options) + inner_path = main_hop.split("://")[1] + globbed_paths = fs.glob(inner_path) + protocol = fs.protocol if isinstance(fs.protocol, str) else fs.protocol[-1] + return ["::".join([f"{protocol}://{globbed_path}"] + rest_hops) for globbed_path in globbed_paths] + + +def xwalk(urlpath, download_config: Optional[DownloadConfig] = None, **kwargs): + """Extend `os.walk` function to support remote files. + + Args: + urlpath (`str`): URL root path. + download_config : mainly use token or storage_options to support different platforms and auth types. + **kwargs: Additional keyword arguments forwarded to the underlying filesystem. + + + Yields: + `tuple`: 3-tuple (dirpath, dirnames, filenames). + """ + main_hop, *rest_hops = _as_str(urlpath).split("::") + if is_local_path(main_hop): + yield from os.walk(main_hop, **kwargs) + else: + # walking inside a zip in a private repo requires authentication + urlpath, storage_options = _prepare_path_and_storage_options(urlpath, download_config=download_config) + main_hop, *rest_hops = urlpath.split("::") + fs, *_ = url_to_fs(urlpath, **storage_options) + inner_path = main_hop.split("://")[-1] + if inner_path.strip("/") and not fs.isdir(inner_path): + return [] + protocol = fs.protocol if isinstance(fs.protocol, str) else fs.protocol[-1] + for dirpath, dirnames, filenames in fs.walk(inner_path, **kwargs): + yield "::".join([f"{protocol}://{dirpath}"] + rest_hops), dirnames, filenames + + +class xPath(type(Path())): + """Extension of `pathlib.Path` to support both local paths and remote URLs.""" + + def __str__(self): + path_str = super().__str__() + main_hop, *rest_hops = path_str.split("::") + if is_local_path(main_hop): + return main_hop + path_as_posix = path_str.replace("\\", "/") + path_as_posix = SINGLE_SLASH_AFTER_PROTOCOL_PATTERN.sub("://", path_as_posix) + path_as_posix += "//" if path_as_posix.endswith(":") else "" # Add slashes to root of the protocol + return path_as_posix + + def exists(self, download_config: Optional[DownloadConfig] = None): + """Extend `pathlib.Path.exists` method to support both local and remote files. + + Args: + download_config : mainly use token or storage_options to support different platforms and auth types. + + Returns: + `bool` + """ + return xexists(str(self), download_config=download_config) + + def glob(self, pattern, download_config: Optional[DownloadConfig] = None): + """Glob function for argument of type :obj:`~pathlib.Path` that supports both local paths end remote URLs. + + Args: + pattern (`str`): Pattern that resulting paths must match. + download_config : mainly use token or storage_options to support different platforms and auth types. + + Yields: + [`xPath`] + """ + posix_path = self.as_posix() + main_hop, *rest_hops = posix_path.split("::") + if is_local_path(main_hop): + yield from Path(main_hop).glob(pattern) + else: + # globbing inside a zip in a private repo requires authentication + if rest_hops: + urlpath = rest_hops[0] + urlpath, storage_options = _prepare_path_and_storage_options(urlpath, download_config=download_config) + storage_options = {urlpath.split("://")[0]: storage_options} + posix_path = "::".join([main_hop, urlpath, *rest_hops[1:]]) + else: + storage_options = None + fs, *_ = url_to_fs(xjoin(posix_path, pattern), **(storage_options or {})) + globbed_paths = fs.glob(xjoin(main_hop, pattern)) + for globbed_path in globbed_paths: + yield type(self)("::".join([f"{fs.protocol}://{globbed_path}"] + rest_hops)) + + def rglob(self, pattern, **kwargs): + """Rglob function for argument of type :obj:`~pathlib.Path` that supports both local paths end remote URLs. + + Args: + pattern (`str`): Pattern that resulting paths must match. + + Yields: + [`xPath`] + """ + return self.glob("**/" + pattern, **kwargs) + + @property + def parent(self) -> "xPath": + """Name function for argument of type :obj:`~pathlib.Path` that supports both local paths end remote URLs. + + Returns: + [`xPath`] + """ + return type(self)(xdirname(self.as_posix())) + + @property + def name(self) -> str: + """Name function for argument of type :obj:`~pathlib.Path` that supports both local paths end remote URLs. + + Returns: + `str` + """ + return PurePosixPath(self.as_posix().split("::")[0]).name + + @property + def stem(self) -> str: + """Stem function for argument of type :obj:`~pathlib.Path` that supports both local paths end remote URLs. + + Returns: + `str` + """ + return PurePosixPath(self.as_posix().split("::")[0]).stem + + @property + def suffix(self) -> str: + """Suffix function for argument of type :obj:`~pathlib.Path` that supports both local paths end remote URLs. + + Returns: + `str` + """ + return PurePosixPath(self.as_posix().split("::")[0]).suffix + + def open(self, *args, **kwargs): + """Extend :func:`xopen` to support argument of type :obj:`~pathlib.Path`. + + Args: + **args: Arguments passed to :func:`fsspec.open`. + **kwargs: Keyword arguments passed to :func:`fsspec.open`. + + Returns: + `io.FileIO`: File-like object. + """ + return xopen(str(self), *args, **kwargs) + + def joinpath(self, *p: tuple[str, ...]) -> "xPath": + """Extend :func:`xjoin` to support argument of type :obj:`~pathlib.Path`. + + Args: + *p (`tuple` of `str`): Other path components. + + Returns: + [`xPath`] + """ + return type(self)(xjoin(self.as_posix(), *p)) + + def __truediv__(self, p: str) -> "xPath": + return self.joinpath(p) + + def with_suffix(self, suffix): + main_hop, *rest_hops = str(self).split("::") + if is_local_path(main_hop): + return type(self)(str(super().with_suffix(suffix))) + return type(self)("::".join([type(self)(PurePosixPath(main_hop).with_suffix(suffix)).as_posix()] + rest_hops)) + + +def _as_str(path: Union[str, Path, xPath]): + return str(path) if isinstance(path, xPath) else str(xPath(str(path))) + + +def xgzip_open(filepath_or_buffer, *args, download_config: Optional[DownloadConfig] = None, **kwargs): + import gzip + + if hasattr(filepath_or_buffer, "read"): + return gzip.open(filepath_or_buffer, *args, **kwargs) + else: + filepath_or_buffer = str(filepath_or_buffer) + return gzip.open(xopen(filepath_or_buffer, "rb", download_config=download_config), *args, **kwargs) + + +def xnumpy_load(filepath_or_buffer, *args, download_config: Optional[DownloadConfig] = None, **kwargs): + import numpy as np + + if hasattr(filepath_or_buffer, "read"): + return np.load(filepath_or_buffer, *args, **kwargs) + else: + filepath_or_buffer = str(filepath_or_buffer) + return np.load(xopen(filepath_or_buffer, "rb", download_config=download_config), *args, **kwargs) + + +def xpandas_read_csv(filepath_or_buffer, download_config: Optional[DownloadConfig] = None, **kwargs): + import pandas as pd + + if hasattr(filepath_or_buffer, "read"): + return pd.read_csv(filepath_or_buffer, **kwargs) + else: + filepath_or_buffer = str(filepath_or_buffer) + if kwargs.get("compression", "infer") == "infer": + kwargs["compression"] = _get_extraction_protocol(filepath_or_buffer, download_config=download_config) + return pd.read_csv(xopen(filepath_or_buffer, "rb", download_config=download_config), **kwargs) + + +def xpandas_read_excel(filepath_or_buffer, download_config: Optional[DownloadConfig] = None, **kwargs): + import pandas as pd + + if hasattr(filepath_or_buffer, "read"): + try: + return pd.read_excel(filepath_or_buffer, **kwargs) + except ValueError: # Cannot seek streaming HTTP file + return pd.read_excel(BytesIO(filepath_or_buffer.read()), **kwargs) + else: + filepath_or_buffer = str(filepath_or_buffer) + try: + return pd.read_excel(xopen(filepath_or_buffer, "rb", download_config=download_config), **kwargs) + except ValueError: # Cannot seek streaming HTTP file + return pd.read_excel( + BytesIO(xopen(filepath_or_buffer, "rb", download_config=download_config).read()), **kwargs + ) + + +def xpyarrow_parquet_read_table(filepath_or_buffer, download_config: Optional[DownloadConfig] = None, **kwargs): + import pyarrow.parquet as pq + + if hasattr(filepath_or_buffer, "read"): + return pq.read_table(filepath_or_buffer, **kwargs) + else: + filepath_or_buffer = str(filepath_or_buffer) + return pq.read_table(xopen(filepath_or_buffer, mode="rb", download_config=download_config), **kwargs) + + +def xsio_loadmat(filepath_or_buffer, download_config: Optional[DownloadConfig] = None, **kwargs): + import scipy.io as sio + + if hasattr(filepath_or_buffer, "read"): + return sio.loadmat(filepath_or_buffer, **kwargs) + else: + return sio.loadmat(xopen(filepath_or_buffer, "rb", download_config=download_config), **kwargs) + + +def xet_parse(source, parser=None, download_config: Optional[DownloadConfig] = None): + """Extend `xml.etree.ElementTree.parse` function to support remote files. + + Args: + source: File path or file object. + parser (`XMLParser`, *optional*, default `XMLParser`): Parser instance. + download_config : mainly use token or storage_options to support different platforms and auth types. + + Returns: + `xml.etree.ElementTree.Element`: Root element of the given source document. + """ + if hasattr(source, "read"): + return ET.parse(source, parser=parser) + else: + with xopen(source, "rb", download_config=download_config) as f: + return ET.parse(f, parser=parser) + + +def xxml_dom_minidom_parse(filename_or_file, download_config: Optional[DownloadConfig] = None, **kwargs): + """Extend `xml.dom.minidom.parse` function to support remote files. + + Args: + filename_or_file (`str` or file): File path or file object. + download_config : mainly use token or storage_options to support different platforms and auth types. + **kwargs (optional): Additional keyword arguments passed to `xml.dom.minidom.parse`. + + Returns: + :obj:`xml.dom.minidom.Document`: Parsed document. + """ + if hasattr(filename_or_file, "read"): + return xml.dom.minidom.parse(filename_or_file, **kwargs) + else: + with xopen(filename_or_file, "rb", download_config=download_config) as f: + return xml.dom.minidom.parse(f, **kwargs) + + +class ArchiveIterable(TrackedIterableFromGenerator): + """An iterable of (path, fileobj) from a TAR archive, used by `iter_archive`""" + + @staticmethod + def _iter_tar(f): + stream = tarfile.open(fileobj=f, mode="r|*") + for tarinfo in stream: + file_path = tarinfo.name + if not tarinfo.isreg(): + continue + if file_path is None: + continue + if os.path.basename(file_path).startswith((".", "__")): + # skipping hidden files + continue + file_obj = stream.extractfile(tarinfo) + yield file_path, file_obj + stream.members = [] + del stream + + @staticmethod + def _iter_zip(f): + zipf = zipfile.ZipFile(f) + for member in zipf.infolist(): + file_path = member.filename + if member.is_dir(): + continue + if file_path is None: + continue + if os.path.basename(file_path).startswith((".", "__")): + # skipping hidden files + continue + file_obj = zipf.open(member) + yield file_path, file_obj + + @classmethod + def _iter_from_fileobj(cls, f) -> Generator[tuple, None, None]: + compression = _get_extraction_protocol_with_magic_number(f) + if compression == "zip": + yield from cls._iter_zip(f) + else: + yield from cls._iter_tar(f) + + @classmethod + def _iter_from_urlpath( + cls, urlpath: str, download_config: Optional[DownloadConfig] = None + ) -> Generator[tuple, None, None]: + compression = _get_extraction_protocol(urlpath, download_config=download_config) + # Set block_size=0 to get faster streaming + # (e.g. for hf:// and https:// it uses streaming Requests file-like instances) + with xopen(urlpath, "rb", download_config=download_config, block_size=0) as f: + if compression == "zip": + yield from cls._iter_zip(f) + else: + yield from cls._iter_tar(f) + + @classmethod + def from_buf(cls, fileobj) -> "ArchiveIterable": + return cls(cls._iter_from_fileobj, fileobj) + + @classmethod + def from_urlpath(cls, urlpath_or_buf, download_config: Optional[DownloadConfig] = None) -> "ArchiveIterable": + return cls(cls._iter_from_urlpath, urlpath_or_buf, download_config) + + +class FilesIterable(TrackedIterableFromGenerator): + """An iterable of paths from a list of directories or files""" + + @classmethod + def _iter_from_urlpaths( + cls, urlpaths: Union[str, list[str]], download_config: Optional[DownloadConfig] = None + ) -> Generator[str, None, None]: + if not isinstance(urlpaths, list): + urlpaths = [urlpaths] + for urlpath in urlpaths: + if xisfile(urlpath, download_config=download_config): + yield urlpath + elif xisdir(urlpath, download_config=download_config): + for dirpath, dirnames, filenames in xwalk(urlpath, download_config=download_config): + # in-place modification to prune the search + dirnames[:] = sorted([dirname for dirname in dirnames if not dirname.startswith((".", "__"))]) + if xbasename(dirpath).startswith((".", "__")): + # skipping hidden directories + continue + for filename in sorted(filenames): + if filename.startswith((".", "__")): + # skipping hidden files + continue + yield xjoin(dirpath, filename) + else: + raise FileNotFoundError(urlpath) + + @classmethod + def from_urlpaths(cls, urlpaths, download_config: Optional[DownloadConfig] = None) -> "FilesIterable": + return cls(cls._iter_from_urlpaths, urlpaths, download_config) diff --git a/lib/python3.12/site-packages/datasets/utils/filelock.py b/lib/python3.12/site-packages/datasets/utils/filelock.py new file mode 100644 index 0000000000000000000000000000000000000000..df0728efe644d8eb32f0e578a85e39ba366e9743 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/utils/filelock.py @@ -0,0 +1,11 @@ +# deprecated, please use the `filelock` package instead + +from filelock import ( # noqa: F401 # imported for backward compatibility TODO: remove in 3.0.0 + BaseFileLock, + SoftFileLock, + Timeout, + UnixFileLock, + WindowsFileLock, +) + +from ._filelock import FileLock # noqa: F401 # imported for backward compatibility. TODO: remove in 3.0.0 diff --git a/lib/python3.12/site-packages/datasets/utils/hub.py b/lib/python3.12/site-packages/datasets/utils/hub.py new file mode 100644 index 0000000000000000000000000000000000000000..555157afd520236d4522785c08f3ba1e55468413 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/utils/hub.py @@ -0,0 +1,14 @@ +from functools import partial + +from huggingface_hub import hf_hub_url +from huggingface_hub.utils import get_session, hf_raise_for_status + + +hf_dataset_url = partial(hf_hub_url, repo_type="dataset") + + +def check_auth(hf_api, repo_id, token=None): + headers = hf_api._build_hf_headers(token=token) + path = f"{hf_api.endpoint}/api/datasets/{repo_id}/auth-check" + r = get_session().get(path, headers=headers) + hf_raise_for_status(r) diff --git a/lib/python3.12/site-packages/datasets/utils/info_utils.py b/lib/python3.12/site-packages/datasets/utils/info_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..d93f5b4509f47c5c12ad98c11fb34acfdab9b714 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/utils/info_utils.py @@ -0,0 +1,106 @@ +import enum +import os +from typing import Optional + +from huggingface_hub.utils import insecure_hashlib + +from .. import config +from ..exceptions import ( + ExpectedMoreDownloadedFilesError, + ExpectedMoreSplitsError, + NonMatchingChecksumError, + NonMatchingSplitsSizesError, + UnexpectedDownloadedFileError, + UnexpectedSplitsError, +) +from .logging import get_logger + + +logger = get_logger(__name__) + + +class VerificationMode(enum.Enum): + """`Enum` that specifies which verification checks to run. + + The default mode is `BASIC_CHECKS`, which will perform only rudimentary checks to avoid slowdowns + when generating/downloading a dataset for the first time. + + The verification modes: + + | | Verification checks | + |---------------------------|------------------------------------------------------------------------------ | + | `ALL_CHECKS` | Split checks, uniqueness of the keys yielded in case of the GeneratorBuilder | + | | and the validity (number of files, checksums, etc.) of downloaded files | + | `BASIC_CHECKS` (default) | Same as `ALL_CHECKS` but without checking downloaded files | + | `NO_CHECKS` | None | + + """ + + ALL_CHECKS = "all_checks" + BASIC_CHECKS = "basic_checks" + NO_CHECKS = "no_checks" + + +def verify_checksums(expected_checksums: Optional[dict], recorded_checksums: dict, verification_name=None): + if expected_checksums is None: + logger.info("Unable to verify checksums.") + return + if len(set(expected_checksums) - set(recorded_checksums)) > 0: + raise ExpectedMoreDownloadedFilesError(str(set(expected_checksums) - set(recorded_checksums))) + if len(set(recorded_checksums) - set(expected_checksums)) > 0: + raise UnexpectedDownloadedFileError(str(set(recorded_checksums) - set(expected_checksums))) + bad_urls = [url for url in expected_checksums if expected_checksums[url] != recorded_checksums[url]] + for_verification_name = " for " + verification_name if verification_name is not None else "" + if len(bad_urls) > 0: + raise NonMatchingChecksumError( + f"Checksums didn't match{for_verification_name}:\n" + f"{bad_urls}\n" + "Set `verification_mode='no_checks'` to skip checksums verification and ignore this error" + ) + logger.info("All the checksums matched successfully" + for_verification_name) + + +def verify_splits(expected_splits: Optional[dict], recorded_splits: dict): + if expected_splits is None: + logger.info("Unable to verify splits sizes.") + return + if len(set(expected_splits) - set(recorded_splits)) > 0: + raise ExpectedMoreSplitsError(str(set(expected_splits) - set(recorded_splits))) + if len(set(recorded_splits) - set(expected_splits)) > 0: + raise UnexpectedSplitsError(str(set(recorded_splits) - set(expected_splits))) + bad_splits = [ + {"expected": expected_splits[name], "recorded": recorded_splits[name]} + for name in expected_splits + if expected_splits[name].num_examples != recorded_splits[name].num_examples + ] + if len(bad_splits) > 0: + raise NonMatchingSplitsSizesError(str(bad_splits)) + logger.info("All the splits matched successfully.") + + +def get_size_checksum_dict(path: str, record_checksum: bool = True) -> dict: + """Compute the file size and the sha256 checksum of a file""" + if record_checksum: + m = insecure_hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + m.update(chunk) + checksum = m.hexdigest() + else: + checksum = None + return {"num_bytes": os.path.getsize(path), "checksum": checksum} + + +def is_small_dataset(dataset_size): + """Check if `dataset_size` is smaller than `config.IN_MEMORY_MAX_SIZE`. + + Args: + dataset_size (int): Dataset size in bytes. + + Returns: + bool: Whether `dataset_size` is smaller than `config.IN_MEMORY_MAX_SIZE`. + """ + if dataset_size and config.IN_MEMORY_MAX_SIZE: + return dataset_size < config.IN_MEMORY_MAX_SIZE + else: + return False diff --git a/lib/python3.12/site-packages/datasets/utils/logging.py b/lib/python3.12/site-packages/datasets/utils/logging.py new file mode 100644 index 0000000000000000000000000000000000000000..1417f82ac0d42f0206b2bba2516de082a6f4f980 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/utils/logging.py @@ -0,0 +1,178 @@ +# Copyright 2020 Optuna, Hugging Face +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Logging utilities.""" + +import logging +import os +from logging import ( + CRITICAL, # NOQA + DEBUG, # NOQA + ERROR, # NOQA + FATAL, # NOQA + INFO, # NOQA + NOTSET, # NOQA + WARN, # NOQA + WARNING, # NOQA +) +from typing import Optional + +from .tqdm import ( # noqa: F401 # imported for backward compatibility + disable_progress_bar, + enable_progress_bar, + is_progress_bar_enabled, + tqdm, +) + + +log_levels = { + "debug": logging.DEBUG, + "info": logging.INFO, + "warning": logging.WARNING, + "error": logging.ERROR, + "critical": logging.CRITICAL, +} + +_default_log_level = logging.WARNING + + +def _get_default_logging_level(): + """ + If DATASETS_VERBOSITY env var is set to one of the valid choices return that as the new default level. + If it is not - fall back to ``_default_log_level`` + """ + env_level_str = os.getenv("DATASETS_VERBOSITY", None) + if env_level_str: + if env_level_str in log_levels: + return log_levels[env_level_str] + else: + logging.getLogger().warning( + f"Unknown option DATASETS_VERBOSITY={env_level_str}, has to be one of: {', '.join(log_levels.keys())}" + ) + return _default_log_level + + +def _get_library_name() -> str: + return __name__.split(".")[0] + + +def _get_library_root_logger() -> logging.Logger: + return logging.getLogger(_get_library_name()) + + +def _configure_library_root_logger() -> None: + # Apply our default configuration to the library root logger. + library_root_logger = _get_library_root_logger() + library_root_logger.addHandler(logging.StreamHandler()) + library_root_logger.setLevel(_get_default_logging_level()) + + +def _reset_library_root_logger() -> None: + library_root_logger = _get_library_root_logger() + library_root_logger.setLevel(logging.NOTSET) + + +def get_logger(name: Optional[str] = None) -> logging.Logger: + """Return a logger with the specified name. + This function can be used in dataset scripts. + """ + if name is None: + name = _get_library_name() + return logging.getLogger(name) + + +def get_verbosity() -> int: + """Return the current level for the HuggingFace datasets library's root logger. + Returns: + Logging level, e.g., `datasets.logging.DEBUG` and `datasets.logging.INFO`. + + + + HuggingFace datasets library has following logging levels: + - `datasets.logging.CRITICAL`, `datasets.logging.FATAL` + - `datasets.logging.ERROR` + - `datasets.logging.WARNING`, `datasets.logging.WARN` + - `datasets.logging.INFO` + - `datasets.logging.DEBUG` + + + """ + return _get_library_root_logger().getEffectiveLevel() + + +def set_verbosity(verbosity: int) -> None: + """Set the level for the Hugging Face Datasets library's root logger. + Args: + verbosity: + Logging level, e.g., `datasets.logging.DEBUG` and `datasets.logging.INFO`. + """ + _get_library_root_logger().setLevel(verbosity) + + +def set_verbosity_info(): + """Set the level for the Hugging Face datasets library's root logger to `INFO`. + + This will display most of the logging information and tqdm bars. + + Shortcut to `datasets.logging.set_verbosity(datasets.logging.INFO)`. + """ + return set_verbosity(INFO) + + +def set_verbosity_warning(): + """Set the level for the Hugging Face datasets library's root logger to `WARNING`. + + This will display only the warning and errors logging information and tqdm bars. + + Shortcut to `datasets.logging.set_verbosity(datasets.logging.WARNING)`. + """ + return set_verbosity(WARNING) + + +def set_verbosity_debug(): + """Set the level for the Hugging Face datasets library's root logger to `DEBUG`. + + This will display all the logging information and tqdm bars. + + Shortcut to `datasets.logging.set_verbosity(datasets.logging.DEBUG)`. + """ + return set_verbosity(DEBUG) + + +def set_verbosity_error(): + """Set the level for the Hugging Face datasets library's root logger to `ERROR`. + + This will display only the errors logging information and tqdm bars. + + Shortcut to `datasets.logging.set_verbosity(datasets.logging.ERROR)`. + """ + return set_verbosity(ERROR) + + +def disable_propagation() -> None: + """Disable propagation of the library log outputs. + Note that log propagation is disabled by default. + """ + _get_library_root_logger().propagate = False + + +def enable_propagation() -> None: + """Enable propagation of the library log outputs. + Please disable the Hugging Face datasets library's default handler to prevent double logging if the root logger has + been configured. + """ + _get_library_root_logger().propagate = True + + +# Configure the library root logger at the module level (singleton-like) +_configure_library_root_logger() diff --git a/lib/python3.12/site-packages/datasets/utils/metadata.py b/lib/python3.12/site-packages/datasets/utils/metadata.py new file mode 100644 index 0000000000000000000000000000000000000000..cceecf35d63f518fcb7a62c804b9679e5d0c144e --- /dev/null +++ b/lib/python3.12/site-packages/datasets/utils/metadata.py @@ -0,0 +1,233 @@ +import re +import textwrap +from collections import Counter +from itertools import groupby +from operator import itemgetter +from typing import Any, ClassVar, Optional + +import yaml +from huggingface_hub import DatasetCardData + +from ..config import METADATA_CONFIGS_FIELD +from ..features import Features +from ..info import DatasetInfo, DatasetInfosDict +from ..naming import _split_re +from ..utils.logging import get_logger + + +logger = get_logger(__name__) + + +class _NoDuplicateSafeLoader(yaml.SafeLoader): + def _check_no_duplicates_on_constructed_node(self, node): + keys = [self.constructed_objects[key_node] for key_node, _ in node.value] + keys = [tuple(key) if isinstance(key, list) else key for key in keys] + counter = Counter(keys) + duplicate_keys = [key for key in counter if counter[key] > 1] + if duplicate_keys: + raise TypeError(f"Got duplicate yaml keys: {duplicate_keys}") + + def construct_mapping(self, node, deep=False): + mapping = super().construct_mapping(node, deep=deep) + self._check_no_duplicates_on_constructed_node(node) + return mapping + + +def _split_yaml_from_readme(readme_content: str) -> tuple[Optional[str], str]: + full_content = list(readme_content.splitlines()) + if full_content and full_content[0] == "---" and "---" in full_content[1:]: + sep_idx = full_content[1:].index("---") + 1 + yamlblock = "\n".join(full_content[1:sep_idx]) + return yamlblock, "\n".join(full_content[sep_idx + 1 :]) + + return None, "\n".join(full_content) + + +class MetadataConfigs(dict[str, dict[str, Any]]): + """Should be in format {config_name: {**config_params}}.""" + + FIELD_NAME: ClassVar[str] = METADATA_CONFIGS_FIELD + + @staticmethod + def _raise_if_data_files_field_not_valid(metadata_config: dict): + yaml_data_files = metadata_config.get("data_files") + if yaml_data_files is not None: + yaml_error_message = textwrap.dedent( + f""" + Expected data_files in YAML to be either a string or a list of strings + or a list of dicts with two keys: 'split' and 'path', but got {yaml_data_files} + Examples of data_files in YAML: + + data_files: data.csv + + data_files: data/*.png + + data_files: + - part0/* + - part1/* + + data_files: + - split: train + path: train/* + - split: test + path: test/* + + data_files: + - split: train + path: + - train/part1/* + - train/part2/* + - split: test + path: test/* + + PS: some symbols like dashes '-' are not allowed in split names + """ + ) + if not isinstance(yaml_data_files, (list, str)): + raise ValueError(yaml_error_message) + if isinstance(yaml_data_files, list): + for yaml_data_files_item in yaml_data_files: + if ( + not isinstance(yaml_data_files_item, (str, dict)) + or isinstance(yaml_data_files_item, dict) + and not ( + len(yaml_data_files_item) == 2 + and "split" in yaml_data_files_item + and re.match(_split_re, yaml_data_files_item["split"]) + and isinstance(yaml_data_files_item.get("path"), (str, list)) + ) + ): + raise ValueError(yaml_error_message) + + @classmethod + def _from_exported_parquet_files_and_dataset_infos( + cls, + parquet_commit_hash: str, + exported_parquet_files: list[dict[str, Any]], + dataset_infos: DatasetInfosDict, + ) -> "MetadataConfigs": + metadata_configs = { + config_name: { + "data_files": [ + { + "split": split_name, + "path": [ + parquet_file["url"].replace("refs%2Fconvert%2Fparquet", parquet_commit_hash) + for parquet_file in parquet_files_for_split + ], + } + for split_name, parquet_files_for_split in groupby(parquet_files_for_config, itemgetter("split")) + ], + "version": str(dataset_infos.get(config_name, DatasetInfo()).version or "0.0.0"), + } + for config_name, parquet_files_for_config in groupby(exported_parquet_files, itemgetter("config")) + } + if dataset_infos: + # Preserve order of configs and splits + metadata_configs = { + config_name: { + "data_files": [ + data_file + for split_name in dataset_info.splits + for data_file in metadata_configs[config_name]["data_files"] + if data_file["split"] == split_name + ], + "version": metadata_configs[config_name]["version"], + } + for config_name, dataset_info in dataset_infos.items() + } + return cls(metadata_configs) + + @classmethod + def from_dataset_card_data(cls, dataset_card_data: DatasetCardData) -> "MetadataConfigs": + if dataset_card_data.get(cls.FIELD_NAME): + metadata_configs = dataset_card_data[cls.FIELD_NAME] + if not isinstance(metadata_configs, list): + raise ValueError(f"Expected {cls.FIELD_NAME} to be a list, but got '{metadata_configs}'") + for metadata_config in metadata_configs: + if "config_name" not in metadata_config: + raise ValueError( + f"Each config must include `config_name` field with a string name of a config, " + f"but got {metadata_config}. " + ) + cls._raise_if_data_files_field_not_valid(metadata_config) + return cls( + { + config.pop("config_name"): { + param: value if param != "features" else Features._from_yaml_list(value) + for param, value in config.items() + } + for metadata_config in metadata_configs + if (config := metadata_config.copy()) + } + ) + return cls() + + def to_dataset_card_data(self, dataset_card_data: DatasetCardData) -> None: + if self: + for metadata_config in self.values(): + self._raise_if_data_files_field_not_valid(metadata_config) + current_metadata_configs = self.from_dataset_card_data(dataset_card_data) + total_metadata_configs = dict(sorted({**current_metadata_configs, **self}.items())) + for config_name, config_metadata in total_metadata_configs.items(): + config_metadata.pop("config_name", None) + dataset_card_data[self.FIELD_NAME] = [ + {"config_name": config_name, **config_metadata} + for config_name, config_metadata in total_metadata_configs.items() + ] + + def get_default_config_name(self) -> Optional[str]: + default_config_name = None + for config_name, metadata_config in self.items(): + if len(self) == 1 or config_name == "default" or metadata_config.get("default"): + if default_config_name is None: + default_config_name = config_name + else: + raise ValueError( + f"Dataset has several default configs: '{default_config_name}' and '{config_name}'." + ) + return default_config_name + + +# DEPRECATED - just here to support old versions of evaluate like 0.2.2 +# To support new tasks on the Hugging Face Hub, please open a PR for this file: +# https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/src/pipelines.ts +known_task_ids = { + "image-classification": [], + "translation": [], + "image-segmentation": [], + "fill-mask": [], + "automatic-speech-recognition": [], + "token-classification": [], + "sentence-similarity": [], + "audio-classification": [], + "question-answering": [], + "summarization": [], + "zero-shot-classification": [], + "table-to-text": [], + "feature-extraction": [], + "other": [], + "multiple-choice": [], + "text-classification": [], + "text-to-image": [], + "text2text-generation": [], + "zero-shot-image-classification": [], + "tabular-classification": [], + "tabular-regression": [], + "image-to-image": [], + "tabular-to-text": [], + "unconditional-image-generation": [], + "text-retrieval": [], + "text-to-speech": [], + "object-detection": [], + "audio-to-audio": [], + "text-generation": [], + "conversational": [], + "table-question-answering": [], + "visual-question-answering": [], + "image-to-text": [], + "reinforcement-learning": [], + "voice-activity-detection": [], + "time-series-forecasting": [], + "document-question-answering": [], +} diff --git a/lib/python3.12/site-packages/datasets/utils/patching.py b/lib/python3.12/site-packages/datasets/utils/patching.py new file mode 100644 index 0000000000000000000000000000000000000000..f245cabd97065d9e82a1320d02999f9ec03bda36 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/utils/patching.py @@ -0,0 +1,119 @@ +from importlib import import_module + +from .logging import get_logger + + +logger = get_logger(__name__) + + +class _PatchedModuleObj: + """Set all the modules components as attributes of the _PatchedModuleObj object.""" + + def __init__(self, module, attrs=None): + attrs = attrs or [] + if module is not None: + for key in module.__dict__: + if key in attrs or not key.startswith("__"): + setattr(self, key, getattr(module, key)) + self._original_module = module._original_module if isinstance(module, _PatchedModuleObj) else module + + +class patch_submodule: + """ + Patch a submodule attribute of an object, by keeping all other submodules intact at all levels. + + Example:: + + >>> import importlib + >>> from datasets.load import dataset_module_factory + >>> from datasets.streaming import patch_submodule, xjoin + >>> + >>> dataset_module = dataset_module_factory("snli") + >>> snli_module = importlib.import_module(dataset_module.module_path) + >>> patcher = patch_submodule(snli_module, "os.path.join", xjoin) + >>> patcher.start() + >>> assert snli_module.os.path.join is xjoin + """ + + _active_patches = [] + + def __init__(self, obj, target: str, new, attrs=None): + self.obj = obj + self.target = target + self.new = new + self.key = target.split(".")[0] + self.original = {} + self.attrs = attrs or [] + + def __enter__(self): + *submodules, target_attr = self.target.split(".") + + # Patch modules: + # it's used to patch attributes of submodules like "os.path.join"; + # in this case we need to patch "os" and "os.path" + + for i in range(len(submodules)): + try: + submodule = import_module(".".join(submodules[: i + 1])) + except ModuleNotFoundError: + continue + # We iterate over all the globals in self.obj in case we find "os" or "os.path" + for attr in self.obj.__dir__(): + obj_attr = getattr(self.obj, attr) + # We don't check for the name of the global, but rather if its value *is* "os" or "os.path". + # This allows to patch renamed modules like "from os import path as ospath". + if obj_attr is submodule or ( + isinstance(obj_attr, _PatchedModuleObj) and obj_attr._original_module is submodule + ): + self.original[attr] = obj_attr + # patch at top level + setattr(self.obj, attr, _PatchedModuleObj(obj_attr, attrs=self.attrs)) + patched = getattr(self.obj, attr) + # construct lower levels patches + for key in submodules[i + 1 :]: + setattr(patched, key, _PatchedModuleObj(getattr(patched, key, None), attrs=self.attrs)) + patched = getattr(patched, key) + # finally set the target attribute + setattr(patched, target_attr, self.new) + + # Patch attribute itself: + # it's used for builtins like "open", + # and also to patch "os.path.join" we may also need to patch "join" + # itself if it was imported as "from os.path import join". + + if submodules: # if it's an attribute of a submodule like "os.path.join" + try: + attr_value = getattr(import_module(".".join(submodules)), target_attr) + except (AttributeError, ModuleNotFoundError): + return + # We iterate over all the globals in self.obj in case we find "os.path.join" + for attr in self.obj.__dir__(): + # We don't check for the name of the global, but rather if its value *is* "os.path.join". + # This allows to patch renamed attributes like "from os.path import join as pjoin". + if getattr(self.obj, attr) is attr_value: + self.original[attr] = getattr(self.obj, attr) + setattr(self.obj, attr, self.new) + elif target_attr in globals()["__builtins__"]: # if it'a s builtin like "open" + self.original[target_attr] = globals()["__builtins__"][target_attr] + setattr(self.obj, target_attr, self.new) + else: + raise RuntimeError(f"Tried to patch attribute {target_attr} instead of a submodule.") + + def __exit__(self, *exc_info): + for attr in list(self.original): + setattr(self.obj, attr, self.original.pop(attr)) + + def start(self): + """Activate a patch.""" + self.__enter__() + self._active_patches.append(self) + + def stop(self): + """Stop an active patch.""" + try: + self._active_patches.remove(self) + except ValueError: + # If the patch hasn't been started this will fail + return None + + return self.__exit__() diff --git a/lib/python3.12/site-packages/datasets/utils/py_utils.py b/lib/python3.12/site-packages/datasets/utils/py_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..9d09b9b7e591ff119bc504aa5820ec5d585e0e80 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/utils/py_utils.py @@ -0,0 +1,744 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +"""Some python utils function and classes.""" + +import copy +import functools +import itertools +import multiprocessing.pool +import os +import queue +import re +import types +import warnings +from collections.abc import Iterable +from contextlib import contextmanager +from dataclasses import fields, is_dataclass +from multiprocessing import Manager +from pathlib import Path +from queue import Empty +from shutil import disk_usage +from typing import Any, Callable, Optional, TypeVar, Union +from urllib.parse import urlparse + +import multiprocess +import multiprocess.pool +import numpy as np +from tqdm.auto import tqdm + +from .. import config +from ..parallel import parallel_map +from . import logging +from . import tqdm as hf_tqdm +from ._dill import ( # noqa: F401 # imported for backward compatibility. TODO: remove in 3.0.0 + Pickler, + dump, + dumps, + pklregister, +) +from ._filelock import FileLock + + +try: # pragma: no branch + from typing import Final + + import typing_extensions as _typing_extensions + from typing_extensions import Literal +except ImportError: + _typing_extensions = Literal = Final = None + + +logger = logging.get_logger(__name__) + + +# NOTE: When used on an instance method, the cache is shared across all +# instances and IS NOT per-instance. +# See +# https://stackoverflow.com/questions/14946264/python-lru-cache-decorator-per-instance +# For @property methods, use @memoized_property below. +memoize = functools.lru_cache + + +def size_str(size_in_bytes): + """Returns a human readable size string. + + If size_in_bytes is None, then returns "Unknown size". + + For example `size_str(1.5 * datasets.units.GiB) == "1.50 GiB"`. + + Args: + size_in_bytes: `int` or `None`, the size, in bytes, that we want to + format as a human-readable size string. + """ + if not size_in_bytes: + return "Unknown size" + + _NAME_LIST = [("PiB", 2**50), ("TiB", 2**40), ("GiB", 2**30), ("MiB", 2**20), ("KiB", 2**10)] + + size_in_bytes = float(size_in_bytes) + for name, size_bytes in _NAME_LIST: + value = size_in_bytes / size_bytes + if value >= 1.0: + return f"{value:.2f} {name}" + return f"{int(size_in_bytes)} bytes" + + +def convert_file_size_to_int(size: Union[int, str]) -> int: + """ + Converts a size expressed as a string with digits an unit (like `"50MB"`) to an integer (in bytes). + + Args: + size (`int` or `str`): The size to convert. Will be directly returned if an `int`. + + Example: + + ```py + >>> convert_file_size_to_int("1MiB") + 1048576 + ``` + """ + if isinstance(size, int): + return size + if size.upper().endswith("PIB"): + return int(size[:-3]) * (2**50) + if size.upper().endswith("TIB"): + return int(size[:-3]) * (2**40) + if size.upper().endswith("GIB"): + return int(size[:-3]) * (2**30) + if size.upper().endswith("MIB"): + return int(size[:-3]) * (2**20) + if size.upper().endswith("KIB"): + return int(size[:-3]) * (2**10) + if size.upper().endswith("PB"): + int_size = int(size[:-2]) * (10**15) + return int_size // 8 if size.endswith("b") else int_size + if size.upper().endswith("TB"): + int_size = int(size[:-2]) * (10**12) + return int_size // 8 if size.endswith("b") else int_size + if size.upper().endswith("GB"): + int_size = int(size[:-2]) * (10**9) + return int_size // 8 if size.endswith("b") else int_size + if size.upper().endswith("MB"): + int_size = int(size[:-2]) * (10**6) + return int_size // 8 if size.endswith("b") else int_size + if size.upper().endswith("KB"): + int_size = int(size[:-2]) * (10**3) + return int_size // 8 if size.endswith("b") else int_size + raise ValueError(f"`size={size}` is not in a valid format. Use an integer followed by the unit, e.g., '5GB'.") + + +def glob_pattern_to_regex(pattern): + # partially taken from fsspec: + # https://github.com/fsspec/filesystem_spec/blob/697d0f8133d8a5fbc3926e4761d7ecd51337ce50/fsspec/asyn.py#L735 + return ( + pattern.replace("\\", r"\\") + .replace(".", r"\.") + .replace("*", ".*") + .replace("+", r"\+") + .replace("//", "/") + .replace("(", r"\(") + .replace(")", r"\)") + .replace("|", r"\|") + .replace("^", r"\^") + .replace("$", r"\$") + .rstrip("/") + .replace("?", ".") + ) + + +def string_to_dict(string: str, pattern: str) -> Optional[dict[str, str]]: + """Un-format a string using a python f-string pattern. + From https://stackoverflow.com/a/36838374 + + Example:: + + >>> p = 'hello, my name is {name} and I am a {age} year old {what}' + >>> s = p.format(name='cody', age=18, what='quarterback') + >>> s + 'hello, my name is cody and I am a 18 year old quarterback' + >>> string_to_dict(s, p) + {'age': '18', 'name': 'cody', 'what': 'quarterback'} + + Args: + string (str): input string + pattern (str): pattern formatted like a python f-string + + Returns: + Optional[dict[str, str]]: dictionary of variable -> value, retrieved from the input using the pattern, or + `None` if the string does not match the pattern. + """ + pattern = re.sub(r"{([^:}]+)(?::[^}]+)?}", r"{\1}", pattern) # remove format specifiers, e.g. {rank:05d} -> {rank} + regex = re.sub(r"{(.+?)}", r"(?P<_\1>.+)", pattern) + result = re.search(regex, string) + if result is None: + return None + values = list(result.groups()) + keys = re.findall(r"{(.+?)}", pattern) + _dict = dict(zip(keys, values)) + return _dict + + +def asdict(obj): + """Convert an object to its dictionary representation recursively. + + + """ + + # Implementation based on https://docs.python.org/3/library/dataclasses.html#dataclasses.asdict + + def _is_dataclass_instance(obj): + # https://docs.python.org/3/library/dataclasses.html#dataclasses.is_dataclass + return is_dataclass(obj) and not isinstance(obj, type) + + def _asdict_inner(obj): + if _is_dataclass_instance(obj): + result = {} + for f in fields(obj): + value = _asdict_inner(getattr(obj, f.name)) + if not f.init or value != f.default or f.metadata.get("include_in_asdict_even_if_is_default", False): + result[f.name] = value + return result + elif isinstance(obj, tuple) and hasattr(obj, "_fields"): + # obj is a namedtuple + return type(obj)(*[_asdict_inner(v) for v in obj]) + elif isinstance(obj, (list, tuple)): + # Assume we can create an object of this type by passing in a + # generator (which is not true for namedtuples, handled + # above). + return type(obj)(_asdict_inner(v) for v in obj) + elif isinstance(obj, dict): + return {_asdict_inner(k): _asdict_inner(v) for k, v in obj.items()} + else: + return copy.deepcopy(obj) + + if not isinstance(obj, dict) and not _is_dataclass_instance(obj): + raise TypeError(f"{obj} is not a dict or a dataclass") + + return _asdict_inner(obj) + + +@contextmanager +def temporary_assignment(obj, attr, value): + """Temporarily assign obj.attr to value.""" + original = getattr(obj, attr, None) + setattr(obj, attr, value) + try: + yield + finally: + setattr(obj, attr, original) + + +@contextmanager +def temp_seed(seed: int, set_pytorch=False, set_tensorflow=False): + """Temporarily set the random seed. This works for python numpy, pytorch and tensorflow.""" + np_state = np.random.get_state() + np.random.seed(seed) + + if set_pytorch and config.TORCH_AVAILABLE: + import torch + + torch_state = torch.random.get_rng_state() + torch.random.manual_seed(seed) + + if torch.cuda.is_available(): + torch_cuda_states = torch.cuda.get_rng_state_all() + torch.cuda.manual_seed_all(seed) + + if set_tensorflow and config.TF_AVAILABLE: + import tensorflow as tf + from tensorflow.python.eager import context as tfpycontext + + tf_state = tf.random.get_global_generator() + temp_gen = tf.random.Generator.from_seed(seed) + tf.random.set_global_generator(temp_gen) + + if not tf.executing_eagerly(): + raise ValueError("Setting random seed for TensorFlow is only available in eager mode") + + tf_context = tfpycontext.context() # eager mode context + tf_seed = tf_context._seed + tf_rng_initialized = hasattr(tf_context, "_rng") + if tf_rng_initialized: + tf_rng = tf_context._rng + tf_context._set_global_seed(seed) + + try: + yield + finally: + np.random.set_state(np_state) + + if set_pytorch and config.TORCH_AVAILABLE: + torch.random.set_rng_state(torch_state) + if torch.cuda.is_available(): + torch.cuda.set_rng_state_all(torch_cuda_states) + + if set_tensorflow and config.TF_AVAILABLE: + tf.random.set_global_generator(tf_state) + + tf_context._seed = tf_seed + if tf_rng_initialized: + tf_context._rng = tf_rng + else: + delattr(tf_context, "_rng") + + +def unique_values(values): + """Iterate over iterable and return only unique values in order.""" + seen = set() + for value in values: + if value not in seen: + seen.add(value) + yield value + + +def no_op_if_value_is_null(func): + """If the value is None, return None, else call `func`.""" + + def wrapper(value): + return func(value) if value is not None else None + + return wrapper + + +def first_non_null_value(iterable): + """Return the index and the value of the first non-null value in the iterable. If all values are None, return -1 as index.""" + for i, value in enumerate(iterable): + if value is not None: + return i, value + return -1, None + + +def first_non_null_non_empty_value(iterable): + """Return the index and the value of the first non-null non-empty value in the iterable. If all values are None or empty, return -1 as index.""" + for i, value in enumerate(iterable): + if value is not None and not (isinstance(value, (dict, list)) and len(value) == 0): + return i, value + return -1, None + + +def zip_dict(*dicts): + """Iterate over items of dictionaries grouped by their keys.""" + for key in unique_values(itertools.chain(*dicts)): # set merge all keys + # Will raise KeyError if the dict don't have the same keys + yield key, tuple(d[key] for d in dicts) + + +class NonMutableDict(dict): + """Dict where keys can only be added but not modified. + + Will raise an error if the user try to overwrite one key. The error message + can be customized during construction. It will be formatted using {key} for + the overwritten key. + """ + + def __init__(self, *args, **kwargs): + self._error_msg = kwargs.pop( + "error_msg", + "Try to overwrite existing key: {key}", + ) + if kwargs: + raise ValueError("NonMutableDict cannot be initialized with kwargs.") + super().__init__(*args, **kwargs) + + def __setitem__(self, key, value): + if key in self: + raise ValueError(self._error_msg.format(key=key)) + return super().__setitem__(key, value) + + def update(self, other): + if any(k in self for k in other): + raise ValueError(self._error_msg.format(key=set(self) & set(other))) + return super().update(other) + + +class classproperty(property): # pylint: disable=invalid-name + """Descriptor to be used as decorator for @classmethods.""" + + def __get__(self, obj, objtype=None): + return self.fget.__get__(None, objtype)() + + +def _single_map_nested(args): + """Apply a function recursively to each element of a nested data struct.""" + function, data_struct, batched, batch_size, types, rank, disable_tqdm, desc = args + + # Singleton first to spare some computation + if not isinstance(data_struct, dict) and not isinstance(data_struct, types): + if batched: + return function([data_struct])[0] + else: + return function(data_struct) + if ( + batched + and not isinstance(data_struct, dict) + and isinstance(data_struct, types) + and all(not isinstance(v, (dict, types)) for v in data_struct) + ): + return [mapped_item for batch in iter_batched(data_struct, batch_size) for mapped_item in function(batch)] + + # Reduce logging to keep things readable in multiprocessing with tqdm + if rank is not None and logging.get_verbosity() < logging.WARNING: + logging.set_verbosity_warning() + # Print at least one thing to fix tqdm in notebooks in multiprocessing + # see https://github.com/tqdm/tqdm/issues/485#issuecomment-473338308 + if rank is not None and not disable_tqdm and any("notebook" in tqdm_cls.__name__ for tqdm_cls in tqdm.__mro__): + print(" ", end="", flush=True) + + # Loop over single examples or batches and write to buffer/file if examples are to be updated + pbar_iterable = data_struct.items() if isinstance(data_struct, dict) else data_struct + pbar_desc = (desc + " " if desc is not None else "") + "#" + str(rank) if rank is not None else desc + with hf_tqdm(pbar_iterable, disable=disable_tqdm, position=rank, unit="obj", desc=pbar_desc) as pbar: + if isinstance(data_struct, dict): + return { + k: _single_map_nested((function, v, batched, batch_size, types, None, True, None)) for k, v in pbar + } + else: + mapped = [_single_map_nested((function, v, batched, batch_size, types, None, True, None)) for v in pbar] + if isinstance(data_struct, list): + return mapped + elif isinstance(data_struct, tuple): + return tuple(mapped) + else: + return np.array(mapped) + + +def map_nested( + function: Callable[[Any], Any], + data_struct: Any, + dict_only: bool = False, + map_list: bool = True, + map_tuple: bool = False, + map_numpy: bool = False, + num_proc: Optional[int] = None, + parallel_min_length: int = 2, + batched: bool = False, + batch_size: Optional[int] = 1000, + types: Optional[tuple] = None, + disable_tqdm: bool = True, + desc: Optional[str] = None, +) -> Any: + """Apply a function recursively to each element of a nested data struct. + + Use multiprocessing if num_proc > 1 and the length of data_struct is greater than or equal to + `parallel_min_length`. + + + + Before version 2.5.0, multiprocessing was not used if `num_proc` was greater than or equal to ``len(iterable)``. + + Now, if `num_proc` is greater than or equal to ``len(iterable)``, `num_proc` is set to ``len(iterable)`` and + multiprocessing is used. + + + + Args: + function (`Callable`): Function to be applied to `data_struct`. + data_struct (`Any`): Data structure to apply `function` to. + dict_only (`bool`, default `False`): Whether only apply `function` recursively to `dict` values in + `data_struct`. + map_list (`bool`, default `True`): Whether also apply `function` recursively to `list` elements (besides `dict` + values). + map_tuple (`bool`, default `False`): Whether also apply `function` recursively to `tuple` elements (besides + `dict` values). + map_numpy (`bool, default `False`): Whether also apply `function` recursively to `numpy.array` elements (besides + `dict` values). + num_proc (`int`, *optional*): Number of processes. + The level in the data struct used for multiprocessing is the first level that has smaller sub-structs, + starting from the root. + parallel_min_length (`int`, default `2`): Minimum length of `data_struct` required for parallel + processing. + + batched (`bool`, defaults to `False`): + Provide batch of items to `function`. + + batch_size (`int`, *optional*, defaults to `1000`): + Number of items per batch provided to `function` if `batched=True`. + If `batch_size <= 0` or `batch_size == None`, provide the full iterable as a single batch to `function`. + + types (`tuple`, *optional*): Additional types (besides `dict` values) to apply `function` recursively to their + elements. + disable_tqdm (`bool`, default `True`): Whether to disable the tqdm progressbar. + desc (`str`, *optional*): Prefix for the tqdm progressbar. + + Returns: + `Any` + """ + if types is None: + types = [] + if not dict_only: + if map_list: + types.append(list) + if map_tuple: + types.append(tuple) + if map_numpy: + types.append(np.ndarray) + types = tuple(types) + + # Singleton + if not isinstance(data_struct, dict) and not isinstance(data_struct, types): + if batched: + data_struct = [data_struct] + mapped = function(data_struct) + if batched: + mapped = mapped[0] + return mapped + + iterable = list(data_struct.values()) if isinstance(data_struct, dict) else data_struct + + if num_proc is None: + num_proc = 1 + if any(isinstance(v, types) and len(v) > len(iterable) for v in iterable): + mapped = [ + map_nested( + function=function, + data_struct=obj, + num_proc=num_proc, + parallel_min_length=parallel_min_length, + batched=batched, + batch_size=batch_size, + types=types, + ) + for obj in iterable + ] + elif num_proc != -1 and num_proc <= 1 or len(iterable) < parallel_min_length: + if batched: + if batch_size is None or batch_size <= 0: + batch_size = max(len(iterable) // num_proc + int(len(iterable) % num_proc > 0), 1) + iterable = list(iter_batched(iterable, batch_size)) + mapped = [ + _single_map_nested((function, obj, batched, batch_size, types, None, True, None)) + for obj in hf_tqdm(iterable, disable=disable_tqdm, desc=desc) + ] + if batched: + mapped = [mapped_item for mapped_batch in mapped for mapped_item in mapped_batch] + else: + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + message=".* is experimental and might be subject to breaking changes in the future\\.$", + category=UserWarning, + ) + if batched: + if batch_size is None or batch_size <= 0: + batch_size = len(iterable) // num_proc + int(len(iterable) % num_proc > 0) + iterable = list(iter_batched(iterable, batch_size)) + mapped = parallel_map( + function, iterable, num_proc, batched, batch_size, types, disable_tqdm, desc, _single_map_nested + ) + if batched: + mapped = [mapped_item for mapped_batch in mapped for mapped_item in mapped_batch] + + if isinstance(data_struct, dict): + return dict(zip(data_struct.keys(), mapped)) + else: + if isinstance(data_struct, list): + return mapped + elif isinstance(data_struct, tuple): + return tuple(mapped) + else: + return np.array(mapped) + + +class NestedDataStructure: + def __init__(self, data=None): + self.data = data if data is not None else [] + + def flatten(self, data=None): + data = data if data is not None else self.data + if isinstance(data, dict): + return self.flatten(list(data.values())) + elif isinstance(data, (list, tuple)): + return [flattened for item in data for flattened in self.flatten(item)] + else: + return [data] + + +def has_sufficient_disk_space(needed_bytes, directory="."): + try: + free_bytes = disk_usage(os.path.abspath(directory)).free + except OSError: + return True + return needed_bytes < free_bytes + + +def _convert_github_url(url_path: str) -> tuple[str, Optional[str]]: + """Convert a link to a file on a github repo in a link to the raw github object.""" + parsed = urlparse(url_path) + sub_directory = None + if parsed.scheme in ("http", "https", "s3") and parsed.netloc == "github.com": + if "blob" in url_path: + if not url_path.endswith(".py"): + raise ValueError(f"External import from github at {url_path} should point to a file ending with '.py'") + url_path = url_path.replace("blob", "raw") # Point to the raw file + else: + # Parse github url to point to zip + github_path = parsed.path[1:] + repo_info, branch = github_path.split("/tree/") if "/tree/" in github_path else (github_path, "master") + repo_owner, repo_name = repo_info.split("/") + url_path = f"https://github.com/{repo_owner}/{repo_name}/archive/{branch}.zip" + sub_directory = f"{repo_name}-{branch}" + return url_path, sub_directory + + +def lock_importable_file(importable_local_file: str) -> FileLock: + # Check the directory with a unique name in our dataset folder + # path is: ./datasets/dataset_name/hash_from_code/script.py + # we use a hash as subdirectory_name to be able to have multiple versions of a dataset processing file together + importable_directory_path = str(Path(importable_local_file).resolve().parent.parent) + lock_path = importable_directory_path + ".lock" + return FileLock(lock_path) + + +def get_imports(file_path: str) -> tuple[str, str, str, str]: + """Find whether we should import or clone additional files for a given processing script. + And list the import. + + We allow: + - library dependencies, + - local dependencies and + - external dependencies whose url is specified with a comment starting from "# From:' followed by the raw url to a file, an archive or a github repository. + external dependencies will be downloaded (and extracted if needed in the dataset folder). + We also add an `__init__.py` to each sub-folder of a downloaded folder so the user can import from them in the script. + + Note that only direct import in the dataset processing script will be handled + We don't recursively explore the additional import to download further files. + + Example:: + + import tensorflow + import .c4_utils + import .clicr.dataset-code.build_json_dataset # From: https://raw.githubusercontent.com/clips/clicr/master/dataset-code/build_json_dataset + """ + lines = [] + with open(file_path, encoding="utf-8") as f: + lines.extend(f.readlines()) + + logger.debug(f"Checking {file_path} for additional imports.") + imports: list[tuple[str, str, str, Optional[str]]] = [] + is_in_docstring = False + for line in lines: + docstr_start_match = re.findall(r'[\s\S]*?"""[\s\S]*?', line) + + if len(docstr_start_match) == 1: + # flip True <=> False only if doctstring + # starts at line without finishing + is_in_docstring = not is_in_docstring + + if is_in_docstring: + # import statements in doctstrings should + # not be added as required dependencies + continue + + match = re.match(r"^import\s+(\.?)([^\s\.]+)[^#\r\n]*(?:#\s+From:\s+)?([^\r\n]*)", line, flags=re.MULTILINE) + if match is None: + match = re.match( + r"^from\s+(\.?)([^\s\.]+)(?:[^\s]*)\s+import\s+[^#\r\n]*(?:#\s+From:\s+)?([^\r\n]*)", + line, + flags=re.MULTILINE, + ) + if match is None: + continue + if match.group(1): + # The import starts with a '.', we will download the relevant file + if any(imp[1] == match.group(2) for imp in imports): + # We already have this import + continue + if match.group(3): + # The import has a comment with 'From:', we'll retrieve it from the given url + url_path = match.group(3) + url_path, sub_directory = _convert_github_url(url_path) + imports.append(("external", match.group(2), url_path, sub_directory)) + elif match.group(2): + # The import should be at the same place as the file + imports.append(("internal", match.group(2), match.group(2), None)) + else: + if match.group(3): + # The import has a comment with `From: git+https:...`, asks user to pip install from git. + url_path = match.group(3) + imports.append(("library", match.group(2), url_path, None)) + else: + imports.append(("library", match.group(2), match.group(2), None)) + + return imports + + +def copyfunc(func): + result = types.FunctionType(func.__code__, func.__globals__, func.__name__, func.__defaults__, func.__closure__) + result.__kwdefaults__ = func.__kwdefaults__ + return result + + +Y = TypeVar("Y") + + +def _write_generator_to_queue(queue: queue.Queue, func: Callable[..., Iterable[Y]], kwargs: dict) -> int: + for i, result in enumerate(func(**kwargs)): + queue.put(result) + return i + + +def _get_pool_pid(pool: Union[multiprocessing.pool.Pool, multiprocess.pool.Pool]) -> set[int]: + return {f.pid for f in pool._pool} + + +def iflatmap_unordered( + pool: Union[multiprocessing.pool.Pool, multiprocess.pool.Pool], + func: Callable[..., Iterable[Y]], + *, + kwargs_iterable: Iterable[dict], +) -> Iterable[Y]: + initial_pool_pid = _get_pool_pid(pool) + pool_changed = False + manager_cls = Manager if isinstance(pool, multiprocessing.pool.Pool) else multiprocess.Manager + with manager_cls() as manager: + queue = manager.Queue() + async_results = [ + pool.apply_async(_write_generator_to_queue, (queue, func, kwargs)) for kwargs in kwargs_iterable + ] + try: + while True: + try: + yield queue.get(timeout=0.05) + except Empty: + if all(async_result.ready() for async_result in async_results) and queue.empty(): + break + if _get_pool_pid(pool) != initial_pool_pid: + pool_changed = True + # One of the subprocesses has died. We should not wait forever. + raise RuntimeError( + "One of the subprocesses has abruptly died during map operation." + "To debug the error, disable multiprocessing." + ) + finally: + if not pool_changed: + # we get the result in case there's an error to raise + [async_result.get(timeout=0.05) for async_result in async_results] + + +T = TypeVar("T") + + +def iter_batched(iterable: Iterable[T], n: int) -> Iterable[list[T]]: + if n < 1: + raise ValueError(f"Invalid batch size {n}") + batch = [] + for item in iterable: + batch.append(item) + if len(batch) == n: + yield batch + batch = [] + if batch: + yield batch diff --git a/lib/python3.12/site-packages/datasets/utils/resources/__init__.py b/lib/python3.12/site-packages/datasets/utils/resources/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lib/python3.12/site-packages/datasets/utils/resources/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/datasets/utils/resources/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ff69dd048ad533f5aa27cff123b2203d18e56b77 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/utils/resources/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/utils/resources/creators.json b/lib/python3.12/site-packages/datasets/utils/resources/creators.json new file mode 100644 index 0000000000000000000000000000000000000000..d9e15f0039cc27ed8abd9fdf394423a3fada2c95 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/utils/resources/creators.json @@ -0,0 +1,17 @@ +{ + "language": [ + "found", + "crowdsourced", + "expert-generated", + "machine-generated", + "other" + ], + "annotations": [ + "found", + "crowdsourced", + "expert-generated", + "machine-generated", + "no-annotation", + "other" + ] +} diff --git a/lib/python3.12/site-packages/datasets/utils/resources/languages.json b/lib/python3.12/site-packages/datasets/utils/resources/languages.json new file mode 100644 index 0000000000000000000000000000000000000000..ea7686f956b898af3faf97b86be89b71d88855d4 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/utils/resources/languages.json @@ -0,0 +1,8026 @@ +{ + "code": "Programming language (C++, Java, Javascript, Python, etc.)", + "aa": "Afar", + "aaa": "Ghotuo", + "aab": "Alumu-Tesu", + "aac": "Ari", + "aad": "Amal", + "aae": "Arbëreshë Albanian", + "aaf": "Aranadan", + "aag": "Ambrak", + "aah": "Abu' Arapesh", + "aai": "Arifama-Miniafia", + "aak": "Ankave", + "aal": "Afade", + "aan": "Anambé", + "aao": "Algerian Saharan Arabic", + "aap": "Pará Arára", + "aaq": "Eastern Abnaki", + "aas": "Aasáx", + "aat": "Arvanitika Albanian", + "aau": "Abau", + "aav": "Austro-Asiatic languages", + "aaw": "Solong", + "aax": "Mandobo Atas", + "aaz": "Amarasi", + "ab": "Abkhazian", + "aba": "Abé", + "abb": "Bankon", + "abc": "Ambala Ayta", + "abd": "Manide", + "abe": "Western Abnaki", + "abf": "Abai Sungai", + "abg": "Abaga", + "abh": "Tajiki Arabic", + "abi": "Abidji", + "abj": "Aka-Bea", + "abl": "Lampung Nyo", + "abm": "Abanyom", + "abn": "Abua", + "abo": "Abon", + "abp": "Abellen Ayta", + "abq": "Abaza", + "abr": "Abron", + "abs": "Ambonese Malay", + "abt": "Ambulas", + "abu": "Abure", + "abv": "Baharna Arabic", + "abw": "Pal", + "abx": "Inabaknon", + "aby": "Aneme Wake", + "abz": "Abui", + "aca": "Achagua", + "acb": "Áncá", + "acd": "Gikyode", + "ace": "Achinese", + "acf": "Saint Lucian Creole French", + "ach": "Acoli", + "aci": "Aka-Cari", + "ack": "Aka-Kora", + "acl": "Akar-Bale", + "acm": "Mesopotamian Arabic", + "acn": "Achang", + "acp": "Eastern Acipa", + "acq": "Ta'izzi-Adeni Arabic", + "acr": "Achi", + "acs": "Acroá", + "act": "Achterhoeks", + "acu": "Achuar-Shiwiar", + "acv": "Achumawi", + "acw": "Hijazi Arabic", + "acx": "Omani Arabic", + "acy": "Cypriot Arabic", + "acz": "Acheron", + "ada": "Adangme", + "adb": "Atauran", + "add": "Lidzonka; Dzodinka", + "ade": "Adele", + "adf": "Dhofari Arabic", + "adg": "Andegerebinha", + "adh": "Adhola", + "adi": "Adi", + "adj": "Adioukrou", + "adl": "Galo", + "adn": "Adang", + "ado": "Abu", + "adq": "Adangbe", + "adr": "Adonara", + "ads": "Adamorobe Sign Language", + "adt": "Adnyamathanha", + "adu": "Aduge", + "adw": "Amundava", + "adx": "Amdo Tibetan", + "ady": "Adyghe; Adygei", + "adz": "Adzera", + "ae": "Avestan", + "aea": "Areba", + "aeb": "Tunisian Arabic", + "aec": "Saidi Arabic", + "aed": "Argentine Sign Language", + "aee": "Northeast Pashai; Northeast Pashayi", + "aek": "Haeke", + "ael": "Ambele", + "aem": "Arem", + "aen": "Armenian Sign Language", + "aeq": "Aer", + "aer": "Eastern Arrernte", + "aes": "Alsea", + "aeu": "Akeu", + "aew": "Ambakich", + "aey": "Amele", + "aez": "Aeka", + "af": "Afrikaans", + "afa": "Afro-Asiatic languages", + "afb": "Gulf Arabic", + "afd": "Andai", + "afe": "Putukwam", + "afg": "Afghan Sign Language", + "afh": "Afrihili", + "afi": "Akrukay; Chini", + "afk": "Nanubae", + "afn": "Defaka", + "afo": "Eloyi", + "afp": "Tapei", + "afs": "Afro-Seminole Creole", + "aft": "Afitti", + "afu": "Awutu", + "afz": "Obokuitai", + "aga": "Aguano", + "agb": "Legbo", + "agc": "Agatu", + "agd": "Agarabi", + "age": "Angal", + "agf": "Arguni", + "agg": "Angor", + "agh": "Ngelima", + "agi": "Agariya", + "agj": "Argobba", + "agk": "Isarog Agta", + "agl": "Fembe", + "agm": "Angaataha", + "agn": "Agutaynen", + "ago": "Tainae", + "agq": "Aghem", + "agr": "Aguaruna", + "ags": "Esimbi", + "agt": "Central Cagayan Agta", + "agu": "Aguacateco", + "agv": "Remontado Dumagat", + "agw": "Kahua", + "agx": "Aghul", + "agy": "Southern Alta", + "agz": "Mt. Iriga Agta", + "aha": "Ahanta", + "ahb": "Axamb", + "ahg": "Qimant", + "ahh": "Aghu", + "ahi": "Tiagbamrin Aizi", + "ahk": "Akha", + "ahl": "Igo", + "ahm": "Mobumrin Aizi", + "ahn": "Àhàn", + "aho": "Ahom", + "ahp": "Aproumu Aizi", + "ahr": "Ahirani", + "ahs": "Ashe", + "aht": "Ahtena", + "aia": "Arosi", + "aib": "Ainu (China)", + "aic": "Ainbai", + "aid": "Alngith", + "aie": "Amara", + "aif": "Agi", + "aig": "Antigua and Barbuda Creole English", + "aih": "Ai-Cham", + "aii": "Assyrian Neo-Aramaic", + "aij": "Lishanid Noshan", + "aik": "Ake", + "ail": "Aimele", + "aim": "Aimol", + "ain": "Ainu (Japan)", + "aio": "Aiton", + "aip": "Burumakok", + "aiq": "Aimaq", + "air": "Airoran", + "ait": "Arikem", + "aiw": "Aari", + "aix": "Aighon", + "aiy": "Ali", + "aja": "Aja (South Sudan)", + "ajg": "Aja (Benin)", + "aji": "Ajië", + "ajn": "Andajin", + "ajp": "South Levantine Arabic", + "ajs": "Algerian Jewish Sign Language", + "aju": "Judeo-Moroccan Arabic", + "ajw": "Ajawa", + "ajz": "Amri Karbi", + "ak": "Akan", + "akb": "Batak Angkola", + "akc": "Mpur", + "akd": "Ukpet-Ehom", + "ake": "Akawaio", + "akf": "Akpa", + "akg": "Anakalangu", + "akh": "Angal Heneng", + "aki": "Aiome", + "akj": "Aka-Jeru", + "akk": "Akkadian", + "akl": "Aklanon", + "akm": "Aka-Bo", + "ako": "Akurio", + "akp": "Siwu", + "akq": "Ak", + "akr": "Araki", + "aks": "Akaselem", + "akt": "Akolet", + "aku": "Akum", + "akv": "Akhvakh", + "akw": "Akwa", + "akx": "Aka-Kede", + "aky": "Aka-Kol", + "akz": "Alabama", + "ala": "Alago", + "alc": "Qawasqar", + "ald": "Alladian", + "ale": "Aleut", + "alf": "Alege", + "alg": "Algonquian languages", + "alh": "Alawa", + "ali": "Amaimon", + "alj": "Alangan", + "alk": "Alak", + "all": "Allar", + "alm": "Amblong", + "aln": "Gheg Albanian", + "alo": "Larike-Wakasihu", + "alp": "Alune", + "alq": "Algonquin", + "alr": "Alutor", + "als": "Tosk Albanian", + "alt": "Southern Altai", + "alu": "'Are'are", + "alv": "Atlantic-Congo languages", + "alw": "Alaba-K’abeena; Wanbasana", + "alx": "Amol", + "aly": "Alyawarr", + "alz": "Alur", + "am": "Amharic", + "ama": "Amanayé", + "amb": "Ambo", + "amc": "Amahuaca", + "ame": "Yanesha'", + "amf": "Hamer-Banna", + "amg": "Amurdak", + "ami": "Amis", + "amj": "Amdang", + "amk": "Ambai", + "aml": "War-Jaintia", + "amm": "Ama (Papua New Guinea)", + "amn": "Amanab", + "amo": "Amo", + "amp": "Alamblak", + "amq": "Amahai", + "amr": "Amarakaeri", + "ams": "Southern Amami-Oshima", + "amt": "Amto", + "amu": "Guerrero Amuzgo", + "amv": "Ambelau", + "amw": "Western Neo-Aramaic", + "amx": "Anmatyerre", + "amy": "Ami", + "amz": "Atampaya", + "an": "Aragonese", + "ana": "Andaqui", + "anb": "Andoa", + "anc": "Ngas", + "and": "Ansus", + "ane": "Xârâcùù", + "anf": "Animere", + "ang": "Old English (ca. 450-1100)", + "anh": "Nend", + "ani": "Andi", + "anj": "Anor", + "ank": "Goemai", + "anl": "Anu-Hkongso Chin", + "anm": "Anal", + "ann": "Obolo", + "ano": "Andoque", + "anp": "Angika", + "anq": "Jarawa (India)", + "anr": "Andh", + "ans": "Anserma", + "ant": "Antakarinya; Antikarinya", + "anu": "Anuak", + "anv": "Denya", + "anw": "Anaang", + "anx": "Andra-Hus", + "any": "Anyin", + "anz": "Anem", + "aoa": "Angolar", + "aob": "Abom", + "aoc": "Pemon", + "aod": "Andarum", + "aoe": "Angal Enen", + "aof": "Bragat", + "aog": "Angoram", + "aoi": "Anindilyakwa", + "aoj": "Mufian", + "aok": "Arhö", + "aol": "Alor", + "aom": "Ömie", + "aon": "Bumbita Arapesh", + "aor": "Aore", + "aos": "Taikat", + "aot": "Atong (India); A'tong", + "aou": "A'ou", + "aox": "Atorada", + "aoz": "Uab Meto", + "apa": "Apache languages", + "apb": "Sa'a", + "apc": "North Levantine Arabic", + "apd": "Sudanese Arabic", + "ape": "Bukiyip", + "apf": "Pahanan Agta", + "apg": "Ampanang", + "aph": "Athpariya", + "api": "Apiaká", + "apj": "Jicarilla Apache", + "apk": "Kiowa Apache", + "apl": "Lipan Apache", + "apm": "Mescalero-Chiricahua Apache", + "apn": "Apinayé", + "apo": "Ambul", + "app": "Apma", + "apq": "A-Pucikwar", + "apr": "Arop-Lokep", + "aps": "Arop-Sissano", + "apt": "Apatani", + "apu": "Apurinã", + "apv": "Alapmunte", + "apw": "Western Apache", + "apx": "Aputai", + "apy": "Apalaí", + "apz": "Safeyoka", + "aqa": "Alacalufan languages", + "aqc": "Archi", + "aqd": "Ampari Dogon", + "aqg": "Arigidi", + "aqk": "Aninka", + "aql": "Algic languages", + "aqm": "Atohwaim", + "aqn": "Northern Alta", + "aqp": "Atakapa", + "aqr": "Arhâ", + "aqt": "Angaité", + "aqz": "Akuntsu", + "ar": "Arabic", + "arb": "Standard Arabic", + "arc": "Official Aramaic (700-300 BCE); Imperial Aramaic (700-300 BCE)", + "ard": "Arabana", + "are": "Western Arrarnta", + "arh": "Arhuaco", + "ari": "Arikara", + "arj": "Arapaso", + "ark": "Arikapú", + "arl": "Arabela", + "arn": "Mapudungun; Mapuche", + "aro": "Araona", + "arp": "Arapaho", + "arq": "Algerian Arabic", + "arr": "Karo (Brazil)", + "ars": "Najdi Arabic", + "art": "Artificial languages", + "aru": "Aruá (Amazonas State); Arawá", + "arv": "Arbore", + "arw": "Arawak", + "arx": "Aruá (Rodonia State)", + "ary": "Moroccan Arabic", + "arz": "Egyptian Arabic", + "as": "Assamese", + "asa": "Asu (Tanzania)", + "asb": "Assiniboine", + "asc": "Casuarina Coast Asmat", + "ase": "American Sign Language", + "asf": "Auslan; Australian Sign Language", + "asg": "Cishingini", + "ash": "Abishira", + "asi": "Buruwai", + "asj": "Sari", + "ask": "Ashkun", + "asl": "Asilulu", + "asn": "Xingú Asuriní", + "aso": "Dano", + "asp": "Algerian Sign Language", + "asq": "Austrian Sign Language", + "asr": "Asuri", + "ass": "Ipulo", + "ast": "Asturian; Asturleonese; Bable; Leonese", + "asu": "Tocantins Asurini", + "asv": "Asoa", + "asw": "Australian Aborigines Sign Language", + "asx": "Muratayak", + "asy": "Yaosakor Asmat", + "asz": "As", + "ata": "Pele-Ata", + "atb": "Zaiwa", + "atc": "Atsahuaca", + "atd": "Ata Manobo", + "ate": "Atemble", + "atg": "Ivbie North-Okpela-Arhe", + "ath": "Athapascan languages", + "ati": "Attié", + "atj": "Atikamekw", + "atk": "Ati", + "atl": "Mt. Iraya Agta", + "atm": "Ata", + "atn": "Ashtiani", + "ato": "Atong (Cameroon)", + "atp": "Pudtol Atta", + "atq": "Aralle-Tabulahan", + "atr": "Waimiri-Atroari", + "ats": "Gros Ventre", + "att": "Pamplona Atta", + "atu": "Reel", + "atv": "Northern Altai", + "atw": "Atsugewi", + "atx": "Arutani", + "aty": "Aneityum", + "atz": "Arta", + "aua": "Asumboa", + "aub": "Alugu", + "auc": "Waorani", + "aud": "Anuta", + "auf": "Arauan languages", + "aug": "Aguna", + "auh": "Aushi", + "aui": "Anuki", + "auj": "Awjilah", + "auk": "Heyo", + "aul": "Aulua", + "aum": "Asu (Nigeria)", + "aun": "Molmo One", + "auo": "Auyokawa", + "aup": "Makayam", + "auq": "Anus; Korur", + "aur": "Aruek", + "aus": "Australian languages", + "aut": "Austral", + "auu": "Auye", + "auw": "Awyi", + "aux": "Aurá", + "auy": "Awiyaana", + "auz": "Uzbeki Arabic", + "av": "Avaric", + "avb": "Avau", + "avd": "Alviri-Vidari", + "avi": "Avikam", + "avk": "Kotava", + "avl": "Eastern Egyptian Bedawi Arabic", + "avm": "Angkamuthi", + "avn": "Avatime", + "avo": "Agavotaguerra", + "avs": "Aushiri", + "avt": "Au", + "avu": "Avokaya", + "avv": "Avá-Canoeiro", + "awa": "Awadhi", + "awb": "Awa (Papua New Guinea)", + "awc": "Cicipu", + "awd": "Arawakan languages", + "awe": "Awetí", + "awg": "Anguthimri", + "awh": "Awbono", + "awi": "Aekyom", + "awk": "Awabakal", + "awm": "Arawum", + "awn": "Awngi", + "awo": "Awak", + "awr": "Awera", + "aws": "South Awyu", + "awt": "Araweté", + "awu": "Central Awyu", + "awv": "Jair Awyu", + "aww": "Awun", + "awx": "Awara", + "awy": "Edera Awyu", + "axb": "Abipon", + "axe": "Ayerrerenge", + "axg": "Mato Grosso Arára", + "axk": "Yaka (Central African Republic)", + "axl": "Lower Southern Aranda", + "axm": "Middle Armenian", + "axx": "Xârâgurè", + "ay": "Aymara", + "aya": "Awar", + "ayb": "Ayizo Gbe", + "ayc": "Southern Aymara", + "ayd": "Ayabadhu", + "aye": "Ayere", + "ayg": "Ginyanga", + "ayh": "Hadrami Arabic", + "ayi": "Leyigha", + "ayk": "Akuku", + "ayl": "Libyan Arabic", + "ayn": "Sanaani Arabic", + "ayo": "Ayoreo", + "ayp": "North Mesopotamian Arabic", + "ayq": "Ayi (Papua New Guinea)", + "ayr": "Central Aymara", + "ays": "Sorsogon Ayta", + "ayt": "Magbukun Ayta", + "ayu": "Ayu", + "ayz": "Mai Brat", + "az": "Azerbaijani", + "aza": "Azha", + "azb": "South Azerbaijani", + "azc": "Uto-Aztecan languages", + "azd": "Eastern Durango Nahuatl", + "azg": "San Pedro Amuzgos Amuzgo", + "azj": "North Azerbaijani", + "azm": "Ipalapa Amuzgo", + "azn": "Western Durango Nahuatl", + "azo": "Awing", + "azt": "Faire Atta", + "azz": "Highland Puebla Nahuatl", + "ba": "Bashkir", + "baa": "Babatana", + "bab": "Bainouk-Gunyuño", + "bac": "Badui", + "bad": "Banda languages", + "bae": "Baré", + "baf": "Nubaca", + "bag": "Tuki", + "bah": "Bahamas Creole English", + "bai": "Bamileke languages", + "baj": "Barakai", + "bal": "Baluchi", + "ban": "Balinese", + "bao": "Waimaha", + "bap": "Bantawa", + "bar": "Bavarian", + "bas": "Basa (Cameroon)", + "bat": "Baltic languages", + "bau": "Bada (Nigeria)", + "bav": "Vengo", + "baw": "Bambili-Bambui", + "bax": "Bamun", + "bay": "Batuley", + "bba": "Baatonum", + "bbb": "Barai", + "bbc": "Batak Toba", + "bbd": "Bau", + "bbe": "Bangba", + "bbf": "Baibai", + "bbg": "Barama", + "bbh": "Bugan", + "bbi": "Barombi", + "bbj": "Ghomálá'", + "bbk": "Babanki", + "bbl": "Bats", + "bbm": "Babango", + "bbn": "Uneapa", + "bbo": "Northern Bobo Madaré; Konabéré", + "bbp": "West Central Banda", + "bbq": "Bamali", + "bbr": "Girawa", + "bbs": "Bakpinka", + "bbt": "Mburku", + "bbu": "Kulung (Nigeria)", + "bbv": "Karnai", + "bbw": "Baba", + "bbx": "Bubia", + "bby": "Befang", + "bca": "Central Bai", + "bcb": "Bainouk-Samik", + "bcc": "Southern Balochi", + "bcd": "North Babar", + "bce": "Bamenyam", + "bcf": "Bamu", + "bcg": "Baga Pokur", + "bch": "Bariai", + "bci": "Baoulé", + "bcj": "Bardi", + "bck": "Bunuba", + "bcl": "Central Bikol", + "bcm": "Bannoni", + "bcn": "Bali (Nigeria)", + "bco": "Kaluli", + "bcp": "Bali (Democratic Republic of Congo)", + "bcq": "Bench", + "bcr": "Babine", + "bcs": "Kohumono", + "bct": "Bendi", + "bcu": "Awad Bing", + "bcv": "Shoo-Minda-Nye", + "bcw": "Bana", + "bcy": "Bacama", + "bcz": "Bainouk-Gunyaamolo", + "bda": "Bayot", + "bdb": "Basap", + "bdc": "Emberá-Baudó", + "bdd": "Bunama", + "bde": "Bade", + "bdf": "Biage", + "bdg": "Bonggi", + "bdh": "Baka (South Sudan)", + "bdi": "Burun", + "bdj": "Bai (South Sudan); Bai", + "bdk": "Budukh", + "bdl": "Indonesian Bajau", + "bdm": "Buduma", + "bdn": "Baldemu", + "bdo": "Morom", + "bdp": "Bende", + "bdq": "Bahnar", + "bdr": "West Coast Bajau", + "bds": "Burunge", + "bdt": "Bokoto", + "bdu": "Oroko", + "bdv": "Bodo Parja", + "bdw": "Baham", + "bdx": "Budong-Budong", + "bdy": "Bandjalang", + "bdz": "Badeshi", + "be": "Belarusian", + "bea": "Beaver", + "beb": "Bebele", + "bec": "Iceve-Maci", + "bed": "Bedoanas", + "bee": "Byangsi", + "bef": "Benabena", + "beg": "Belait", + "beh": "Biali", + "bei": "Bekati'", + "bej": "Beja; Bedawiyet", + "bek": "Bebeli", + "bem": "Bemba (Zambia)", + "beo": "Beami", + "bep": "Besoa", + "beq": "Beembe", + "ber": "Berber languages", + "bes": "Besme", + "bet": "Guiberoua Béte", + "beu": "Blagar", + "bev": "Daloa Bété", + "bew": "Betawi", + "bex": "Jur Modo", + "bey": "Beli (Papua New Guinea)", + "bez": "Bena (Tanzania)", + "bfa": "Bari", + "bfb": "Pauri Bareli", + "bfc": "Panyi Bai; Northern Bai", + "bfd": "Bafut", + "bfe": "Betaf; Tena", + "bff": "Bofi", + "bfg": "Busang Kayan", + "bfh": "Blafe", + "bfi": "British Sign Language", + "bfj": "Bafanji", + "bfk": "Ban Khor Sign Language", + "bfl": "Banda-Ndélé", + "bfm": "Mmen", + "bfn": "Bunak", + "bfo": "Malba Birifor", + "bfp": "Beba", + "bfq": "Badaga", + "bfr": "Bazigar", + "bfs": "Southern Bai", + "bft": "Balti", + "bfu": "Gahri", + "bfw": "Bondo", + "bfx": "Bantayanon", + "bfy": "Bagheli", + "bfz": "Mahasu Pahari", + "bg": "Bulgarian", + "bga": "Gwamhi-Wuri", + "bgb": "Bobongko", + "bgc": "Haryanvi", + "bgd": "Rathwi Bareli", + "bge": "Bauria", + "bgf": "Bangandu", + "bgg": "Bugun", + "bgi": "Giangan", + "bgj": "Bangolan", + "bgk": "Bit; Buxinhua", + "bgl": "Bo (Laos)", + "bgn": "Western Balochi", + "bgo": "Baga Koga", + "bgp": "Eastern Balochi", + "bgq": "Bagri", + "bgr": "Bawm Chin", + "bgs": "Tagabawa", + "bgt": "Bughotu", + "bgu": "Mbongno", + "bgv": "Warkay-Bipim", + "bgw": "Bhatri", + "bgx": "Balkan Gagauz Turkish", + "bgy": "Benggoi", + "bgz": "Banggai", + "bh": "Bihari languages", + "bha": "Bharia", + "bhb": "Bhili", + "bhc": "Biga", + "bhd": "Bhadrawahi", + "bhe": "Bhaya", + "bhf": "Odiai", + "bhg": "Binandere", + "bhh": "Bukharic", + "bhi": "Bhilali", + "bhj": "Bahing", + "bhl": "Bimin", + "bhm": "Bathari", + "bhn": "Bohtan Neo-Aramaic", + "bho": "Bhojpuri", + "bhp": "Bima", + "bhq": "Tukang Besi South", + "bhr": "Bara Malagasy", + "bhs": "Buwal", + "bht": "Bhattiyali", + "bhu": "Bhunjia", + "bhv": "Bahau", + "bhw": "Biak", + "bhx": "Bhalay", + "bhy": "Bhele", + "bhz": "Bada (Indonesia)", + "bi": "Bislama", + "bia": "Badimaya", + "bib": "Bissa; Bisa", + "bid": "Bidiyo", + "bie": "Bepour", + "bif": "Biafada", + "big": "Biangai", + "bik": "Bikol", + "bil": "Bile", + "bim": "Bimoba", + "bin": "Bini; Edo", + "bio": "Nai", + "bip": "Bila", + "biq": "Bipi", + "bir": "Bisorio", + "bit": "Berinomo", + "biu": "Biete", + "biv": "Southern Birifor", + "biw": "Kol (Cameroon)", + "bix": "Bijori", + "biy": "Birhor", + "biz": "Baloi", + "bja": "Budza", + "bjb": "Banggarla", + "bjc": "Bariji", + "bje": "Biao-Jiao Mien", + "bjf": "Barzani Jewish Neo-Aramaic", + "bjg": "Bidyogo", + "bjh": "Bahinemo", + "bji": "Burji", + "bjj": "Kanauji", + "bjk": "Barok", + "bjl": "Bulu (Papua New Guinea)", + "bjm": "Bajelani", + "bjn": "Banjar", + "bjo": "Mid-Southern Banda", + "bjp": "Fanamaket", + "bjr": "Binumarien", + "bjs": "Bajan", + "bjt": "Balanta-Ganja", + "bju": "Busuu", + "bjv": "Bedjond", + "bjw": "Bakwé", + "bjx": "Banao Itneg", + "bjy": "Bayali", + "bjz": "Baruga", + "bka": "Kyak", + "bkc": "Baka (Cameroon)", + "bkd": "Binukid; Talaandig", + "bkf": "Beeke", + "bkg": "Buraka", + "bkh": "Bakoko", + "bki": "Baki", + "bkj": "Pande", + "bkk": "Brokskat", + "bkl": "Berik", + "bkm": "Kom (Cameroon)", + "bkn": "Bukitan", + "bko": "Kwa'", + "bkp": "Boko (Democratic Republic of Congo)", + "bkq": "Bakairí", + "bkr": "Bakumpai", + "bks": "Northern Sorsoganon", + "bkt": "Boloki", + "bku": "Buhid", + "bkv": "Bekwarra", + "bkw": "Bekwel", + "bkx": "Baikeno", + "bky": "Bokyi", + "bkz": "Bungku", + "bla": "Siksika", + "blb": "Bilua", + "blc": "Bella Coola", + "bld": "Bolango", + "ble": "Balanta-Kentohe", + "blf": "Buol", + "blh": "Kuwaa", + "bli": "Bolia", + "blj": "Bolongan", + "blk": "Pa'o Karen; Pa'O", + "bll": "Biloxi", + "blm": "Beli (South Sudan)", + "bln": "Southern Catanduanes Bikol", + "blo": "Anii", + "blp": "Blablanga", + "blq": "Baluan-Pam", + "blr": "Blang", + "bls": "Balaesang", + "blt": "Tai Dam", + "blv": "Kibala; Bolo", + "blw": "Balangao", + "blx": "Mag-Indi Ayta", + "bly": "Notre", + "blz": "Balantak", + "bm": "Bambara", + "bma": "Lame", + "bmb": "Bembe", + "bmc": "Biem", + "bmd": "Baga Manduri", + "bme": "Limassa", + "bmf": "Bom-Kim", + "bmg": "Bamwe", + "bmh": "Kein", + "bmi": "Bagirmi", + "bmj": "Bote-Majhi", + "bmk": "Ghayavi", + "bml": "Bomboli", + "bmm": "Northern Betsimisaraka Malagasy", + "bmn": "Bina (Papua New Guinea)", + "bmo": "Bambalang", + "bmp": "Bulgebi", + "bmq": "Bomu", + "bmr": "Muinane", + "bms": "Bilma Kanuri", + "bmt": "Biao Mon", + "bmu": "Somba-Siawari", + "bmv": "Bum", + "bmw": "Bomwali", + "bmx": "Baimak", + "bmz": "Baramu", + "bn": "Bengali; Bangla", + "bna": "Bonerate", + "bnb": "Bookan", + "bnc": "Bontok", + "bnd": "Banda (Indonesia)", + "bne": "Bintauna", + "bnf": "Masiwang", + "bng": "Benga", + "bni": "Bangi", + "bnj": "Eastern Tawbuid", + "bnk": "Bierebo", + "bnl": "Boon", + "bnm": "Batanga", + "bnn": "Bunun", + "bno": "Bantoanon", + "bnp": "Bola", + "bnq": "Bantik", + "bnr": "Butmas-Tur", + "bns": "Bundeli", + "bnt": "Bantu languages", + "bnu": "Bentong", + "bnv": "Bonerif; Beneraf; Edwas", + "bnw": "Bisis", + "bnx": "Bangubangu", + "bny": "Bintulu", + "bnz": "Beezen", + "bo": "Tibetan", + "boa": "Bora", + "bob": "Aweer", + "boe": "Mundabli", + "bof": "Bolon", + "bog": "Bamako Sign Language", + "boh": "Boma", + "boi": "Barbareño", + "boj": "Anjam", + "bok": "Bonjo", + "bol": "Bole", + "bom": "Berom", + "bon": "Bine", + "boo": "Tiemacèwè Bozo", + "bop": "Bonkiman", + "boq": "Bogaya", + "bor": "Borôro", + "bot": "Bongo", + "bou": "Bondei", + "bov": "Tuwuli", + "bow": "Rema", + "box": "Buamu", + "boy": "Bodo (Central African Republic)", + "boz": "Tiéyaxo Bozo", + "bpa": "Daakaka", + "bpc": "Mbuk", + "bpd": "Banda-Banda", + "bpe": "Bauni", + "bpg": "Bonggo", + "bph": "Botlikh", + "bpi": "Bagupi", + "bpj": "Binji", + "bpk": "Orowe; 'Ôrôê", + "bpl": "Broome Pearling Lugger Pidgin", + "bpm": "Biyom", + "bpn": "Dzao Min", + "bpo": "Anasi", + "bpp": "Kaure", + "bpq": "Banda Malay", + "bpr": "Koronadal Blaan", + "bps": "Sarangani Blaan", + "bpt": "Barrow Point", + "bpu": "Bongu", + "bpv": "Bian Marind", + "bpw": "Bo (Papua New Guinea)", + "bpx": "Palya Bareli", + "bpy": "Bishnupriya", + "bpz": "Bilba", + "bqa": "Tchumbuli", + "bqb": "Bagusa", + "bqc": "Boko (Benin); Boo", + "bqd": "Bung", + "bqf": "Baga Kaloum", + "bqg": "Bago-Kusuntu", + "bqh": "Baima", + "bqi": "Bakhtiari", + "bqj": "Bandial", + "bqk": "Banda-Mbrès", + "bql": "Bilakura", + "bqm": "Wumboko", + "bqn": "Bulgarian Sign Language", + "bqo": "Balo", + "bqp": "Busa", + "bqq": "Biritai", + "bqr": "Burusu", + "bqs": "Bosngun", + "bqt": "Bamukumbit", + "bqu": "Boguru", + "bqv": "Koro Wachi; Begbere-Ejar", + "bqw": "Buru (Nigeria)", + "bqx": "Baangi", + "bqy": "Bengkala Sign Language", + "bqz": "Bakaka", + "br": "Breton", + "bra": "Braj", + "brb": "Brao; Lave", + "brc": "Berbice Creole Dutch", + "brd": "Baraamu", + "brf": "Bira", + "brg": "Baure", + "brh": "Brahui", + "bri": "Mokpwe", + "brj": "Bieria", + "brk": "Birked", + "brl": "Birwa", + "brm": "Barambu", + "brn": "Boruca", + "bro": "Brokkat", + "brp": "Barapasi", + "brq": "Breri", + "brr": "Birao", + "brs": "Baras", + "brt": "Bitare", + "bru": "Eastern Bru", + "brv": "Western Bru", + "brw": "Bellari", + "brx": "Bodo (India)", + "bry": "Burui", + "brz": "Bilbil", + "bs": "Bosnian", + "bsa": "Abinomn", + "bsb": "Brunei Bisaya", + "bsc": "Bassari; Oniyan", + "bse": "Wushi", + "bsf": "Bauchi", + "bsg": "Bashkardi", + "bsh": "Kati", + "bsi": "Bassossi", + "bsj": "Bangwinji", + "bsk": "Burushaski", + "bsl": "Basa-Gumna", + "bsm": "Busami", + "bsn": "Barasana-Eduria", + "bso": "Buso", + "bsp": "Baga Sitemu", + "bsq": "Bassa", + "bsr": "Bassa-Kontagora", + "bss": "Akoose", + "bst": "Basketo", + "bsu": "Bahonsuai", + "bsv": "Baga Sobané", + "bsw": "Baiso", + "bsx": "Yangkam", + "bsy": "Sabah Bisaya", + "bta": "Bata", + "btc": "Bati (Cameroon)", + "btd": "Batak Dairi", + "bte": "Gamo-Ningi", + "btf": "Birgit", + "btg": "Gagnoa Bété", + "bth": "Biatah Bidayuh", + "bti": "Burate", + "btj": "Bacanese Malay", + "btk": "Batak languages", + "btm": "Batak Mandailing", + "btn": "Ratagnon", + "bto": "Rinconada Bikol", + "btp": "Budibud", + "btq": "Batek", + "btr": "Baetora", + "bts": "Batak Simalungun", + "btt": "Bete-Bendi", + "btu": "Batu", + "btv": "Bateri", + "btw": "Butuanon", + "btx": "Batak Karo", + "bty": "Bobot", + "btz": "Batak Alas-Kluet", + "bua": "Buriat", + "bub": "Bua", + "buc": "Bushi", + "bud": "Ntcham", + "bue": "Beothuk", + "buf": "Bushoong", + "bug": "Buginese", + "buh": "Younuo Bunu", + "bui": "Bongili", + "buj": "Basa-Gurmana", + "buk": "Bugawac", + "bum": "Bulu (Cameroon)", + "bun": "Sherbro", + "buo": "Terei", + "bup": "Busoa", + "buq": "Brem", + "bus": "Bokobaru", + "but": "Bungain", + "buu": "Budu", + "buv": "Bun", + "buw": "Bubi", + "bux": "Boghom", + "buy": "Bullom So", + "buz": "Bukwen", + "bva": "Barein", + "bvb": "Bube", + "bvc": "Baelelea", + "bvd": "Baeggu", + "bve": "Berau Malay", + "bvf": "Boor", + "bvg": "Bonkeng", + "bvh": "Bure", + "bvi": "Belanda Viri", + "bvj": "Baan", + "bvk": "Bukat", + "bvl": "Bolivian Sign Language", + "bvm": "Bamunka", + "bvn": "Buna", + "bvo": "Bolgo", + "bvp": "Bumang", + "bvq": "Birri", + "bvr": "Burarra", + "bvt": "Bati (Indonesia)", + "bvu": "Bukit Malay", + "bvv": "Baniva", + "bvw": "Boga", + "bvx": "Dibole", + "bvy": "Baybayanon", + "bvz": "Bauzi", + "bwa": "Bwatoo", + "bwb": "Namosi-Naitasiri-Serua", + "bwc": "Bwile", + "bwd": "Bwaidoka", + "bwe": "Bwe Karen", + "bwf": "Boselewa", + "bwg": "Barwe", + "bwh": "Bishuo", + "bwi": "Baniwa", + "bwj": "Láá Láá Bwamu", + "bwk": "Bauwaki", + "bwl": "Bwela", + "bwm": "Biwat", + "bwn": "Wunai Bunu", + "bwo": "Boro (Ethiopia); Borna (Ethiopia)", + "bwp": "Mandobo Bawah", + "bwq": "Southern Bobo Madaré", + "bwr": "Bura-Pabir", + "bws": "Bomboma", + "bwt": "Bafaw-Balong", + "bwu": "Buli (Ghana)", + "bww": "Bwa", + "bwx": "Bu-Nao Bunu", + "bwy": "Cwi Bwamu", + "bwz": "Bwisi", + "bxa": "Tairaha", + "bxb": "Belanda Bor", + "bxc": "Molengue", + "bxd": "Pela", + "bxe": "Birale", + "bxf": "Bilur; Minigir", + "bxg": "Bangala", + "bxh": "Buhutu", + "bxi": "Pirlatapa", + "bxj": "Bayungu", + "bxk": "Bukusu; Lubukusu", + "bxl": "Jalkunan", + "bxm": "Mongolia Buriat", + "bxn": "Burduna", + "bxo": "Barikanchi", + "bxp": "Bebil", + "bxq": "Beele", + "bxr": "Russia Buriat", + "bxs": "Busam", + "bxu": "China Buriat", + "bxv": "Berakou", + "bxw": "Bankagooma", + "bxz": "Binahari", + "bya": "Batak", + "byb": "Bikya", + "byc": "Ubaghara", + "byd": "Benyadu'", + "bye": "Pouye", + "byf": "Bete", + "byg": "Baygo", + "byh": "Bhujel", + "byi": "Buyu", + "byj": "Bina (Nigeria)", + "byk": "Biao", + "byl": "Bayono", + "bym": "Bidjara", + "byn": "Bilin; Blin", + "byo": "Biyo", + "byp": "Bumaji", + "byq": "Basay", + "byr": "Baruya; Yipma", + "bys": "Burak", + "byt": "Berti", + "byv": "Medumba", + "byw": "Belhariya", + "byx": "Qaqet", + "byz": "Banaro", + "bza": "Bandi", + "bzb": "Andio", + "bzc": "Southern Betsimisaraka Malagasy", + "bzd": "Bribri", + "bze": "Jenaama Bozo", + "bzf": "Boikin", + "bzg": "Babuza", + "bzh": "Mapos Buang", + "bzi": "Bisu", + "bzj": "Belize Kriol English", + "bzk": "Nicaragua Creole English", + "bzl": "Boano (Sulawesi)", + "bzm": "Bolondo", + "bzn": "Boano (Maluku)", + "bzo": "Bozaba", + "bzp": "Kemberano", + "bzq": "Buli (Indonesia)", + "bzr": "Biri", + "bzs": "Brazilian Sign Language", + "bzt": "Brithenig", + "bzu": "Burmeso", + "bzv": "Naami", + "bzw": "Basa (Nigeria)", + "bzx": "Kɛlɛngaxo Bozo", + "bzy": "Obanliku", + "bzz": "Evant", + "ca": "Catalan; Valencian", + "caa": "Chortí", + "cab": "Garifuna", + "cac": "Chuj", + "cad": "Caddo", + "cae": "Lehar; Laalaa", + "caf": "Southern Carrier", + "cag": "Nivaclé", + "cah": "Cahuarano", + "cai": "Central American Indian languages", + "caj": "Chané", + "cak": "Kaqchikel; Cakchiquel", + "cal": "Carolinian", + "cam": "Cemuhî", + "can": "Chambri", + "cao": "Chácobo", + "cap": "Chipaya", + "caq": "Car Nicobarese", + "car": "Galibi Carib", + "cas": "Tsimané", + "cau": "Caucasian languages", + "cav": "Cavineña", + "caw": "Callawalla", + "cax": "Chiquitano", + "cay": "Cayuga", + "caz": "Canichana", + "cba": "Chibchan languages", + "cbb": "Cabiyarí", + "cbc": "Carapana", + "cbd": "Carijona", + "cbg": "Chimila", + "cbi": "Chachi", + "cbj": "Ede Cabe", + "cbk": "Chavacano", + "cbl": "Bualkhaw Chin", + "cbn": "Nyahkur", + "cbo": "Izora", + "cbq": "Tsucuba; Cuba", + "cbr": "Cashibo-Cacataibo", + "cbs": "Cashinahua", + "cbt": "Chayahuita", + "cbu": "Candoshi-Shapra", + "cbv": "Cacua", + "cbw": "Kinabalian", + "cby": "Carabayo", + "ccc": "Chamicuro", + "ccd": "Cafundo Creole", + "cce": "Chopi", + "ccg": "Samba Daka", + "cch": "Atsam", + "ccj": "Kasanga", + "ccl": "Cutchi-Swahili", + "ccm": "Malaccan Creole Malay", + "ccn": "North Caucasian languages", + "cco": "Comaltepec Chinantec", + "ccp": "Chakma", + "ccr": "Cacaopera", + "ccs": "South Caucasian languages", + "cda": "Choni", + "cdc": "Chadic languages", + "cdd": "Caddoan languages", + "cde": "Chenchu", + "cdf": "Chiru", + "cdh": "Chambeali", + "cdi": "Chodri", + "cdj": "Churahi", + "cdm": "Chepang", + "cdn": "Chaudangsi", + "cdo": "Min Dong Chinese", + "cdr": "Cinda-Regi-Tiyal", + "cds": "Chadian Sign Language", + "cdy": "Chadong", + "cdz": "Koda", + "ce": "Chechen", + "cea": "Lower Chehalis", + "ceb": "Cebuano", + "ceg": "Chamacoco", + "cek": "Eastern Khumi Chin", + "cel": "Celtic languages", + "cen": "Cen", + "cet": "Centúúm", + "cey": "Ekai Chin", + "cfa": "Dijim-Bwilim", + "cfd": "Cara", + "cfg": "Como Karim", + "cfm": "Falam Chin", + "cga": "Changriwa", + "cgc": "Kagayanen", + "cgg": "Chiga", + "cgk": "Chocangacakha", + "ch": "Chamorro", + "chb": "Chibcha", + "chc": "Catawba", + "chd": "Highland Oaxaca Chontal", + "chf": "Tabasco Chontal", + "chg": "Chagatai", + "chh": "Chinook", + "chj": "Ojitlán Chinantec", + "chk": "Chuukese", + "chl": "Cahuilla", + "chm": "Mari (Russia)", + "chn": "Chinook jargon", + "cho": "Choctaw", + "chp": "Chipewyan; Dene Suline", + "chq": "Quiotepec Chinantec", + "chr": "Cherokee", + "cht": "Cholón", + "chw": "Chuwabu", + "chx": "Chantyal", + "chy": "Cheyenne", + "chz": "Ozumacín Chinantec", + "cia": "Cia-Cia", + "cib": "Ci Gbe", + "cic": "Chickasaw", + "cid": "Chimariko", + "cie": "Cineni", + "cih": "Chinali", + "cik": "Chitkuli Kinnauri", + "cim": "Cimbrian", + "cin": "Cinta Larga", + "cip": "Chiapanec", + "cir": "Tiri; Haméa; Méa", + "ciw": "Chippewa", + "ciy": "Chaima", + "cja": "Western Cham", + "cje": "Chru", + "cjh": "Upper Chehalis", + "cji": "Chamalal", + "cjk": "Chokwe", + "cjm": "Eastern Cham", + "cjn": "Chenapian", + "cjo": "Ashéninka Pajonal", + "cjp": "Cabécar", + "cjs": "Shor", + "cjv": "Chuave", + "cjy": "Jinyu Chinese", + "ckb": "Central Kurdish", + "ckh": "Chak", + "ckl": "Cibak", + "ckm": "Chakavian", + "ckn": "Kaang Chin", + "cko": "Anufo", + "ckq": "Kajakse", + "ckr": "Kairak", + "cks": "Tayo", + "ckt": "Chukot", + "cku": "Koasati", + "ckv": "Kavalan", + "ckx": "Caka", + "cky": "Cakfem-Mushere", + "ckz": "Cakchiquel-Quiché Mixed Language", + "cla": "Ron", + "clc": "Chilcotin", + "cld": "Chaldean Neo-Aramaic", + "cle": "Lealao Chinantec", + "clh": "Chilisso", + "cli": "Chakali", + "clj": "Laitu Chin", + "clk": "Idu-Mishmi", + "cll": "Chala", + "clm": "Clallam", + "clo": "Lowland Oaxaca Chontal", + "clt": "Lautu Chin", + "clu": "Caluyanun", + "clw": "Chulym", + "cly": "Eastern Highland Chatino", + "cma": "Maa", + "cmc": "Chamic languages", + "cme": "Cerma", + "cmg": "Classical Mongolian", + "cmi": "Emberá-Chamí", + "cml": "Campalagian", + "cmm": "Michigamea", + "cmn": "Mandarin Chinese", + "cmo": "Central Mnong", + "cmr": "Mro-Khimi Chin", + "cms": "Messapic", + "cmt": "Camtho", + "cna": "Changthang", + "cnb": "Chinbon Chin", + "cnc": "Côông", + "cng": "Northern Qiang", + "cnh": "Hakha Chin; Haka Chin", + "cni": "Asháninka", + "cnk": "Khumi Chin", + "cnl": "Lalana Chinantec", + "cno": "Con", + "cnp": "Northern Ping Chinese; Northern Pinghua", + "cnq": "Chung", + "cnr": "Montenegrin", + "cns": "Central Asmat", + "cnt": "Tepetotutla Chinantec", + "cnu": "Chenoua", + "cnw": "Ngawn Chin", + "cnx": "Middle Cornish", + "co": "Corsican", + "coa": "Cocos Islands Malay", + "cob": "Chicomuceltec", + "coc": "Cocopa", + "cod": "Cocama-Cocamilla", + "coe": "Koreguaje", + "cof": "Colorado", + "cog": "Chong", + "coh": "Chonyi-Dzihana-Kauma; Chichonyi-Chidzihana-Chikauma", + "coj": "Cochimi", + "cok": "Santa Teresa Cora", + "col": "Columbia-Wenatchi", + "com": "Comanche", + "con": "Cofán", + "coo": "Comox", + "cop": "Coptic", + "coq": "Coquille", + "cot": "Caquinte", + "cou": "Wamey", + "cov": "Cao Miao", + "cow": "Cowlitz", + "cox": "Nanti", + "coz": "Chochotec", + "cpa": "Palantla Chinantec", + "cpb": "Ucayali-Yurúa Ashéninka", + "cpc": "Ajyíninka Apurucayali", + "cpe": "English-based creoles and pidgins", + "cpf": "French-based creoles and pidgins", + "cpg": "Cappadocian Greek", + "cpi": "Chinese Pidgin English", + "cpn": "Cherepon", + "cpo": "Kpeego", + "cpp": "Portuguese-based creoles and pidgins", + "cps": "Capiznon", + "cpu": "Pichis Ashéninka", + "cpx": "Pu-Xian Chinese", + "cpy": "South Ucayali Ashéninka", + "cqd": "Chuanqiandian Cluster Miao", + "cr": "Cree", + "cra": "Chara", + "crb": "Island Carib", + "crc": "Lonwolwol", + "crd": "Coeur d'Alene", + "crf": "Caramanta", + "crg": "Michif", + "crh": "Crimean Tatar; Crimean Turkish", + "cri": "Sãotomense", + "crj": "Southern East Cree", + "crk": "Plains Cree", + "crl": "Northern East Cree", + "crm": "Moose Cree", + "crn": "El Nayar Cora", + "cro": "Crow", + "crp": "Creoles and pidgins", + "crq": "Iyo'wujwa Chorote", + "crr": "Carolina Algonquian", + "crs": "Seselwa Creole French", + "crt": "Iyojwa'ja Chorote", + "crv": "Chaura", + "crw": "Chrau", + "crx": "Carrier", + "cry": "Cori", + "crz": "Cruzeño", + "cs": "Czech", + "csa": "Chiltepec Chinantec", + "csb": "Kashubian", + "csc": "Catalan Sign Language; Lengua de señas catalana; Llengua de Signes Catalana", + "csd": "Chiangmai Sign Language", + "cse": "Czech Sign Language", + "csf": "Cuba Sign Language", + "csg": "Chilean Sign Language", + "csh": "Asho Chin", + "csi": "Coast Miwok", + "csj": "Songlai Chin", + "csk": "Jola-Kasa", + "csl": "Chinese Sign Language", + "csm": "Central Sierra Miwok", + "csn": "Colombian Sign Language", + "cso": "Sochiapam Chinantec; Sochiapan Chinantec", + "csp": "Southern Ping Chinese; Southern Pinghua", + "csq": "Croatia Sign Language", + "csr": "Costa Rican Sign Language", + "css": "Southern Ohlone", + "cst": "Northern Ohlone", + "csu": "Central Sudanic languages", + "csv": "Sumtu Chin", + "csw": "Swampy Cree", + "csx": "Cambodian Sign Language", + "csy": "Siyin Chin", + "csz": "Coos", + "cta": "Tataltepec Chatino", + "ctc": "Chetco", + "ctd": "Tedim Chin", + "cte": "Tepinapa Chinantec", + "ctg": "Chittagonian", + "cth": "Thaiphum Chin", + "ctl": "Tlacoatzintepec Chinantec", + "ctm": "Chitimacha", + "ctn": "Chhintange", + "cto": "Emberá-Catío", + "ctp": "Western Highland Chatino", + "cts": "Northern Catanduanes Bikol", + "ctt": "Wayanad Chetti", + "ctu": "Chol", + "cty": "Moundadan Chetty", + "ctz": "Zacatepec Chatino", + "cu": "Church Slavic; Church Slavonic; Old Bulgarian; Old Church Slavonic; Old Slavonic", + "cua": "Cua", + "cub": "Cubeo", + "cuc": "Usila Chinantec", + "cuh": "Chuka; Gichuka", + "cui": "Cuiba", + "cuj": "Mashco Piro", + "cuk": "San Blas Kuna", + "cul": "Culina; Kulina", + "cuo": "Cumanagoto", + "cup": "Cupeño", + "cuq": "Cun", + "cur": "Chhulung", + "cus": "Cushitic languages", + "cut": "Teutila Cuicatec", + "cuu": "Tai Ya", + "cuv": "Cuvok", + "cuw": "Chukwa", + "cux": "Tepeuxila Cuicatec", + "cuy": "Cuitlatec", + "cv": "Chuvash", + "cvg": "Chug", + "cvn": "Valle Nacional Chinantec", + "cwa": "Kabwa", + "cwb": "Maindo", + "cwd": "Woods Cree", + "cwe": "Kwere", + "cwg": "Chewong; Cheq Wong", + "cwt": "Kuwaataay", + "cy": "Welsh", + "cya": "Nopala Chatino", + "cyb": "Cayubaba", + "cyo": "Cuyonon", + "czh": "Huizhou Chinese", + "czk": "Knaanic", + "czn": "Zenzontepec Chatino", + "czo": "Min Zhong Chinese", + "czt": "Zotung Chin", + "da": "Danish", + "daa": "Dangaléat", + "dac": "Dambi", + "dad": "Marik", + "dae": "Duupa", + "dag": "Dagbani", + "dah": "Gwahatike", + "dai": "Day", + "daj": "Dar Fur Daju", + "dak": "Dakota", + "dal": "Dahalo", + "dam": "Damakawa", + "dao": "Daai Chin", + "daq": "Dandami Maria", + "dar": "Dargwa", + "das": "Daho-Doo", + "dau": "Dar Sila Daju", + "dav": "Taita; Dawida", + "daw": "Davawenyo", + "dax": "Dayi", + "day": "Land Dayak languages", + "daz": "Dao", + "dba": "Bangime", + "dbb": "Deno", + "dbd": "Dadiya", + "dbe": "Dabe", + "dbf": "Edopi", + "dbg": "Dogul Dom Dogon", + "dbi": "Doka", + "dbj": "Ida'an", + "dbl": "Dyirbal", + "dbm": "Duguri", + "dbn": "Duriankere", + "dbo": "Dulbu", + "dbp": "Duwai", + "dbq": "Daba", + "dbr": "Dabarre", + "dbt": "Ben Tey Dogon", + "dbu": "Bondum Dom Dogon", + "dbv": "Dungu", + "dbw": "Bankan Tey Dogon", + "dby": "Dibiyaso", + "dcc": "Deccan", + "dcr": "Negerhollands", + "dda": "Dadi Dadi", + "ddd": "Dongotono", + "dde": "Doondo", + "ddg": "Fataluku", + "ddi": "West Goodenough", + "ddj": "Jaru", + "ddn": "Dendi (Benin)", + "ddo": "Dido", + "ddr": "Dhudhuroa", + "dds": "Donno So Dogon", + "ddw": "Dawera-Daweloor", + "de": "German", + "dec": "Dagik", + "ded": "Dedua", + "dee": "Dewoin", + "def": "Dezfuli", + "deg": "Degema", + "deh": "Dehwari", + "dei": "Demisa", + "dek": "Dek", + "del": "Delaware", + "dem": "Dem", + "den": "Slave (Athapascan)", + "dep": "Pidgin Delaware", + "deq": "Dendi (Central African Republic)", + "der": "Deori", + "des": "Desano", + "dev": "Domung", + "dez": "Dengese", + "dga": "Southern Dagaare", + "dgb": "Bunoge Dogon", + "dgc": "Casiguran Dumagat Agta", + "dgd": "Dagaari Dioula", + "dge": "Degenan", + "dgg": "Doga", + "dgh": "Dghwede", + "dgi": "Northern Dagara", + "dgk": "Dagba", + "dgl": "Andaandi; Dongolawi", + "dgn": "Dagoman", + "dgo": "Dogri (individual language)", + "dgr": "Dogrib; Tłı̨chǫ", + "dgs": "Dogoso", + "dgt": "Ndra'ngith", + "dgw": "Daungwurrung", + "dgx": "Doghoro", + "dgz": "Daga", + "dhd": "Dhundari", + "dhg": "Dhangu-Djangu; Dhangu; Djangu", + "dhi": "Dhimal", + "dhl": "Dhalandji", + "dhm": "Zemba", + "dhn": "Dhanki", + "dho": "Dhodia", + "dhr": "Dhargari", + "dhs": "Dhaiso", + "dhu": "Dhurga", + "dhv": "Dehu; Drehu", + "dhw": "Dhanwar (Nepal)", + "dhx": "Dhungaloo", + "dia": "Dia", + "dib": "South Central Dinka", + "dic": "Lakota Dida", + "did": "Didinga", + "dif": "Dieri; Diyari", + "dig": "Digo; Chidigo", + "dih": "Kumiai", + "dii": "Dimbong", + "dij": "Dai", + "dik": "Southwestern Dinka", + "dil": "Dilling", + "dim": "Dime", + "din": "Dinka", + "dio": "Dibo", + "dip": "Northeastern Dinka", + "diq": "Dimli (individual language)", + "dir": "Dirim", + "dis": "Dimasa", + "diu": "Diriku", + "diw": "Northwestern Dinka", + "dix": "Dixon Reef", + "diy": "Diuwe", + "diz": "Ding", + "dja": "Djadjawurrung", + "djb": "Djinba", + "djc": "Dar Daju Daju", + "djd": "Djamindjung; Ngaliwurru", + "dje": "Zarma", + "djf": "Djangun", + "dji": "Djinang", + "djj": "Djeebbana", + "djk": "Eastern Maroon Creole; Businenge Tongo; Nenge", + "djm": "Jamsay Dogon", + "djn": "Jawoyn; Djauan", + "djo": "Jangkang", + "djr": "Djambarrpuyngu", + "dju": "Kapriman", + "djw": "Djawi", + "dka": "Dakpakha", + "dkg": "Kadung", + "dkk": "Dakka", + "dkr": "Kuijau", + "dks": "Southeastern Dinka", + "dkx": "Mazagway", + "dlg": "Dolgan", + "dlk": "Dahalik", + "dlm": "Dalmatian", + "dln": "Darlong", + "dma": "Duma", + "dmb": "Mombo Dogon", + "dmc": "Gavak", + "dmd": "Madhi Madhi", + "dme": "Dugwor", + "dmf": "Medefaidrin", + "dmg": "Upper Kinabatangan", + "dmk": "Domaaki", + "dml": "Dameli", + "dmm": "Dama", + "dmn": "Mande languages", + "dmo": "Kemedzung", + "dmr": "East Damar", + "dms": "Dampelas", + "dmu": "Dubu; Tebi", + "dmv": "Dumpas", + "dmw": "Mudburra", + "dmx": "Dema", + "dmy": "Demta; Sowari", + "dna": "Upper Grand Valley Dani", + "dnd": "Daonda", + "dne": "Ndendeule", + "dng": "Dungan", + "dni": "Lower Grand Valley Dani", + "dnj": "Dan", + "dnk": "Dengka", + "dnn": "Dzùùngoo", + "dno": "Ndrulo; Northern Lendu", + "dnr": "Danaru", + "dnt": "Mid Grand Valley Dani", + "dnu": "Danau", + "dnv": "Danu", + "dnw": "Western Dani", + "dny": "Dení", + "doa": "Dom", + "dob": "Dobu", + "doc": "Northern Dong", + "doe": "Doe", + "dof": "Domu", + "doh": "Dong", + "doi": "Dogri (macrolanguage)", + "dok": "Dondo", + "dol": "Doso", + "don": "Toura (Papua New Guinea)", + "doo": "Dongo", + "dop": "Lukpa", + "doq": "Dominican Sign Language", + "dor": "Dori'o", + "dos": "Dogosé", + "dot": "Dass", + "dov": "Dombe", + "dow": "Doyayo", + "dox": "Bussa", + "doy": "Dompo", + "doz": "Dorze", + "dpp": "Papar", + "dra": "Dravidian languages", + "drb": "Dair", + "drc": "Minderico", + "drd": "Darmiya", + "dre": "Dolpo", + "drg": "Rungus", + "dri": "C'Lela", + "drl": "Paakantyi", + "drn": "West Damar", + "dro": "Daro-Matu Melanau", + "drq": "Dura", + "drs": "Gedeo", + "drt": "Drents", + "dru": "Rukai", + "dry": "Darai", + "dsb": "Lower Sorbian", + "dse": "Dutch Sign Language", + "dsh": "Daasanach", + "dsi": "Disa", + "dsl": "Danish Sign Language", + "dsn": "Dusner", + "dso": "Desiya", + "dsq": "Tadaksahak", + "dsz": "Mardin Sign Language", + "dta": "Daur", + "dtb": "Labuk-Kinabatangan Kadazan", + "dtd": "Ditidaht", + "dth": "Adithinngithigh", + "dti": "Ana Tinga Dogon", + "dtk": "Tene Kan Dogon", + "dtm": "Tomo Kan Dogon", + "dtn": "Daatsʼíin", + "dto": "Tommo So Dogon", + "dtp": "Kadazan Dusun; Central Dusun", + "dtr": "Lotud", + "dts": "Toro So Dogon", + "dtt": "Toro Tegu Dogon", + "dtu": "Tebul Ure Dogon", + "dty": "Dotyali", + "dua": "Duala", + "dub": "Dubli", + "duc": "Duna", + "due": "Umiray Dumaget Agta", + "duf": "Dumbea; Drubea", + "dug": "Duruma; Chiduruma", + "duh": "Dungra Bhil", + "dui": "Dumun", + "duk": "Uyajitaya", + "dul": "Alabat Island Agta", + "dum": "Middle Dutch (ca. 1050-1350)", + "dun": "Dusun Deyah", + "duo": "Dupaninan Agta", + "dup": "Duano", + "duq": "Dusun Malang", + "dur": "Dii", + "dus": "Dumi", + "duu": "Drung", + "duv": "Duvle", + "duw": "Dusun Witu", + "dux": "Duungooma", + "duy": "Dicamay Agta", + "duz": "Duli-Gey", + "dv": "Dhivehi; Divehi; Maldivian", + "dva": "Duau", + "dwa": "Diri", + "dwk": "Dawik Kui", + "dwr": "Dawro", + "dws": "Dutton World Speedwords", + "dwu": "Dhuwal", + "dww": "Dawawa", + "dwy": "Dhuwaya", + "dwz": "Dewas Rai", + "dya": "Dyan", + "dyb": "Dyaberdyaber", + "dyd": "Dyugun", + "dyg": "Villa Viciosa Agta", + "dyi": "Djimini Senoufo", + "dym": "Yanda Dom Dogon", + "dyn": "Dyangadi; Dhanggatti", + "dyo": "Jola-Fonyi", + "dyu": "Dyula", + "dyy": "Djabugay; Dyaabugay", + "dz": "Dzongkha", + "dza": "Tunzu", + "dze": "Djiwarli", + "dzg": "Dazaga", + "dzl": "Dzalakha", + "dzn": "Dzando", + "eaa": "Karenggapa", + "ebc": "Beginci", + "ebg": "Ebughu", + "ebk": "Eastern Bontok", + "ebo": "Teke-Ebo", + "ebr": "Ebrié", + "ebu": "Embu; Kiembu", + "ecr": "Eteocretan", + "ecs": "Ecuadorian Sign Language", + "ecy": "Eteocypriot", + "ee": "Ewe", + "eee": "E", + "efa": "Efai", + "efe": "Efe", + "efi": "Efik", + "ega": "Ega", + "egl": "Emilian", + "egm": "Benamanga", + "ego": "Eggon", + "egx": "Egyptian languages", + "egy": "Egyptian (Ancient)", + "ehs": "Miyakubo Sign Language", + "ehu": "Ehueun", + "eip": "Eipomek", + "eit": "Eitiep", + "eiv": "Askopan", + "eja": "Ejamat", + "eka": "Ekajuk", + "eke": "Ekit", + "ekg": "Ekari", + "eki": "Eki", + "ekk": "Standard Estonian", + "ekl": "Kol (Bangladesh); Kol", + "ekm": "Elip", + "eko": "Koti", + "ekp": "Ekpeye", + "ekr": "Yace", + "eky": "Eastern Kayah", + "el": "Modern Greek (1453-)", + "ele": "Elepi", + "elh": "El Hugeirat", + "eli": "Nding", + "elk": "Elkei", + "elm": "Eleme", + "elo": "El Molo", + "elu": "Elu", + "elx": "Elamite", + "ema": "Emai-Iuleha-Ora", + "emb": "Embaloh", + "eme": "Emerillon", + "emg": "Eastern Meohang", + "emi": "Mussau-Emira", + "emk": "Eastern Maninkakan", + "emm": "Mamulique", + "emn": "Eman", + "emp": "Northern Emberá", + "emq": "Eastern Minyag", + "ems": "Pacific Gulf Yupik", + "emu": "Eastern Muria", + "emw": "Emplawas", + "emx": "Erromintxela", + "emy": "Epigraphic Mayan", + "emz": "Mbessa", + "en": "English", + "ena": "Apali", + "enb": "Markweeta", + "enc": "En", + "end": "Ende", + "enf": "Forest Enets", + "enh": "Tundra Enets", + "enl": "Enlhet", + "enm": "Middle English (1100-1500)", + "enn": "Engenni", + "eno": "Enggano", + "enq": "Enga", + "enr": "Emumu; Emem", + "enu": "Enu", + "env": "Enwan (Edo State)", + "enw": "Enwan (Akwa Ibom State)", + "enx": "Enxet", + "eo": "Esperanto", + "eot": "Beti (Côte d'Ivoire)", + "epi": "Epie", + "era": "Eravallan", + "erg": "Sie", + "erh": "Eruwa", + "eri": "Ogea", + "erk": "South Efate", + "ero": "Horpa", + "err": "Erre", + "ers": "Ersu", + "ert": "Eritai", + "erw": "Erokwanas", + "es": "Spanish; Castilian", + "ese": "Ese Ejja", + "esg": "Aheri Gondi", + "esh": "Eshtehardi", + "esi": "North Alaskan Inupiatun", + "esk": "Northwest Alaska Inupiatun", + "esl": "Egypt Sign Language", + "esm": "Esuma", + "esn": "Salvadoran Sign Language", + "eso": "Estonian Sign Language", + "esq": "Esselen", + "ess": "Central Siberian Yupik", + "esu": "Central Yupik", + "esx": "Eskimo-Aleut languages", + "esy": "Eskayan", + "et": "Estonian", + "etb": "Etebi", + "etc": "Etchemin", + "eth": "Ethiopian Sign Language", + "etn": "Eton (Vanuatu)", + "eto": "Eton (Cameroon)", + "etr": "Edolo", + "ets": "Yekhee", + "ett": "Etruscan", + "etu": "Ejagham", + "etx": "Eten", + "etz": "Semimi", + "eu": "Basque", + "euq": "Basque (family)", + "eve": "Even", + "evh": "Uvbie", + "evn": "Evenki", + "ewo": "Ewondo", + "ext": "Extremaduran", + "eya": "Eyak", + "eyo": "Keiyo", + "eza": "Ezaa", + "eze": "Uzekwe", + "fa": "Persian", + "faa": "Fasu", + "fab": "Fa d'Ambu", + "fad": "Wagi", + "faf": "Fagani", + "fag": "Finongan", + "fah": "Baissa Fali", + "fai": "Faiwol", + "faj": "Faita", + "fak": "Fang (Cameroon)", + "fal": "South Fali", + "fam": "Fam", + "fan": "Fang (Equatorial Guinea)", + "fap": "Paloor", + "far": "Fataleka", + "fat": "Fanti", + "fau": "Fayu", + "fax": "Fala", + "fay": "Southwestern Fars", + "faz": "Northwestern Fars", + "fbl": "West Albay Bikol", + "fcs": "Quebec Sign Language", + "fer": "Feroge", + "ff": "Fulah", + "ffi": "Foia Foia", + "ffm": "Maasina Fulfulde", + "fgr": "Fongoro", + "fi": "Finnish", + "fia": "Nobiin", + "fie": "Fyer", + "fif": "Faifi", + "fil": "Filipino; Pilipino", + "fip": "Fipa", + "fir": "Firan", + "fit": "Tornedalen Finnish; Meänkieli", + "fiu": "Finno-Ugrian languages", + "fiw": "Fiwaga", + "fj": "Fijian", + "fkk": "Kirya-Konzəl", + "fkv": "Kven Finnish", + "fla": "Kalispel-Pend d'Oreille", + "flh": "Foau", + "fli": "Fali", + "fll": "North Fali", + "fln": "Flinders Island", + "flr": "Fuliiru", + "fly": "Flaaitaal; Tsotsitaal", + "fmp": "Fe'fe'", + "fmu": "Far Western Muria", + "fnb": "Fanbak", + "fng": "Fanagalo", + "fni": "Fania", + "fo": "Faroese", + "fod": "Foodo", + "foi": "Foi", + "fom": "Foma", + "fon": "Fon", + "for": "Fore", + "fos": "Siraya", + "fox": "Formosan languages", + "fpe": "Fernando Po Creole English", + "fqs": "Fas", + "fr": "French", + "frc": "Cajun French", + "frd": "Fordata", + "frk": "Frankish", + "frm": "Middle French (ca. 1400-1600)", + "fro": "Old French (842-ca. 1400)", + "frp": "Arpitan; Francoprovençal", + "frq": "Forak", + "frr": "Northern Frisian", + "frs": "Eastern Frisian", + "frt": "Fortsenal", + "fse": "Finnish Sign Language", + "fsl": "French Sign Language", + "fss": "Finland-Swedish Sign Language; finlandssvenskt teckenspråk; suomenruotsalainen viittomakieli", + "fub": "Adamawa Fulfulde", + "fuc": "Pulaar", + "fud": "East Futuna", + "fue": "Borgu Fulfulde", + "fuf": "Pular", + "fuh": "Western Niger Fulfulde", + "fui": "Bagirmi Fulfulde", + "fuj": "Ko", + "fum": "Fum", + "fun": "Fulniô", + "fuq": "Central-Eastern Niger Fulfulde", + "fur": "Friulian", + "fut": "Futuna-Aniwa", + "fuu": "Furu", + "fuv": "Nigerian Fulfulde", + "fuy": "Fuyug", + "fvr": "Fur", + "fwa": "Fwâi", + "fwe": "Fwe", + "fy": "Western Frisian", + "ga": "Irish", + "gaa": "Ga", + "gab": "Gabri", + "gac": "Mixed Great Andamanese", + "gad": "Gaddang", + "gae": "Guarequena", + "gaf": "Gende", + "gag": "Gagauz", + "gah": "Alekano", + "gai": "Borei", + "gaj": "Gadsup", + "gak": "Gamkonora", + "gal": "Galolen", + "gam": "Kandawo", + "gan": "Gan Chinese", + "gao": "Gants", + "gap": "Gal", + "gaq": "Gata'", + "gar": "Galeya", + "gas": "Adiwasi Garasia", + "gat": "Kenati", + "gau": "Mudhili Gadaba", + "gaw": "Nobonob", + "gax": "Borana-Arsi-Guji Oromo", + "gay": "Gayo", + "gaz": "West Central Oromo", + "gba": "Gbaya (Central African Republic)", + "gbb": "Kaytetye", + "gbd": "Karajarri", + "gbe": "Niksek", + "gbf": "Gaikundi", + "gbg": "Gbanziri", + "gbh": "Defi Gbe", + "gbi": "Galela", + "gbj": "Bodo Gadaba", + "gbk": "Gaddi", + "gbl": "Gamit", + "gbm": "Garhwali", + "gbn": "Mo'da", + "gbo": "Northern Grebo", + "gbp": "Gbaya-Bossangoa", + "gbq": "Gbaya-Bozoum", + "gbr": "Gbagyi", + "gbs": "Gbesi Gbe", + "gbu": "Gagadu", + "gbv": "Gbanu", + "gbw": "Gabi-Gabi", + "gbx": "Eastern Xwla Gbe", + "gby": "Gbari", + "gbz": "Zoroastrian Dari", + "gcc": "Mali", + "gcd": "Ganggalida", + "gce": "Galice", + "gcf": "Guadeloupean Creole French", + "gcl": "Grenadian Creole English", + "gcn": "Gaina", + "gcr": "Guianese Creole French", + "gct": "Colonia Tovar German", + "gd": "Scottish Gaelic; Gaelic", + "gda": "Gade Lohar", + "gdb": "Pottangi Ollar Gadaba", + "gdc": "Gugu Badhun", + "gdd": "Gedaged", + "gde": "Gude", + "gdf": "Guduf-Gava", + "gdg": "Ga'dang", + "gdh": "Gadjerawang; Gajirrabeng", + "gdi": "Gundi", + "gdj": "Gurdjar", + "gdk": "Gadang", + "gdl": "Dirasha", + "gdm": "Laal", + "gdn": "Umanakaina", + "gdo": "Ghodoberi", + "gdq": "Mehri", + "gdr": "Wipi", + "gds": "Ghandruk Sign Language", + "gdt": "Kungardutyi", + "gdu": "Gudu", + "gdx": "Godwari", + "gea": "Geruma", + "geb": "Kire", + "gec": "Gboloo Grebo", + "ged": "Gade", + "gef": "Gerai", + "geg": "Gengle", + "geh": "Hutterite German; Hutterisch", + "gei": "Gebe", + "gej": "Gen", + "gek": "Ywom", + "gel": "ut-Ma'in", + "gem": "Germanic languages", + "geq": "Geme", + "ges": "Geser-Gorom", + "gev": "Eviya", + "gew": "Gera", + "gex": "Garre", + "gey": "Enya", + "gez": "Geez", + "gfk": "Patpatar", + "gft": "Gafat", + "gga": "Gao", + "ggb": "Gbii", + "ggd": "Gugadj", + "gge": "Gurr-goni", + "ggg": "Gurgula", + "ggk": "Kungarakany", + "ggl": "Ganglau", + "ggt": "Gitua", + "ggu": "Gagu; Gban", + "ggw": "Gogodala", + "gha": "Ghadamès", + "ghc": "Hiberno-Scottish Gaelic", + "ghe": "Southern Ghale", + "ghh": "Northern Ghale", + "ghk": "Geko Karen", + "ghl": "Ghulfan", + "ghn": "Ghanongga", + "gho": "Ghomara", + "ghr": "Ghera", + "ghs": "Guhu-Samane", + "ght": "Kuke; Kutang Ghale", + "gia": "Kija", + "gib": "Gibanawa", + "gic": "Gail", + "gid": "Gidar", + "gie": "Gaɓogbo; Guébie", + "gig": "Goaria", + "gih": "Githabul", + "gii": "Girirra", + "gil": "Gilbertese", + "gim": "Gimi (Eastern Highlands)", + "gin": "Hinukh", + "gip": "Gimi (West New Britain)", + "giq": "Green Gelao", + "gir": "Red Gelao", + "gis": "North Giziga", + "git": "Gitxsan", + "giu": "Mulao", + "giw": "White Gelao", + "gix": "Gilima", + "giy": "Giyug", + "giz": "South Giziga", + "gjk": "Kachi Koli", + "gjm": "Gunditjmara", + "gjn": "Gonja", + "gjr": "Gurindji Kriol", + "gju": "Gujari", + "gka": "Guya", + "gkd": "Magɨ (Madang Province)", + "gke": "Ndai", + "gkn": "Gokana", + "gko": "Kok-Nar", + "gkp": "Guinea Kpelle", + "gku": "ǂUngkue", + "gl": "Galician", + "glb": "Belning", + "glc": "Bon Gula", + "gld": "Nanai", + "glh": "Northwest Pashai; Northwest Pashayi", + "glj": "Gula Iro", + "glk": "Gilaki", + "gll": "Garlali", + "glo": "Galambu", + "glr": "Glaro-Twabo", + "glu": "Gula (Chad)", + "glw": "Glavda", + "gly": "Gule", + "gma": "Gambera", + "gmb": "Gula'alaa", + "gmd": "Mághdì", + "gme": "East Germanic languages", + "gmg": "Magɨyi", + "gmh": "Middle High German (ca. 1050-1500)", + "gml": "Middle Low German", + "gmm": "Gbaya-Mbodomo", + "gmn": "Gimnime", + "gmq": "North Germanic languages", + "gmr": "Mirning; Mirniny", + "gmu": "Gumalu", + "gmv": "Gamo", + "gmw": "West Germanic languages", + "gmx": "Magoma", + "gmy": "Mycenaean Greek", + "gmz": "Mgbolizhia", + "gn": "Guarani", + "gna": "Kaansa", + "gnb": "Gangte", + "gnc": "Guanche", + "gnd": "Zulgo-Gemzek", + "gne": "Ganang", + "gng": "Ngangam", + "gnh": "Lere", + "gni": "Gooniyandi", + "gnj": "Ngen", + "gnk": "ǁGana", + "gnl": "Gangulu", + "gnm": "Ginuman", + "gnn": "Gumatj", + "gno": "Northern Gondi", + "gnq": "Gana", + "gnr": "Gureng Gureng", + "gnt": "Guntai", + "gnu": "Gnau", + "gnw": "Western Bolivian Guaraní", + "gnz": "Ganzi", + "goa": "Guro", + "gob": "Playero", + "goc": "Gorakor", + "god": "Godié", + "goe": "Gongduk", + "gof": "Gofa", + "gog": "Gogo", + "goh": "Old High German (ca. 750-1050)", + "goi": "Gobasi", + "goj": "Gowlan", + "gok": "Gowli", + "gol": "Gola", + "gom": "Goan Konkani", + "gon": "Gondi", + "goo": "Gone Dau", + "gop": "Yeretuar", + "goq": "Gorap", + "gor": "Gorontalo", + "gos": "Gronings", + "got": "Gothic", + "gou": "Gavar", + "gov": "Goo", + "gow": "Gorowa", + "gox": "Gobu", + "goy": "Goundo", + "goz": "Gozarkhani", + "gpa": "Gupa-Abawa", + "gpe": "Ghanaian Pidgin English", + "gpn": "Taiap", + "gqa": "Ga'anda", + "gqi": "Guiqiong", + "gqn": "Guana (Brazil)", + "gqr": "Gor", + "gqu": "Qau", + "gra": "Rajput Garasia", + "grb": "Grebo", + "grc": "Ancient Greek (to 1453)", + "grd": "Guruntum-Mbaaru", + "grg": "Madi", + "grh": "Gbiri-Niragu", + "gri": "Ghari", + "grj": "Southern Grebo", + "grk": "Greek languages", + "grm": "Kota Marudu Talantang", + "gro": "Groma", + "grq": "Gorovu", + "grr": "Taznatit", + "grs": "Gresi", + "grt": "Garo", + "gru": "Kistane", + "grv": "Central Grebo", + "grw": "Gweda", + "grx": "Guriaso", + "gry": "Barclayville Grebo", + "grz": "Guramalum", + "gse": "Ghanaian Sign Language", + "gsg": "German Sign Language", + "gsl": "Gusilay", + "gsm": "Guatemalan Sign Language", + "gsn": "Nema; Gusan", + "gso": "Southwest Gbaya", + "gsp": "Wasembo", + "gss": "Greek Sign Language", + "gsw": "Swiss German; Alemannic; Alsatian", + "gta": "Guató", + "gtu": "Aghu-Tharnggala", + "gu": "Gujarati", + "gua": "Shiki", + "gub": "Guajajára", + "guc": "Wayuu", + "gud": "Yocoboué Dida", + "gue": "Gurindji", + "guf": "Gupapuyngu", + "gug": "Paraguayan Guaraní", + "guh": "Guahibo", + "gui": "Eastern Bolivian Guaraní", + "guk": "Gumuz", + "gul": "Sea Island Creole English", + "gum": "Guambiano", + "gun": "Mbyá Guaraní", + "guo": "Guayabero", + "gup": "Gunwinggu", + "guq": "Aché", + "gur": "Farefare", + "gus": "Guinean Sign Language", + "gut": "Maléku Jaíka", + "guu": "Yanomamö", + "guw": "Gun", + "gux": "Gourmanchéma", + "guz": "Gusii; Ekegusii", + "gv": "Manx", + "gva": "Guana (Paraguay)", + "gvc": "Guanano", + "gve": "Duwet", + "gvf": "Golin", + "gvj": "Guajá", + "gvl": "Gulay", + "gvm": "Gurmana", + "gvn": "Kuku-Yalanji", + "gvo": "Gavião Do Jiparaná", + "gvp": "Pará Gavião", + "gvr": "Gurung", + "gvs": "Gumawana", + "gvy": "Guyani", + "gwa": "Mbato", + "gwb": "Gwa", + "gwc": "Gawri; Kalami", + "gwd": "Gawwada", + "gwe": "Gweno", + "gwf": "Gowro", + "gwg": "Moo", + "gwi": "Gwichʼin", + "gwj": "ǀGwi", + "gwm": "Awngthim", + "gwn": "Gwandara", + "gwr": "Gwere", + "gwt": "Gawar-Bati", + "gwu": "Guwamu", + "gww": "Kwini", + "gwx": "Gua", + "gxx": "Wè Southern", + "gya": "Northwest Gbaya", + "gyb": "Garus", + "gyd": "Kayardild", + "gye": "Gyem", + "gyf": "Gungabula", + "gyg": "Gbayi", + "gyi": "Gyele", + "gyl": "Gayil", + "gym": "Ngäbere", + "gyn": "Guyanese Creole English", + "gyo": "Gyalsumdo", + "gyr": "Guarayu", + "gyy": "Gunya", + "gyz": "Geji; Gyaazi", + "gza": "Ganza", + "gzi": "Gazi", + "gzn": "Gane", + "ha": "Hausa", + "haa": "Han", + "hab": "Hanoi Sign Language", + "hac": "Gurani", + "had": "Hatam", + "hae": "Eastern Oromo", + "haf": "Haiphong Sign Language", + "hag": "Hanga", + "hah": "Hahon", + "hai": "Haida", + "haj": "Hajong", + "hak": "Hakka Chinese", + "hal": "Halang", + "ham": "Hewa", + "han": "Hangaza", + "hao": "Hakö", + "hap": "Hupla", + "haq": "Ha", + "har": "Harari", + "has": "Haisla", + "hav": "Havu", + "haw": "Hawaiian", + "hax": "Southern Haida", + "hay": "Haya", + "haz": "Hazaragi", + "hba": "Hamba", + "hbb": "Huba", + "hbn": "Heiban", + "hbo": "Ancient Hebrew", + "hbu": "Habu", + "hca": "Andaman Creole Hindi", + "hch": "Huichol", + "hdn": "Northern Haida", + "hds": "Honduras Sign Language", + "hdy": "Hadiyya", + "he": "Hebrew", + "hea": "Northern Qiandong Miao", + "hed": "Herdé", + "heg": "Helong", + "heh": "Hehe", + "hei": "Heiltsuk", + "hem": "Hemba", + "hgm": "Haiǁom", + "hgw": "Haigwai", + "hhi": "Hoia Hoia", + "hhr": "Kerak", + "hhy": "Hoyahoya", + "hi": "Hindi", + "hia": "Lamang", + "hib": "Hibito", + "hid": "Hidatsa", + "hif": "Fiji Hindi", + "hig": "Kamwe", + "hih": "Pamosu", + "hii": "Hinduri", + "hij": "Hijuk", + "hik": "Seit-Kaitetu", + "hil": "Hiligaynon", + "him": "Himachali languages; Western Pahari languages", + "hio": "Tsoa", + "hir": "Himarimã", + "hit": "Hittite", + "hiw": "Hiw", + "hix": "Hixkaryána", + "hji": "Haji", + "hka": "Kahe", + "hke": "Hunde", + "hkh": "Khah; Poguli", + "hkk": "Hunjara-Kaina Ke", + "hkn": "Mel-Khaonh", + "hks": "Hong Kong Sign Language; Heung Kong Sau Yue", + "hla": "Halia", + "hlb": "Halbi", + "hld": "Halang Doan", + "hle": "Hlersu", + "hlt": "Matu Chin", + "hlu": "Hieroglyphic Luwian", + "hma": "Southern Mashan Hmong; Southern Mashan Miao", + "hmb": "Humburi Senni Songhay", + "hmc": "Central Huishui Hmong; Central Huishui Miao", + "hmd": "Large Flowery Miao; A-hmaos; Da-Hua Miao", + "hme": "Eastern Huishui Hmong; Eastern Huishui Miao", + "hmf": "Hmong Don", + "hmg": "Southwestern Guiyang Hmong", + "hmh": "Southwestern Huishui Hmong; Southwestern Huishui Miao", + "hmi": "Northern Huishui Hmong; Northern Huishui Miao", + "hmj": "Ge; Gejia", + "hmk": "Maek", + "hml": "Luopohe Hmong; Luopohe Miao", + "hmm": "Central Mashan Hmong; Central Mashan Miao", + "hmn": "Hmong; Mong", + "hmp": "Northern Mashan Hmong; Northern Mashan Miao", + "hmq": "Eastern Qiandong Miao", + "hmr": "Hmar", + "hms": "Southern Qiandong Miao", + "hmt": "Hamtai", + "hmu": "Hamap", + "hmv": "Hmong Dô", + "hmw": "Western Mashan Hmong; Western Mashan Miao", + "hmx": "Hmong-Mien languages", + "hmy": "Southern Guiyang Hmong; Southern Guiyang Miao", + "hmz": "Hmong Shua; Sinicized Miao", + "hna": "Mina (Cameroon)", + "hnd": "Southern Hindko", + "hne": "Chhattisgarhi", + "hng": "Hungu", + "hnh": "ǁAni", + "hni": "Hani", + "hnj": "Hmong Njua; Mong Leng; Mong Njua", + "hnn": "Hanunoo", + "hno": "Northern Hindko", + "hns": "Caribbean Hindustani", + "hnu": "Hung", + "ho": "Hiri Motu", + "hoa": "Hoava", + "hob": "Mari (Madang Province)", + "hoc": "Ho", + "hod": "Holma", + "hoe": "Horom", + "hoh": "Hobyót", + "hoi": "Holikachuk", + "hoj": "Hadothi; Haroti", + "hok": "Hokan languages", + "hol": "Holu", + "hom": "Homa", + "hoo": "Holoholo", + "hop": "Hopi", + "hor": "Horo", + "hos": "Ho Chi Minh City Sign Language", + "hot": "Hote; Malê", + "hov": "Hovongan", + "how": "Honi", + "hoy": "Holiya", + "hoz": "Hozo", + "hpo": "Hpon", + "hps": "Hawai'i Sign Language (HSL); Hawai'i Pidgin Sign Language", + "hr": "Croatian", + "hra": "Hrangkhol", + "hrc": "Niwer Mil", + "hre": "Hre", + "hrk": "Haruku", + "hrm": "Horned Miao", + "hro": "Haroi", + "hrp": "Nhirrpi", + "hrt": "Hértevin", + "hru": "Hruso", + "hrw": "Warwar Feni", + "hrx": "Hunsrik", + "hrz": "Harzani", + "hsb": "Upper Sorbian", + "hsh": "Hungarian Sign Language", + "hsl": "Hausa Sign Language", + "hsn": "Xiang Chinese", + "hss": "Harsusi", + "ht": "Haitian; Haitian Creole", + "hti": "Hoti", + "hto": "Minica Huitoto", + "hts": "Hadza", + "htu": "Hitu", + "htx": "Middle Hittite", + "hu": "Hungarian", + "hub": "Huambisa", + "huc": "ǂHua; ǂʼAmkhoe", + "hud": "Huaulu", + "hue": "San Francisco Del Mar Huave", + "huf": "Humene", + "hug": "Huachipaeri", + "huh": "Huilliche", + "hui": "Huli", + "huj": "Northern Guiyang Hmong; Northern Guiyang Miao", + "huk": "Hulung", + "hul": "Hula", + "hum": "Hungana", + "huo": "Hu", + "hup": "Hupa", + "huq": "Tsat", + "hur": "Halkomelem", + "hus": "Huastec", + "hut": "Humla", + "huu": "Murui Huitoto", + "huv": "San Mateo Del Mar Huave", + "huw": "Hukumina", + "hux": "Nüpode Huitoto", + "huy": "Hulaulá", + "huz": "Hunzib", + "hvc": "Haitian Vodoun Culture Language", + "hve": "San Dionisio Del Mar Huave", + "hvk": "Haveke", + "hvn": "Sabu", + "hvv": "Santa María Del Mar Huave", + "hwa": "Wané", + "hwc": "Hawai'i Creole English; Hawai'i Pidgin", + "hwo": "Hwana", + "hy": "Armenian", + "hya": "Hya", + "hyw": "Western Armenian", + "hyx": "Armenian (family)", + "hz": "Herero", + "ia": "Interlingua (International Auxiliary Language Association)", + "iai": "Iaai", + "ian": "Iatmul", + "iar": "Purari", + "iba": "Iban", + "ibb": "Ibibio", + "ibd": "Iwaidja", + "ibe": "Akpes", + "ibg": "Ibanag", + "ibh": "Bih", + "ibl": "Ibaloi", + "ibm": "Agoi", + "ibn": "Ibino", + "ibr": "Ibuoro", + "ibu": "Ibu", + "iby": "Ibani", + "ica": "Ede Ica", + "ich": "Etkywan", + "icl": "Icelandic Sign Language", + "icr": "Islander Creole English", + "id": "Indonesian", + "ida": "Idakho-Isukha-Tiriki; Luidakho-Luisukha-Lutirichi", + "idb": "Indo-Portuguese", + "idc": "Idon; Ajiya", + "idd": "Ede Idaca", + "ide": "Idere", + "idi": "Idi", + "idr": "Indri", + "ids": "Idesa", + "idt": "Idaté", + "idu": "Idoma", + "ie": "Interlingue; Occidental", + "ifa": "Amganad Ifugao", + "ifb": "Batad Ifugao; Ayangan Ifugao", + "ife": "Ifè", + "iff": "Ifo", + "ifk": "Tuwali Ifugao", + "ifm": "Teke-Fuumu", + "ifu": "Mayoyao Ifugao", + "ify": "Keley-I Kallahan", + "ig": "Igbo", + "igb": "Ebira", + "ige": "Igede", + "igg": "Igana", + "igl": "Igala", + "igm": "Kanggape", + "ign": "Ignaciano", + "igo": "Isebe", + "igs": "Interglossa", + "igw": "Igwe", + "ihb": "Iha Based Pidgin", + "ihi": "Ihievbe", + "ihp": "Iha", + "ihw": "Bidhawal", + "ii": "Sichuan Yi; Nuosu", + "iin": "Thiin", + "iir": "Indo-Iranian languages", + "ijc": "Izon", + "ije": "Biseni", + "ijj": "Ede Ije", + "ijn": "Kalabari", + "ijo": "Ijo languages", + "ijs": "Southeast Ijo", + "ik": "Inupiaq", + "ike": "Eastern Canadian Inuktitut", + "iki": "Iko", + "ikk": "Ika", + "ikl": "Ikulu", + "iko": "Olulumo-Ikom", + "ikp": "Ikpeshi", + "ikr": "Ikaranggal", + "iks": "Inuit Sign Language", + "ikt": "Inuinnaqtun; Western Canadian Inuktitut", + "ikv": "Iku-Gora-Ankwa", + "ikw": "Ikwere", + "ikx": "Ik", + "ikz": "Ikizu", + "ila": "Ile Ape", + "ilb": "Ila", + "ilg": "Garig-Ilgar", + "ili": "Ili Turki", + "ilk": "Ilongot", + "ilm": "Iranun (Malaysia)", + "ilo": "Iloko", + "ilp": "Iranun (Philippines)", + "ils": "International Sign", + "ilu": "Ili'uun", + "ilv": "Ilue", + "ima": "Mala Malasar", + "imi": "Anamgura", + "iml": "Miluk", + "imn": "Imonda", + "imo": "Imbongu", + "imr": "Imroing", + "ims": "Marsian", + "imt": "Imotong", + "imy": "Milyan", + "inb": "Inga", + "inc": "Indic languages", + "ine": "Indo-European languages", + "ing": "Degexit'an", + "inh": "Ingush", + "inj": "Jungle Inga", + "inl": "Indonesian Sign Language", + "inm": "Minaean", + "inn": "Isinai", + "ino": "Inoke-Yate", + "inp": "Iñapari", + "ins": "Indian Sign Language", + "int": "Intha", + "inz": "Ineseño", + "io": "Ido", + "ior": "Inor", + "iou": "Tuma-Irumu", + "iow": "Iowa-Oto", + "ipi": "Ipili", + "ipo": "Ipiko", + "iqu": "Iquito", + "iqw": "Ikwo", + "ira": "Iranian languages", + "ire": "Iresim", + "irh": "Irarutu", + "iri": "Rigwe; Irigwe", + "irk": "Iraqw", + "irn": "Irántxe", + "iro": "Iroquoian languages", + "irr": "Ir", + "iru": "Irula", + "irx": "Kamberau", + "iry": "Iraya", + "is": "Icelandic", + "isa": "Isabi", + "isc": "Isconahua", + "isd": "Isnag", + "ise": "Italian Sign Language", + "isg": "Irish Sign Language", + "ish": "Esan", + "isi": "Nkem-Nkum", + "isk": "Ishkashimi", + "ism": "Masimasi", + "isn": "Isanzu", + "iso": "Isoko", + "isr": "Israeli Sign Language", + "ist": "Istriot", + "isu": "Isu (Menchum Division)", + "it": "Italian", + "itb": "Binongan Itneg", + "itc": "Italic languages", + "itd": "Southern Tidung", + "ite": "Itene", + "iti": "Inlaod Itneg", + "itk": "Judeo-Italian", + "itl": "Itelmen", + "itm": "Itu Mbon Uzo", + "ito": "Itonama", + "itr": "Iteri", + "its": "Isekiri", + "itt": "Maeng Itneg", + "itv": "Itawit", + "itw": "Ito", + "itx": "Itik", + "ity": "Moyadan Itneg", + "itz": "Itzá", + "iu": "Inuktitut", + "ium": "Iu Mien", + "ivb": "Ibatan", + "ivv": "Ivatan", + "iwk": "I-Wak", + "iwm": "Iwam", + "iwo": "Iwur", + "iws": "Sepik Iwam", + "ixc": "Ixcatec", + "ixl": "Ixil", + "iya": "Iyayu", + "iyo": "Mesaka", + "iyx": "Yaka (Congo)", + "izh": "Ingrian", + "izr": "Izere", + "izz": "Izii", + "ja": "Japanese", + "jaa": "Jamamadí", + "jab": "Hyam", + "jac": "Popti'; Jakalteko", + "jad": "Jahanka", + "jae": "Yabem", + "jaf": "Jara", + "jah": "Jah Hut", + "jaj": "Zazao", + "jak": "Jakun", + "jal": "Yalahatan", + "jam": "Jamaican Creole English", + "jan": "Jandai", + "jao": "Yanyuwa", + "jaq": "Yaqay", + "jas": "New Caledonian Javanese", + "jat": "Jakati", + "jau": "Yaur", + "jax": "Jambi Malay", + "jay": "Yan-nhangu; Nhangu", + "jaz": "Jawe", + "jbe": "Judeo-Berber", + "jbi": "Badjiri", + "jbj": "Arandai", + "jbk": "Barikewa", + "jbm": "Bijim", + "jbn": "Nafusi", + "jbo": "Lojban", + "jbr": "Jofotek-Bromnya", + "jbt": "Jabutí", + "jbu": "Jukun Takum", + "jbw": "Yawijibaya", + "jcs": "Jamaican Country Sign Language", + "jct": "Krymchak", + "jda": "Jad", + "jdg": "Jadgali", + "jdt": "Judeo-Tat", + "jeb": "Jebero", + "jee": "Jerung", + "jeh": "Jeh", + "jei": "Yei", + "jek": "Jeri Kuo", + "jel": "Yelmek", + "jen": "Dza", + "jer": "Jere", + "jet": "Manem", + "jeu": "Jonkor Bourmataguil", + "jgb": "Ngbee", + "jge": "Judeo-Georgian", + "jgk": "Gwak", + "jgo": "Ngomba", + "jhi": "Jehai", + "jhs": "Jhankot Sign Language", + "jia": "Jina", + "jib": "Jibu", + "jic": "Tol", + "jid": "Bu (Kaduna State)", + "jie": "Jilbe", + "jig": "Jingulu; Djingili", + "jih": "sTodsde; Shangzhai", + "jii": "Jiiddu", + "jil": "Jilim", + "jim": "Jimi (Cameroon)", + "jio": "Jiamao", + "jiq": "Guanyinqiao; Lavrung", + "jit": "Jita", + "jiu": "Youle Jinuo", + "jiv": "Shuar", + "jiy": "Buyuan Jinuo", + "jje": "Jejueo", + "jjr": "Bankal", + "jka": "Kaera", + "jkm": "Mobwa Karen", + "jko": "Kubo", + "jkp": "Paku Karen", + "jkr": "Koro (India)", + "jks": "Amami Koniya Sign Language", + "jku": "Labir", + "jle": "Ngile", + "jls": "Jamaican Sign Language", + "jma": "Dima", + "jmb": "Zumbun", + "jmc": "Machame", + "jmd": "Yamdena", + "jmi": "Jimi (Nigeria)", + "jml": "Jumli", + "jmn": "Makuri Naga", + "jmr": "Kamara", + "jms": "Mashi (Nigeria)", + "jmw": "Mouwase", + "jmx": "Western Juxtlahuaca Mixtec", + "jna": "Jangshung", + "jnd": "Jandavra", + "jng": "Yangman", + "jni": "Janji", + "jnj": "Yemsa", + "jnl": "Rawat", + "jns": "Jaunsari", + "job": "Joba", + "jod": "Wojenaka", + "jog": "Jogi", + "jor": "Jorá", + "jos": "Jordanian Sign Language", + "jow": "Jowulu", + "jpa": "Jewish Palestinian Aramaic", + "jpr": "Judeo-Persian", + "jpx": "Japanese (family)", + "jqr": "Jaqaru", + "jra": "Jarai", + "jrb": "Judeo-Arabic", + "jrr": "Jiru", + "jrt": "Jakattoe", + "jru": "Japrería", + "jsl": "Japanese Sign Language", + "jua": "Júma", + "jub": "Wannu", + "juc": "Jurchen", + "jud": "Worodougou", + "juh": "Hõne", + "jui": "Ngadjuri", + "juk": "Wapan", + "jul": "Jirel", + "jum": "Jumjum", + "jun": "Juang", + "juo": "Jiba", + "jup": "Hupdë", + "jur": "Jurúna", + "jus": "Jumla Sign Language", + "jut": "Jutish", + "juu": "Ju", + "juw": "Wãpha", + "juy": "Juray", + "jv": "Javanese", + "jvd": "Javindo", + "jvn": "Caribbean Javanese", + "jwi": "Jwira-Pepesa", + "jya": "Jiarong", + "jye": "Judeo-Yemeni Arabic", + "jyy": "Jaya", + "ka": "Georgian", + "kaa": "Kara-Kalpak; Karakalpak", + "kab": "Kabyle", + "kac": "Kachin; Jingpho", + "kad": "Adara", + "kae": "Ketangalan", + "kaf": "Katso", + "kag": "Kajaman", + "kah": "Kara (Central African Republic)", + "kai": "Karekare", + "kaj": "Jju", + "kak": "Kalanguya; Kayapa Kallahan", + "kam": "Kamba (Kenya)", + "kao": "Xaasongaxango", + "kap": "Bezhta", + "kaq": "Capanahua", + "kar": "Karen languages", + "kav": "Katukína", + "kaw": "Kawi", + "kax": "Kao", + "kay": "Kamayurá", + "kba": "Kalarko", + "kbb": "Kaxuiâna", + "kbc": "Kadiwéu", + "kbd": "Kabardian", + "kbe": "Kanju", + "kbg": "Khamba", + "kbh": "Camsá", + "kbi": "Kaptiau", + "kbj": "Kari", + "kbk": "Grass Koiari", + "kbl": "Kanembu", + "kbm": "Iwal", + "kbn": "Kare (Central African Republic)", + "kbo": "Keliko", + "kbp": "Kabiyè", + "kbq": "Kamano", + "kbr": "Kafa", + "kbs": "Kande", + "kbt": "Abadi", + "kbu": "Kabutra", + "kbv": "Dera (Indonesia)", + "kbw": "Kaiep", + "kbx": "Ap Ma", + "kby": "Manga Kanuri", + "kbz": "Duhwa", + "kca": "Khanty", + "kcb": "Kawacha", + "kcc": "Lubila", + "kcd": "Ngkâlmpw Kanum", + "kce": "Kaivi", + "kcf": "Ukaan", + "kcg": "Tyap", + "kch": "Vono", + "kci": "Kamantan", + "kcj": "Kobiana", + "kck": "Kalanga", + "kcl": "Kela (Papua New Guinea); Kala", + "kcm": "Gula (Central African Republic)", + "kcn": "Nubi", + "kco": "Kinalakna", + "kcp": "Kanga", + "kcq": "Kamo", + "kcr": "Katla", + "kcs": "Koenoem", + "kct": "Kaian", + "kcu": "Kami (Tanzania)", + "kcv": "Kete", + "kcw": "Kabwari", + "kcx": "Kachama-Ganjule", + "kcy": "Korandje", + "kcz": "Konongo", + "kda": "Worimi", + "kdc": "Kutu", + "kdd": "Yankunytjatjara", + "kde": "Makonde", + "kdf": "Mamusi", + "kdg": "Seba", + "kdh": "Tem", + "kdi": "Kumam", + "kdj": "Karamojong", + "kdk": "Numèè; Kwényi", + "kdl": "Tsikimba", + "kdm": "Kagoma", + "kdn": "Kunda", + "kdo": "Kordofanian languages", + "kdp": "Kaningdon-Nindem", + "kdq": "Koch", + "kdr": "Karaim", + "kdt": "Kuy", + "kdu": "Kadaru", + "kdw": "Koneraw", + "kdx": "Kam", + "kdy": "Keder; Keijar", + "kdz": "Kwaja", + "kea": "Kabuverdianu", + "keb": "Kélé", + "kec": "Keiga", + "ked": "Kerewe", + "kee": "Eastern Keres", + "kef": "Kpessi", + "keg": "Tese", + "keh": "Keak", + "kei": "Kei", + "kej": "Kadar", + "kek": "Kekchí", + "kel": "Kela (Democratic Republic of Congo)", + "kem": "Kemak", + "ken": "Kenyang", + "keo": "Kakwa", + "kep": "Kaikadi", + "keq": "Kamar", + "ker": "Kera", + "kes": "Kugbo", + "ket": "Ket", + "keu": "Akebu", + "kev": "Kanikkaran", + "kew": "West Kewa", + "kex": "Kukna", + "key": "Kupia", + "kez": "Kukele", + "kfa": "Kodava", + "kfb": "Northwestern Kolami", + "kfc": "Konda-Dora", + "kfd": "Korra Koraga", + "kfe": "Kota (India)", + "kff": "Koya", + "kfg": "Kudiya", + "kfh": "Kurichiya", + "kfi": "Kannada Kurumba", + "kfj": "Kemiehua", + "kfk": "Kinnauri", + "kfl": "Kung", + "kfm": "Khunsari", + "kfn": "Kuk", + "kfo": "Koro (Côte d'Ivoire)", + "kfp": "Korwa", + "kfq": "Korku", + "kfr": "Kachhi; Kutchi", + "kfs": "Bilaspuri", + "kft": "Kanjari", + "kfu": "Katkari", + "kfv": "Kurmukar", + "kfw": "Kharam Naga", + "kfx": "Kullu Pahari", + "kfy": "Kumaoni", + "kfz": "Koromfé", + "kg": "Kongo", + "kga": "Koyaga", + "kgb": "Kawe", + "kge": "Komering", + "kgf": "Kube", + "kgg": "Kusunda", + "kgi": "Selangor Sign Language", + "kgj": "Gamale Kham", + "kgk": "Kaiwá", + "kgl": "Kunggari", + "kgm": "Karipúna", + "kgn": "Karingani", + "kgo": "Krongo", + "kgp": "Kaingang", + "kgq": "Kamoro", + "kgr": "Abun", + "kgs": "Kumbainggar", + "kgt": "Somyev", + "kgu": "Kobol", + "kgv": "Karas", + "kgw": "Karon Dori", + "kgx": "Kamaru", + "kgy": "Kyerung", + "kha": "Khasi", + "khb": "Lü", + "khc": "Tukang Besi North", + "khd": "Bädi Kanum", + "khe": "Korowai", + "khf": "Khuen", + "khg": "Khams Tibetan", + "khh": "Kehu", + "khi": "Khoisan languages", + "khj": "Kuturmi", + "khk": "Halh Mongolian", + "khl": "Lusi", + "khn": "Khandesi", + "kho": "Khotanese; Sakan", + "khp": "Kapori; Kapauri", + "khq": "Koyra Chiini Songhay", + "khr": "Kharia", + "khs": "Kasua", + "kht": "Khamti", + "khu": "Nkhumbi", + "khv": "Khvarshi", + "khw": "Khowar", + "khx": "Kanu", + "khy": "Kele (Democratic Republic of Congo)", + "khz": "Keapara", + "ki": "Kikuyu; Gikuyu", + "kia": "Kim", + "kib": "Koalib", + "kic": "Kickapoo", + "kid": "Koshin", + "kie": "Kibet", + "kif": "Eastern Parbate Kham", + "kig": "Kimaama; Kimaghima", + "kih": "Kilmeri", + "kii": "Kitsai", + "kij": "Kilivila", + "kil": "Kariya", + "kim": "Karagas", + "kio": "Kiowa", + "kip": "Sheshi Kham", + "kiq": "Kosadle; Kosare", + "kis": "Kis", + "kit": "Agob", + "kiu": "Kirmanjki (individual language)", + "kiv": "Kimbu", + "kiw": "Northeast Kiwai", + "kix": "Khiamniungan Naga", + "kiy": "Kirikiri", + "kiz": "Kisi", + "kj": "Kuanyama; Kwanyama", + "kja": "Mlap", + "kjb": "Q'anjob'al; Kanjobal", + "kjc": "Coastal Konjo", + "kjd": "Southern Kiwai", + "kje": "Kisar", + "kjg": "Khmu", + "kjh": "Khakas", + "kji": "Zabana", + "kjj": "Khinalugh", + "kjk": "Highland Konjo", + "kjl": "Western Parbate Kham", + "kjm": "Kháng", + "kjn": "Kunjen", + "kjo": "Harijan Kinnauri", + "kjp": "Pwo Eastern Karen", + "kjq": "Western Keres", + "kjr": "Kurudu", + "kjs": "East Kewa", + "kjt": "Phrae Pwo Karen", + "kju": "Kashaya", + "kjv": "Kaikavian Literary Language", + "kjx": "Ramopa", + "kjy": "Erave", + "kjz": "Bumthangkha", + "kk": "Kazakh", + "kka": "Kakanda", + "kkb": "Kwerisa", + "kkc": "Odoodee", + "kkd": "Kinuku", + "kke": "Kakabe", + "kkf": "Kalaktang Monpa", + "kkg": "Mabaka Valley Kalinga", + "kkh": "Khün", + "kki": "Kagulu", + "kkj": "Kako", + "kkk": "Kokota", + "kkl": "Kosarek Yale", + "kkm": "Kiong", + "kkn": "Kon Keu", + "kko": "Karko", + "kkp": "Gugubera; Koko-Bera", + "kkq": "Kaeku", + "kkr": "Kir-Balar", + "kks": "Giiwo", + "kkt": "Koi", + "kku": "Tumi", + "kkv": "Kangean", + "kkw": "Teke-Kukuya", + "kkx": "Kohin", + "kky": "Guugu Yimidhirr; Guguyimidjir", + "kkz": "Kaska", + "kl": "Kalaallisut; Greenlandic", + "kla": "Klamath-Modoc", + "klb": "Kiliwa", + "klc": "Kolbila", + "kld": "Gamilaraay", + "kle": "Kulung (Nepal)", + "klf": "Kendeje", + "klg": "Tagakaulo", + "klh": "Weliki", + "kli": "Kalumpang", + "klj": "Khalaj", + "klk": "Kono (Nigeria)", + "kll": "Kagan Kalagan", + "klm": "Migum", + "kln": "Kalenjin", + "klo": "Kapya", + "klp": "Kamasa", + "klq": "Rumu", + "klr": "Khaling", + "kls": "Kalasha", + "klt": "Nukna", + "klu": "Klao", + "klv": "Maskelynes", + "klw": "Tado; Lindu", + "klx": "Koluwawa", + "kly": "Kalao", + "klz": "Kabola", + "km": "Khmer; Central Khmer", + "kma": "Konni", + "kmb": "Kimbundu", + "kmc": "Southern Dong", + "kmd": "Majukayang Kalinga", + "kme": "Bakole", + "kmf": "Kare (Papua New Guinea)", + "kmg": "Kâte", + "kmh": "Kalam", + "kmi": "Kami (Nigeria)", + "kmj": "Kumarbhag Paharia", + "kmk": "Limos Kalinga", + "kml": "Tanudan Kalinga", + "kmm": "Kom (India)", + "kmn": "Awtuw", + "kmo": "Kwoma", + "kmp": "Gimme", + "kmq": "Kwama", + "kmr": "Northern Kurdish", + "kms": "Kamasau", + "kmt": "Kemtuik", + "kmu": "Kanite", + "kmv": "Karipúna Creole French", + "kmw": "Komo (Democratic Republic of Congo)", + "kmx": "Waboda", + "kmy": "Koma", + "kmz": "Khorasani Turkish", + "kn": "Kannada", + "kna": "Dera (Nigeria)", + "knb": "Lubuagan Kalinga", + "knc": "Central Kanuri", + "knd": "Konda", + "kne": "Kankanaey", + "knf": "Mankanya", + "kng": "Koongo", + "kni": "Kanufi", + "knj": "Western Kanjobal", + "knk": "Kuranko", + "knl": "Keninjal", + "knm": "Kanamarí", + "knn": "Konkani (individual language)", + "kno": "Kono (Sierra Leone)", + "knp": "Kwanja", + "knq": "Kintaq", + "knr": "Kaningra", + "kns": "Kensiu", + "knt": "Panoan Katukína", + "knu": "Kono (Guinea)", + "knv": "Tabo", + "knw": "Kung-Ekoka", + "knx": "Kendayan; Salako", + "kny": "Kanyok", + "knz": "Kalamsé", + "ko": "Korean", + "koa": "Konomala", + "koc": "Kpati", + "kod": "Kodi", + "koe": "Kacipo-Bale Suri", + "kof": "Kubi", + "kog": "Cogui; Kogi", + "koh": "Koyo", + "koi": "Komi-Permyak", + "kok": "Konkani (macrolanguage)", + "kol": "Kol (Papua New Guinea)", + "koo": "Konzo", + "kop": "Waube", + "koq": "Kota (Gabon)", + "kos": "Kosraean", + "kot": "Lagwan", + "kou": "Koke", + "kov": "Kudu-Camo", + "kow": "Kugama", + "koy": "Koyukon", + "koz": "Korak", + "kpa": "Kutto", + "kpb": "Mullu Kurumba", + "kpc": "Curripaco", + "kpd": "Koba", + "kpe": "Kpelle", + "kpf": "Komba", + "kpg": "Kapingamarangi", + "kph": "Kplang", + "kpi": "Kofei", + "kpj": "Karajá", + "kpk": "Kpan", + "kpl": "Kpala", + "kpm": "Koho", + "kpn": "Kepkiriwát", + "kpo": "Ikposo", + "kpq": "Korupun-Sela", + "kpr": "Korafe-Yegha", + "kps": "Tehit", + "kpt": "Karata", + "kpu": "Kafoa", + "kpv": "Komi-Zyrian", + "kpw": "Kobon", + "kpx": "Mountain Koiali", + "kpy": "Koryak", + "kpz": "Kupsabiny", + "kqa": "Mum", + "kqb": "Kovai", + "kqc": "Doromu-Koki", + "kqd": "Koy Sanjaq Surat", + "kqe": "Kalagan", + "kqf": "Kakabai", + "kqg": "Khe", + "kqh": "Kisankasa", + "kqi": "Koitabu", + "kqj": "Koromira", + "kqk": "Kotafon Gbe", + "kql": "Kyenele", + "kqm": "Khisa", + "kqn": "Kaonde", + "kqo": "Eastern Krahn", + "kqp": "Kimré", + "kqq": "Krenak", + "kqr": "Kimaragang", + "kqs": "Northern Kissi", + "kqt": "Klias River Kadazan", + "kqu": "Seroa", + "kqv": "Okolod", + "kqw": "Kandas", + "kqx": "Mser", + "kqy": "Koorete", + "kqz": "Korana", + "kr": "Kanuri", + "kra": "Kumhali", + "krb": "Karkin", + "krc": "Karachay-Balkar", + "krd": "Kairui-Midiki", + "kre": "Panará", + "krf": "Koro (Vanuatu)", + "krh": "Kurama", + "kri": "Krio", + "krj": "Kinaray-A", + "krk": "Kerek", + "krl": "Karelian", + "krn": "Sapo", + "kro": "Kru languages", + "krp": "Korop", + "krr": "Krung", + "krs": "Gbaya (Sudan)", + "krt": "Tumari Kanuri", + "kru": "Kurukh", + "krv": "Kavet", + "krw": "Western Krahn", + "krx": "Karon", + "kry": "Kryts", + "krz": "Sota Kanum", + "ks": "Kashmiri", + "ksa": "Shuwa-Zamani", + "ksb": "Shambala", + "ksc": "Southern Kalinga", + "ksd": "Kuanua", + "kse": "Kuni", + "ksf": "Bafia", + "ksg": "Kusaghe", + "ksh": "Kölsch", + "ksi": "Krisa; I'saka", + "ksj": "Uare", + "ksk": "Kansa", + "ksl": "Kumalu", + "ksm": "Kumba", + "ksn": "Kasiguranin", + "kso": "Kofa", + "ksp": "Kaba", + "ksq": "Kwaami", + "ksr": "Borong", + "kss": "Southern Kisi", + "kst": "Winyé", + "ksu": "Khamyang", + "ksv": "Kusu", + "ksw": "S'gaw Karen", + "ksx": "Kedang", + "ksy": "Kharia Thar", + "ksz": "Kodaku", + "kta": "Katua", + "ktb": "Kambaata", + "ktc": "Kholok", + "ktd": "Kokata; Kukatha", + "kte": "Nubri", + "ktf": "Kwami", + "ktg": "Kalkutung", + "kth": "Karanga", + "kti": "North Muyu", + "ktj": "Plapo Krumen", + "ktk": "Kaniet", + "ktl": "Koroshi", + "ktm": "Kurti", + "ktn": "Karitiâna", + "kto": "Kuot", + "ktp": "Kaduo", + "ktq": "Katabaga", + "kts": "South Muyu", + "ktt": "Ketum", + "ktu": "Kituba (Democratic Republic of Congo)", + "ktv": "Eastern Katu", + "ktw": "Kato", + "ktx": "Kaxararí", + "kty": "Kango (Bas-Uélé District)", + "ktz": "Juǀʼhoan; Juǀʼhoansi", + "ku": "Kurdish", + "kub": "Kutep", + "kuc": "Kwinsu", + "kud": "'Auhelawa", + "kue": "Kuman (Papua New Guinea)", + "kuf": "Western Katu", + "kug": "Kupa", + "kuh": "Kushi", + "kui": "Kuikúro-Kalapálo; Kalapalo", + "kuj": "Kuria", + "kuk": "Kepo'", + "kul": "Kulere", + "kum": "Kumyk", + "kun": "Kunama", + "kuo": "Kumukio", + "kup": "Kunimaipa", + "kuq": "Karipuna", + "kus": "Kusaal", + "kut": "Kutenai", + "kuu": "Upper Kuskokwim", + "kuv": "Kur", + "kuw": "Kpagua", + "kux": "Kukatja", + "kuy": "Kuuku-Ya'u", + "kuz": "Kunza", + "kv": "Komi", + "kva": "Bagvalal", + "kvb": "Kubu", + "kvc": "Kove", + "kvd": "Kui (Indonesia)", + "kve": "Kalabakan", + "kvf": "Kabalai", + "kvg": "Kuni-Boazi", + "kvh": "Komodo", + "kvi": "Kwang", + "kvj": "Psikye", + "kvk": "Korean Sign Language", + "kvl": "Kayaw", + "kvm": "Kendem", + "kvn": "Border Kuna", + "kvo": "Dobel", + "kvp": "Kompane", + "kvq": "Geba Karen", + "kvr": "Kerinci", + "kvt": "Lahta Karen; Lahta", + "kvu": "Yinbaw Karen", + "kvv": "Kola", + "kvw": "Wersing", + "kvx": "Parkari Koli", + "kvy": "Yintale Karen; Yintale", + "kvz": "Tsakwambo; Tsaukambo", + "kw": "Cornish", + "kwa": "Dâw", + "kwb": "Kwa", + "kwc": "Likwala", + "kwd": "Kwaio", + "kwe": "Kwerba", + "kwf": "Kwara'ae", + "kwg": "Sara Kaba Deme", + "kwh": "Kowiai", + "kwi": "Awa-Cuaiquer", + "kwj": "Kwanga", + "kwk": "Kwakiutl", + "kwl": "Kofyar", + "kwm": "Kwambi", + "kwn": "Kwangali", + "kwo": "Kwomtari", + "kwp": "Kodia", + "kwr": "Kwer", + "kws": "Kwese", + "kwt": "Kwesten", + "kwu": "Kwakum", + "kwv": "Sara Kaba Náà", + "kww": "Kwinti", + "kwx": "Khirwar", + "kwy": "San Salvador Kongo", + "kwz": "Kwadi", + "kxa": "Kairiru", + "kxb": "Krobu", + "kxc": "Konso; Khonso", + "kxd": "Brunei", + "kxf": "Manumanaw Karen; Manumanaw", + "kxh": "Karo (Ethiopia)", + "kxi": "Keningau Murut", + "kxj": "Kulfa", + "kxk": "Zayein Karen", + "kxm": "Northern Khmer", + "kxn": "Kanowit-Tanjong Melanau", + "kxo": "Kanoé", + "kxp": "Wadiyara Koli", + "kxq": "Smärky Kanum", + "kxr": "Koro (Papua New Guinea)", + "kxs": "Kangjia", + "kxt": "Koiwat", + "kxv": "Kuvi", + "kxw": "Konai", + "kxx": "Likuba", + "kxy": "Kayong", + "kxz": "Kerewo", + "ky": "Kirghiz; Kyrgyz", + "kya": "Kwaya", + "kyb": "Butbut Kalinga", + "kyc": "Kyaka", + "kyd": "Karey", + "kye": "Krache", + "kyf": "Kouya", + "kyg": "Keyagana", + "kyh": "Karok", + "kyi": "Kiput", + "kyj": "Karao", + "kyk": "Kamayo", + "kyl": "Kalapuya", + "kym": "Kpatili", + "kyn": "Northern Binukidnon", + "kyo": "Kelon", + "kyp": "Kang", + "kyq": "Kenga", + "kyr": "Kuruáya", + "kys": "Baram Kayan", + "kyt": "Kayagar", + "kyu": "Western Kayah", + "kyv": "Kayort", + "kyw": "Kudmali", + "kyx": "Rapoisi", + "kyy": "Kambaira", + "kyz": "Kayabí", + "kza": "Western Karaboro", + "kzb": "Kaibobo", + "kzc": "Bondoukou Kulango", + "kzd": "Kadai", + "kze": "Kosena", + "kzf": "Da'a Kaili", + "kzg": "Kikai", + "kzi": "Kelabit", + "kzk": "Kazukuru", + "kzl": "Kayeli", + "kzm": "Kais", + "kzn": "Kokola", + "kzo": "Kaningi", + "kzp": "Kaidipang", + "kzq": "Kaike", + "kzr": "Karang", + "kzs": "Sugut Dusun", + "kzu": "Kayupulau", + "kzv": "Komyandaret", + "kzw": "Karirí-Xocó", + "kzx": "Kamarian", + "kzy": "Kango (Tshopo District)", + "kzz": "Kalabra", + "la": "Latin", + "laa": "Southern Subanen", + "lab": "Linear A", + "lac": "Lacandon", + "lad": "Ladino", + "lae": "Pattani", + "laf": "Lafofa", + "lag": "Langi", + "lah": "Lahnda", + "lai": "Lambya", + "laj": "Lango (Uganda)", + "lal": "Lalia", + "lam": "Lamba", + "lan": "Laru", + "lap": "Laka (Chad)", + "laq": "Qabiao", + "lar": "Larteh", + "las": "Lama (Togo)", + "lau": "Laba", + "law": "Lauje", + "lax": "Tiwa", + "lay": "Lama Bai", + "laz": "Aribwatsa", + "lb": "Luxembourgish; Letzeburgesch", + "lbb": "Label", + "lbc": "Lakkia", + "lbe": "Lak", + "lbf": "Tinani", + "lbg": "Laopang", + "lbi": "La'bi", + "lbj": "Ladakhi", + "lbk": "Central Bontok", + "lbl": "Libon Bikol", + "lbm": "Lodhi", + "lbn": "Rmeet", + "lbo": "Laven", + "lbq": "Wampar", + "lbr": "Lohorung", + "lbs": "Libyan Sign Language", + "lbt": "Lachi", + "lbu": "Labu", + "lbv": "Lavatbura-Lamusong", + "lbw": "Tolaki", + "lbx": "Lawangan", + "lby": "Lamalama; Lamu-Lamu", + "lbz": "Lardil", + "lcc": "Legenyem", + "lcd": "Lola", + "lce": "Loncong; Sekak", + "lcf": "Lubu", + "lch": "Luchazi", + "lcl": "Lisela", + "lcm": "Tungag", + "lcp": "Western Lawa", + "lcq": "Luhu", + "lcs": "Lisabata-Nuniali", + "lda": "Kla-Dan", + "ldb": "Dũya", + "ldd": "Luri", + "ldg": "Lenyima", + "ldh": "Lamja-Dengsa-Tola", + "ldi": "Laari", + "ldj": "Lemoro", + "ldk": "Leelau", + "ldl": "Kaan", + "ldm": "Landoma", + "ldn": "Láadan", + "ldo": "Loo", + "ldp": "Tso", + "ldq": "Lufu", + "lea": "Lega-Shabunda", + "leb": "Lala-Bisa", + "lec": "Leco", + "led": "Lendu", + "lee": "Lyélé", + "lef": "Lelemi", + "leh": "Lenje", + "lei": "Lemio", + "lej": "Lengola", + "lek": "Leipon", + "lel": "Lele (Democratic Republic of Congo)", + "lem": "Nomaande", + "len": "Lenca", + "leo": "Leti (Cameroon)", + "lep": "Lepcha", + "leq": "Lembena", + "ler": "Lenkau", + "les": "Lese", + "let": "Lesing-Gelimi; Amio-Gelimi", + "leu": "Kara (Papua New Guinea)", + "lev": "Lamma", + "lew": "Ledo Kaili", + "lex": "Luang", + "ley": "Lemolang", + "lez": "Lezghian", + "lfa": "Lefa", + "lfn": "Lingua Franca Nova", + "lg": "Ganda; Luganda", + "lga": "Lungga", + "lgb": "Laghu", + "lgg": "Lugbara", + "lgh": "Laghuu", + "lgi": "Lengilu", + "lgk": "Lingarak; Neverver", + "lgl": "Wala", + "lgm": "Lega-Mwenga", + "lgn": "T'apo; Opuuo", + "lgo": "Lango (South Sudan)", + "lgq": "Logba", + "lgr": "Lengo", + "lgt": "Pahi", + "lgu": "Longgu", + "lgz": "Ligenza", + "lha": "Laha (Viet Nam)", + "lhh": "Laha (Indonesia)", + "lhi": "Lahu Shi", + "lhl": "Lahul Lohar", + "lhm": "Lhomi", + "lhn": "Lahanan", + "lhp": "Lhokpu", + "lhs": "Mlahsö", + "lht": "Lo-Toga", + "lhu": "Lahu", + "li": "Limburgan; Limburger; Limburgish", + "lia": "West-Central Limba", + "lib": "Likum", + "lic": "Hlai", + "lid": "Nyindrou", + "lie": "Likila", + "lif": "Limbu", + "lig": "Ligbi", + "lih": "Lihir", + "lij": "Ligurian", + "lik": "Lika", + "lil": "Lillooet", + "lio": "Liki", + "lip": "Sekpele", + "liq": "Libido", + "lir": "Liberian English", + "lis": "Lisu", + "liu": "Logorik", + "liv": "Liv", + "liw": "Col", + "lix": "Liabuku", + "liy": "Banda-Bambari", + "liz": "Libinza", + "lja": "Golpa", + "lje": "Rampi", + "lji": "Laiyolo", + "ljl": "Li'o", + "ljp": "Lampung Api", + "ljw": "Yirandali", + "ljx": "Yuru", + "lka": "Lakalei", + "lkb": "Kabras; Lukabaras", + "lkc": "Kucong", + "lkd": "Lakondê", + "lke": "Kenyi", + "lkh": "Lakha", + "lki": "Laki", + "lkj": "Remun", + "lkl": "Laeko-Libuat", + "lkm": "Kalaamaya", + "lkn": "Lakon; Vure", + "lko": "Khayo; Olukhayo", + "lkr": "Päri", + "lks": "Kisa; Olushisa", + "lkt": "Lakota", + "lku": "Kungkari", + "lky": "Lokoya", + "lla": "Lala-Roba", + "llb": "Lolo", + "llc": "Lele (Guinea)", + "lld": "Ladin", + "lle": "Lele (Papua New Guinea)", + "llf": "Hermit", + "llg": "Lole", + "llh": "Lamu", + "lli": "Teke-Laali", + "llj": "Ladji Ladji", + "llk": "Lelak", + "lll": "Lilau", + "llm": "Lasalimu", + "lln": "Lele (Chad)", + "llp": "North Efate", + "llq": "Lolak", + "lls": "Lithuanian Sign Language", + "llu": "Lau", + "llx": "Lauan", + "lma": "East Limba", + "lmb": "Merei", + "lmc": "Limilngan", + "lmd": "Lumun", + "lme": "Pévé", + "lmf": "South Lembata", + "lmg": "Lamogai", + "lmh": "Lambichhong", + "lmi": "Lombi", + "lmj": "West Lembata", + "lmk": "Lamkang", + "lml": "Hano", + "lmn": "Lambadi", + "lmo": "Lombard", + "lmp": "Limbum", + "lmq": "Lamatuka", + "lmr": "Lamalera", + "lmu": "Lamenu", + "lmv": "Lomaiviti", + "lmw": "Lake Miwok", + "lmx": "Laimbue", + "lmy": "Lamboya", + "ln": "Lingala", + "lna": "Langbashe", + "lnb": "Mbalanhu", + "lnd": "Lundayeh; Lun Bawang", + "lng": "Langobardic", + "lnh": "Lanoh", + "lni": "Daantanai'", + "lnj": "Leningitij", + "lnl": "South Central Banda", + "lnm": "Langam", + "lnn": "Lorediakarkar", + "lns": "Lamnso'", + "lnu": "Longuda", + "lnw": "Lanima", + "lnz": "Lonzo", + "lo": "Lao", + "loa": "Loloda", + "lob": "Lobi", + "loc": "Inonhan", + "loe": "Saluan", + "lof": "Logol", + "log": "Logo", + "loh": "Narim", + "loi": "Loma (Côte d'Ivoire)", + "loj": "Lou", + "lok": "Loko", + "lol": "Mongo", + "lom": "Loma (Liberia)", + "lon": "Malawi Lomwe", + "loo": "Lombo", + "lop": "Lopa", + "loq": "Lobala", + "lor": "Téén", + "los": "Loniu", + "lot": "Otuho", + "lou": "Louisiana Creole", + "lov": "Lopi", + "low": "Tampias Lobu", + "lox": "Loun", + "loy": "Loke", + "loz": "Lozi", + "lpa": "Lelepa", + "lpe": "Lepki", + "lpn": "Long Phuri Naga", + "lpo": "Lipo", + "lpx": "Lopit", + "lqr": "Logir", + "lra": "Rara Bakati'", + "lrc": "Northern Luri", + "lre": "Laurentian", + "lrg": "Laragia", + "lri": "Marachi; Olumarachi", + "lrk": "Loarki", + "lrl": "Lari", + "lrm": "Marama; Olumarama", + "lrn": "Lorang", + "lro": "Laro", + "lrr": "Southern Yamphu", + "lrt": "Larantuka Malay", + "lrv": "Larevat", + "lrz": "Lemerig", + "lsa": "Lasgerdi", + "lsb": "Burundian Sign Language; Langue des Signes Burundaise", + "lsc": "Albarradas Sign Language; Lengua de señas Albarradas", + "lsd": "Lishana Deni", + "lse": "Lusengo", + "lsh": "Lish", + "lsi": "Lashi", + "lsl": "Latvian Sign Language", + "lsm": "Saamia; Olusamia", + "lsn": "Tibetan Sign Language", + "lso": "Laos Sign Language", + "lsp": "Panamanian Sign Language; Lengua de Señas Panameñas", + "lsr": "Aruop", + "lss": "Lasi", + "lst": "Trinidad and Tobago Sign Language", + "lsv": "Sivia Sign Language", + "lsw": "Seychelles Sign Language; Lalang Siny Seselwa; Langue des Signes Seychelloise", + "lsy": "Mauritian Sign Language", + "lt": "Lithuanian", + "ltc": "Late Middle Chinese", + "ltg": "Latgalian", + "lth": "Thur", + "lti": "Leti (Indonesia)", + "ltn": "Latundê", + "lto": "Tsotso; Olutsotso", + "lts": "Tachoni; Lutachoni", + "ltu": "Latu", + "lu": "Luba-Katanga", + "lua": "Luba-Lulua", + "luc": "Aringa", + "lud": "Ludian", + "lue": "Luvale", + "luf": "Laua", + "lui": "Luiseno", + "luj": "Luna", + "luk": "Lunanakha", + "lul": "Olu'bo", + "lum": "Luimbi", + "lun": "Lunda", + "luo": "Luo (Kenya and Tanzania); Dholuo", + "lup": "Lumbu", + "luq": "Lucumi", + "lur": "Laura", + "lus": "Lushai", + "lut": "Lushootseed", + "luu": "Lumba-Yakkha", + "luv": "Luwati", + "luw": "Luo (Cameroon)", + "luy": "Luyia; Oluluyia", + "luz": "Southern Luri", + "lv": "Latvian", + "lva": "Maku'a", + "lvi": "Lavi", + "lvk": "Lavukaleve", + "lvs": "Standard Latvian", + "lvu": "Levuka", + "lwa": "Lwalu", + "lwe": "Lewo Eleng", + "lwg": "Wanga; Oluwanga", + "lwh": "White Lachi", + "lwl": "Eastern Lawa", + "lwm": "Laomian", + "lwo": "Luwo", + "lws": "Malawian Sign Language", + "lwt": "Lewotobi", + "lwu": "Lawu", + "lww": "Lewo", + "lxm": "Lakurumau", + "lya": "Layakha", + "lyg": "Lyngngam", + "lyn": "Luyana", + "lzh": "Literary Chinese", + "lzl": "Litzlitz", + "lzn": "Leinong Naga", + "lzz": "Laz", + "maa": "San Jerónimo Tecóatl Mazatec", + "mab": "Yutanduchi Mixtec", + "mad": "Madurese", + "mae": "Bo-Rukul", + "maf": "Mafa", + "mag": "Magahi", + "mai": "Maithili", + "maj": "Jalapa De Díaz Mazatec", + "mak": "Makasar", + "mam": "Mam", + "man": "Mandingo; Manding", + "map": "Austronesian languages", + "maq": "Chiquihuitlán Mazatec", + "mas": "Masai", + "mat": "San Francisco Matlatzinca", + "mau": "Huautla Mazatec", + "mav": "Sateré-Mawé", + "maw": "Mampruli", + "max": "North Moluccan Malay", + "maz": "Central Mazahua", + "mba": "Higaonon", + "mbb": "Western Bukidnon Manobo", + "mbc": "Macushi", + "mbd": "Dibabawon Manobo", + "mbe": "Molale", + "mbf": "Baba Malay", + "mbh": "Mangseng", + "mbi": "Ilianen Manobo", + "mbj": "Nadëb", + "mbk": "Malol", + "mbl": "Maxakalí", + "mbm": "Ombamba", + "mbn": "Macaguán", + "mbo": "Mbo (Cameroon)", + "mbp": "Malayo", + "mbq": "Maisin", + "mbr": "Nukak Makú", + "mbs": "Sarangani Manobo", + "mbt": "Matigsalug Manobo", + "mbu": "Mbula-Bwazza", + "mbv": "Mbulungish", + "mbw": "Maring", + "mbx": "Mari (East Sepik Province)", + "mby": "Memoni", + "mbz": "Amoltepec Mixtec", + "mca": "Maca", + "mcb": "Machiguenga", + "mcc": "Bitur", + "mcd": "Sharanahua", + "mce": "Itundujia Mixtec", + "mcf": "Matsés", + "mcg": "Mapoyo", + "mch": "Maquiritari", + "mci": "Mese", + "mcj": "Mvanip", + "mck": "Mbunda", + "mcl": "Macaguaje", + "mcm": "Malaccan Creole Portuguese", + "mcn": "Masana", + "mco": "Coatlán Mixe", + "mcp": "Makaa", + "mcq": "Ese", + "mcr": "Menya", + "mcs": "Mambai", + "mct": "Mengisa", + "mcu": "Cameroon Mambila", + "mcv": "Minanibai", + "mcw": "Mawa (Chad)", + "mcx": "Mpiemo", + "mcy": "South Watut", + "mcz": "Mawan", + "mda": "Mada (Nigeria)", + "mdb": "Morigi", + "mdc": "Male (Papua New Guinea)", + "mdd": "Mbum", + "mde": "Maba (Chad)", + "mdf": "Moksha", + "mdg": "Massalat", + "mdh": "Maguindanaon", + "mdi": "Mamvu", + "mdj": "Mangbetu", + "mdk": "Mangbutu", + "mdl": "Maltese Sign Language", + "mdm": "Mayogo", + "mdn": "Mbati", + "mdp": "Mbala", + "mdq": "Mbole", + "mdr": "Mandar", + "mds": "Maria (Papua New Guinea)", + "mdt": "Mbere", + "mdu": "Mboko", + "mdv": "Santa Lucía Monteverde Mixtec", + "mdw": "Mbosi", + "mdx": "Dizin", + "mdy": "Male (Ethiopia)", + "mdz": "Suruí Do Pará", + "mea": "Menka", + "meb": "Ikobi", + "mec": "Marra", + "med": "Melpa", + "mee": "Mengen", + "mef": "Megam", + "meh": "Southwestern Tlaxiaco Mixtec", + "mei": "Midob", + "mej": "Meyah", + "mek": "Mekeo", + "mel": "Central Melanau", + "mem": "Mangala", + "men": "Mende (Sierra Leone)", + "meo": "Kedah Malay", + "mep": "Miriwoong", + "meq": "Merey", + "mer": "Meru", + "mes": "Masmaje", + "met": "Mato", + "meu": "Motu", + "mev": "Mano", + "mew": "Maaka", + "mey": "Hassaniyya", + "mez": "Menominee", + "mfa": "Pattani Malay", + "mfb": "Bangka", + "mfc": "Mba", + "mfd": "Mendankwe-Nkwen", + "mfe": "Morisyen", + "mff": "Naki", + "mfg": "Mogofin", + "mfh": "Matal", + "mfi": "Wandala", + "mfj": "Mefele", + "mfk": "North Mofu", + "mfl": "Putai", + "mfm": "Marghi South", + "mfn": "Cross River Mbembe", + "mfo": "Mbe", + "mfp": "Makassar Malay", + "mfq": "Moba", + "mfr": "Marrithiyel", + "mfs": "Mexican Sign Language", + "mft": "Mokerang", + "mfu": "Mbwela", + "mfv": "Mandjak", + "mfw": "Mulaha", + "mfx": "Melo", + "mfy": "Mayo", + "mfz": "Mabaan", + "mg": "Malagasy", + "mga": "Middle Irish (900-1200)", + "mgb": "Mararit", + "mgc": "Morokodo", + "mgd": "Moru", + "mge": "Mango", + "mgf": "Maklew", + "mgg": "Mpumpong", + "mgh": "Makhuwa-Meetto", + "mgi": "Lijili", + "mgj": "Abureni", + "mgk": "Mawes", + "mgl": "Maleu-Kilenge", + "mgm": "Mambae", + "mgn": "Mbangi", + "mgo": "Meta'", + "mgp": "Eastern Magar", + "mgq": "Malila", + "mgr": "Mambwe-Lungu", + "mgs": "Manda (Tanzania)", + "mgt": "Mongol", + "mgu": "Mailu", + "mgv": "Matengo", + "mgw": "Matumbi", + "mgy": "Mbunga", + "mgz": "Mbugwe", + "mh": "Marshallese", + "mha": "Manda (India)", + "mhb": "Mahongwe", + "mhc": "Mocho", + "mhd": "Mbugu", + "mhe": "Besisi; Mah Meri", + "mhf": "Mamaa", + "mhg": "Margu", + "mhi": "Ma'di", + "mhj": "Mogholi", + "mhk": "Mungaka", + "mhl": "Mauwake", + "mhm": "Makhuwa-Moniga", + "mhn": "Mócheno", + "mho": "Mashi (Zambia)", + "mhp": "Balinese Malay", + "mhq": "Mandan", + "mhr": "Eastern Mari", + "mhs": "Buru (Indonesia)", + "mht": "Mandahuaca", + "mhu": "Digaro-Mishmi; Darang Deng", + "mhw": "Mbukushu", + "mhx": "Maru; Lhaovo", + "mhy": "Ma'anyan", + "mhz": "Mor (Mor Islands)", + "mi": "Maori", + "mia": "Miami", + "mib": "Atatláhuca Mixtec", + "mic": "Mi'kmaq; Micmac", + "mid": "Mandaic", + "mie": "Ocotepec Mixtec", + "mif": "Mofu-Gudur", + "mig": "San Miguel El Grande Mixtec", + "mih": "Chayuco Mixtec", + "mii": "Chigmecatitlán Mixtec", + "mij": "Abar; Mungbam", + "mik": "Mikasuki", + "mil": "Peñoles Mixtec", + "mim": "Alacatlatzala Mixtec", + "min": "Minangkabau", + "mio": "Pinotepa Nacional Mixtec", + "mip": "Apasco-Apoala Mixtec", + "miq": "Mískito", + "mir": "Isthmus Mixe", + "mit": "Southern Puebla Mixtec", + "miu": "Cacaloxtepec Mixtec", + "miw": "Akoye", + "mix": "Mixtepec Mixtec", + "miy": "Ayutla Mixtec", + "miz": "Coatzospan Mixtec", + "mjb": "Makalero", + "mjc": "San Juan Colorado Mixtec", + "mjd": "Northwest Maidu", + "mje": "Muskum", + "mjg": "Tu", + "mjh": "Mwera (Nyasa)", + "mji": "Kim Mun", + "mjj": "Mawak", + "mjk": "Matukar", + "mjl": "Mandeali", + "mjm": "Medebur", + "mjn": "Ma (Papua New Guinea)", + "mjo": "Malankuravan", + "mjp": "Malapandaram", + "mjq": "Malaryan", + "mjr": "Malavedan", + "mjs": "Miship", + "mjt": "Sauria Paharia", + "mju": "Manna-Dora", + "mjv": "Mannan", + "mjw": "Karbi", + "mjx": "Mahali", + "mjy": "Mahican", + "mjz": "Majhi", + "mk": "Macedonian", + "mka": "Mbre", + "mkb": "Mal Paharia", + "mkc": "Siliput", + "mke": "Mawchi", + "mkf": "Miya", + "mkg": "Mak (China)", + "mkh": "Mon-Khmer languages", + "mki": "Dhatki", + "mkj": "Mokilese", + "mkk": "Byep", + "mkl": "Mokole", + "mkm": "Moklen", + "mkn": "Kupang Malay", + "mko": "Mingang Doso", + "mkp": "Moikodi", + "mkq": "Bay Miwok", + "mkr": "Malas", + "mks": "Silacayoapan Mixtec", + "mkt": "Vamale", + "mku": "Konyanka Maninka", + "mkv": "Mafea", + "mkw": "Kituba (Congo)", + "mkx": "Kinamiging Manobo", + "mky": "East Makian", + "mkz": "Makasae", + "ml": "Malayalam", + "mla": "Malo", + "mlb": "Mbule", + "mlc": "Cao Lan", + "mle": "Manambu", + "mlf": "Mal", + "mlh": "Mape", + "mli": "Malimpung", + "mlj": "Miltu", + "mlk": "Ilwana; Kiwilwana", + "mll": "Malua Bay", + "mlm": "Mulam", + "mln": "Malango", + "mlo": "Mlomp", + "mlp": "Bargam", + "mlq": "Western Maninkakan", + "mlr": "Vame", + "mls": "Masalit", + "mlu": "To'abaita", + "mlv": "Motlav; Mwotlap", + "mlw": "Moloko", + "mlx": "Malfaxal; Naha'ai", + "mlz": "Malaynon", + "mma": "Mama", + "mmb": "Momina", + "mmc": "Michoacán Mazahua", + "mmd": "Maonan", + "mme": "Mae", + "mmf": "Mundat", + "mmg": "North Ambrym", + "mmh": "Mehináku", + "mmi": "Musar", + "mmj": "Majhwar", + "mmk": "Mukha-Dora", + "mml": "Man Met", + "mmm": "Maii", + "mmn": "Mamanwa", + "mmo": "Mangga Buang", + "mmp": "Siawi", + "mmq": "Musak", + "mmr": "Western Xiangxi Miao", + "mmt": "Malalamai", + "mmu": "Mmaala", + "mmv": "Miriti", + "mmw": "Emae", + "mmx": "Madak", + "mmy": "Migaama", + "mmz": "Mabaale", + "mn": "Mongolian", + "mna": "Mbula", + "mnb": "Muna", + "mnc": "Manchu", + "mnd": "Mondé", + "mne": "Naba", + "mnf": "Mundani", + "mng": "Eastern Mnong", + "mnh": "Mono (Democratic Republic of Congo)", + "mni": "Manipuri", + "mnj": "Munji", + "mnk": "Mandinka", + "mnl": "Tiale", + "mnm": "Mapena", + "mnn": "Southern Mnong", + "mno": "Manobo languages", + "mnp": "Min Bei Chinese", + "mnq": "Minriq", + "mnr": "Mono (USA)", + "mns": "Mansi", + "mnu": "Mer", + "mnv": "Rennell-Bellona", + "mnw": "Mon", + "mnx": "Manikion", + "mny": "Manyawa", + "mnz": "Moni", + "moa": "Mwan", + "moc": "Mocoví", + "mod": "Mobilian", + "moe": "Innu; Montagnais", + "mog": "Mongondow", + "moh": "Mohawk", + "moi": "Mboi", + "moj": "Monzombo", + "mok": "Morori", + "mom": "Mangue", + "moo": "Monom", + "mop": "Mopán Maya", + "moq": "Mor (Bomberai Peninsula)", + "mor": "Moro", + "mos": "Mossi", + "mot": "Barí", + "mou": "Mogum", + "mov": "Mohave", + "mow": "Moi (Congo)", + "mox": "Molima", + "moy": "Shekkacho", + "moz": "Mukulu; Gergiko", + "mpa": "Mpoto", + "mpb": "Malak Malak; Mullukmulluk", + "mpc": "Mangarrayi", + "mpd": "Machinere", + "mpe": "Majang", + "mpg": "Marba", + "mph": "Maung", + "mpi": "Mpade", + "mpj": "Martu Wangka; Wangkajunga", + "mpk": "Mbara (Chad)", + "mpl": "Middle Watut", + "mpm": "Yosondúa Mixtec", + "mpn": "Mindiri", + "mpo": "Miu", + "mpp": "Migabac", + "mpq": "Matís", + "mpr": "Vangunu", + "mps": "Dadibi", + "mpt": "Mian", + "mpu": "Makuráp", + "mpv": "Mungkip", + "mpw": "Mapidian", + "mpx": "Misima-Panaeati", + "mpy": "Mapia", + "mpz": "Mpi", + "mqa": "Maba (Indonesia)", + "mqb": "Mbuko", + "mqc": "Mangole", + "mqe": "Matepi", + "mqf": "Momuna", + "mqg": "Kota Bangun Kutai Malay", + "mqh": "Tlazoyaltepec Mixtec", + "mqi": "Mariri", + "mqj": "Mamasa", + "mqk": "Rajah Kabunsuwan Manobo", + "mql": "Mbelime", + "mqm": "South Marquesan", + "mqn": "Moronene", + "mqo": "Modole", + "mqp": "Manipa", + "mqq": "Minokok", + "mqr": "Mander", + "mqs": "West Makian", + "mqt": "Mok", + "mqu": "Mandari", + "mqv": "Mosimo", + "mqw": "Murupi", + "mqx": "Mamuju", + "mqy": "Manggarai", + "mqz": "Pano", + "mr": "Marathi", + "mra": "Mlabri", + "mrb": "Marino", + "mrc": "Maricopa", + "mrd": "Western Magar", + "mre": "Martha's Vineyard Sign Language", + "mrf": "Elseng", + "mrg": "Mising", + "mrh": "Mara Chin", + "mrj": "Western Mari", + "mrk": "Hmwaveke", + "mrl": "Mortlockese", + "mrm": "Merlav; Mwerlap", + "mrn": "Cheke Holo", + "mro": "Mru", + "mrp": "Morouas", + "mrq": "North Marquesan", + "mrr": "Maria (India)", + "mrs": "Maragus", + "mrt": "Marghi Central", + "mru": "Mono (Cameroon)", + "mrv": "Mangareva", + "mrw": "Maranao", + "mrx": "Maremgi; Dineor", + "mry": "Mandaya", + "mrz": "Marind", + "ms": "Malay (macrolanguage)", + "msb": "Masbatenyo", + "msc": "Sankaran Maninka", + "msd": "Yucatec Maya Sign Language", + "mse": "Musey", + "msf": "Mekwei", + "msg": "Moraid", + "msh": "Masikoro Malagasy", + "msi": "Sabah Malay", + "msj": "Ma (Democratic Republic of Congo)", + "msk": "Mansaka", + "msl": "Molof; Poule", + "msm": "Agusan Manobo", + "msn": "Vurës", + "mso": "Mombum", + "msp": "Maritsauá", + "msq": "Caac", + "msr": "Mongolian Sign Language", + "mss": "West Masela", + "msu": "Musom", + "msv": "Maslam", + "msw": "Mansoanka", + "msx": "Moresada", + "msy": "Aruamu", + "msz": "Momare", + "mt": "Maltese", + "mta": "Cotabato Manobo", + "mtb": "Anyin Morofo", + "mtc": "Munit", + "mtd": "Mualang", + "mte": "Mono (Solomon Islands)", + "mtf": "Murik (Papua New Guinea)", + "mtg": "Una", + "mth": "Munggui", + "mti": "Maiwa (Papua New Guinea)", + "mtj": "Moskona", + "mtk": "Mbe'", + "mtl": "Montol", + "mtm": "Mator", + "mtn": "Matagalpa", + "mto": "Totontepec Mixe", + "mtp": "Wichí Lhamtés Nocten", + "mtq": "Muong", + "mtr": "Mewari", + "mts": "Yora", + "mtt": "Mota", + "mtu": "Tututepec Mixtec", + "mtv": "Asaro'o", + "mtw": "Southern Binukidnon", + "mtx": "Tidaá Mixtec", + "mty": "Nabi", + "mua": "Mundang", + "mub": "Mubi", + "muc": "Ajumbu", + "mud": "Mednyj Aleut", + "mue": "Media Lengua", + "mug": "Musgu", + "muh": "Mündü", + "mui": "Musi", + "muj": "Mabire", + "muk": "Mugom", + "mum": "Maiwala", + "mun": "Munda languages", + "muo": "Nyong", + "mup": "Malvi", + "muq": "Eastern Xiangxi Miao", + "mur": "Murle", + "mus": "Creek", + "mut": "Western Muria", + "muu": "Yaaku", + "muv": "Muthuvan", + "mux": "Bo-Ung", + "muy": "Muyang", + "muz": "Mursi", + "mva": "Manam", + "mvb": "Mattole", + "mvd": "Mamboru", + "mve": "Marwari (Pakistan)", + "mvf": "Peripheral Mongolian", + "mvg": "Yucuañe Mixtec", + "mvh": "Mulgi", + "mvi": "Miyako", + "mvk": "Mekmek", + "mvl": "Mbara (Australia)", + "mvn": "Minaveha", + "mvo": "Marovo", + "mvp": "Duri", + "mvq": "Moere", + "mvr": "Marau", + "mvs": "Massep", + "mvt": "Mpotovoro", + "mvu": "Marfa", + "mvv": "Tagal Murut", + "mvw": "Machinga", + "mvx": "Meoswar", + "mvy": "Indus Kohistani", + "mvz": "Mesqan", + "mwa": "Mwatebu", + "mwb": "Juwal", + "mwc": "Are", + "mwe": "Mwera (Chimwera)", + "mwf": "Murrinh-Patha", + "mwg": "Aiklep", + "mwh": "Mouk-Aria", + "mwi": "Labo; Ninde", + "mwk": "Kita Maninkakan", + "mwl": "Mirandese", + "mwm": "Sar", + "mwn": "Nyamwanga", + "mwo": "Central Maewo", + "mwp": "Kala Lagaw Ya", + "mwq": "Mün Chin", + "mwr": "Marwari", + "mws": "Mwimbi-Muthambi", + "mwt": "Moken", + "mwu": "Mittu", + "mwv": "Mentawai", + "mww": "Hmong Daw", + "mwz": "Moingi", + "mxa": "Northwest Oaxaca Mixtec", + "mxb": "Tezoatlán Mixtec", + "mxc": "Manyika", + "mxd": "Modang", + "mxe": "Mele-Fila", + "mxf": "Malgbe", + "mxg": "Mbangala", + "mxh": "Mvuba", + "mxi": "Mozarabic", + "mxj": "Miju-Mishmi; Geman Deng", + "mxk": "Monumbo", + "mxl": "Maxi Gbe", + "mxm": "Meramera", + "mxn": "Moi (Indonesia)", + "mxo": "Mbowe", + "mxp": "Tlahuitoltepec Mixe", + "mxq": "Juquila Mixe", + "mxr": "Murik (Malaysia)", + "mxs": "Huitepec Mixtec", + "mxt": "Jamiltepec Mixtec", + "mxu": "Mada (Cameroon)", + "mxv": "Metlatónoc Mixtec", + "mxw": "Namo", + "mxx": "Mahou; Mawukakan", + "mxy": "Southeastern Nochixtlán Mixtec", + "mxz": "Central Masela", + "my": "Burmese", + "myb": "Mbay", + "myc": "Mayeka", + "mye": "Myene", + "myf": "Bambassi", + "myg": "Manta", + "myh": "Makah", + "myj": "Mangayat", + "myk": "Mamara Senoufo", + "myl": "Moma", + "mym": "Me'en", + "myn": "Mayan languages", + "myo": "Anfillo", + "myp": "Pirahã", + "myr": "Muniche", + "mys": "Mesmes", + "myu": "Mundurukú", + "myv": "Erzya", + "myw": "Muyuw", + "myx": "Masaaba", + "myy": "Macuna", + "myz": "Classical Mandaic", + "mza": "Santa María Zacatepec Mixtec", + "mzb": "Tumzabt", + "mzc": "Madagascar Sign Language", + "mzd": "Malimba", + "mze": "Morawa", + "mzg": "Monastic Sign Language", + "mzh": "Wichí Lhamtés Güisnay", + "mzi": "Ixcatlán Mazatec", + "mzj": "Manya", + "mzk": "Nigeria Mambila", + "mzl": "Mazatlán Mixe", + "mzm": "Mumuye", + "mzn": "Mazanderani", + "mzo": "Matipuhy", + "mzp": "Movima", + "mzq": "Mori Atas", + "mzr": "Marúbo", + "mzs": "Macanese", + "mzt": "Mintil", + "mzu": "Inapang", + "mzv": "Manza", + "mzw": "Deg", + "mzx": "Mawayana", + "mzy": "Mozambican Sign Language", + "mzz": "Maiadomu", + "na": "Nauru", + "naa": "Namla", + "nab": "Southern Nambikuára", + "nac": "Narak", + "nae": "Naka'ela", + "naf": "Nabak", + "nag": "Naga Pidgin", + "nah": "Nahuatl languages", + "nai": "North American Indian languages", + "naj": "Nalu", + "nak": "Nakanai", + "nal": "Nalik", + "nam": "Ngan'gityemerri", + "nan": "Min Nan Chinese", + "nao": "Naaba", + "nap": "Neapolitan", + "naq": "Khoekhoe; Nama (Namibia)", + "nar": "Iguta", + "nas": "Naasioi", + "nat": "Ca̱hungwa̱rya̱; Hungworo", + "naw": "Nawuri", + "nax": "Nakwi", + "nay": "Ngarrindjeri", + "naz": "Coatepec Nahuatl", + "nb": "Norwegian Bokmål", + "nba": "Nyemba", + "nbb": "Ndoe", + "nbc": "Chang Naga", + "nbd": "Ngbinda", + "nbe": "Konyak Naga", + "nbg": "Nagarchal", + "nbh": "Ngamo", + "nbi": "Mao Naga", + "nbj": "Ngarinyman", + "nbk": "Nake", + "nbm": "Ngbaka Ma'bo", + "nbn": "Kuri", + "nbo": "Nkukoli", + "nbp": "Nnam", + "nbq": "Nggem", + "nbr": "Numana", + "nbs": "Namibian Sign Language", + "nbt": "Na", + "nbu": "Rongmei Naga", + "nbv": "Ngamambo", + "nbw": "Southern Ngbandi", + "nby": "Ningera", + "nca": "Iyo", + "ncb": "Central Nicobarese", + "ncc": "Ponam", + "ncd": "Nachering", + "nce": "Yale", + "ncf": "Notsi", + "ncg": "Nisga'a", + "nch": "Central Huasteca Nahuatl", + "nci": "Classical Nahuatl", + "ncj": "Northern Puebla Nahuatl", + "nck": "Na-kara", + "ncl": "Michoacán Nahuatl", + "ncm": "Nambo", + "ncn": "Nauna", + "nco": "Sibe", + "ncq": "Northern Katang", + "ncr": "Ncane", + "ncs": "Nicaraguan Sign Language", + "nct": "Chothe Naga", + "ncu": "Chumburung", + "ncx": "Central Puebla Nahuatl", + "ncz": "Natchez", + "nd": "North Ndebele", + "nda": "Ndasa", + "ndb": "Kenswei Nsei", + "ndc": "Ndau", + "ndd": "Nde-Nsele-Nta", + "ndf": "Nadruvian", + "ndg": "Ndengereko", + "ndh": "Ndali", + "ndi": "Samba Leko", + "ndj": "Ndamba", + "ndk": "Ndaka", + "ndl": "Ndolo", + "ndm": "Ndam", + "ndn": "Ngundi", + "ndp": "Ndo", + "ndq": "Ndombe", + "ndr": "Ndoola", + "nds": "Low German; Low Saxon", + "ndt": "Ndunga", + "ndu": "Dugun", + "ndv": "Ndut", + "ndw": "Ndobo", + "ndx": "Nduga", + "ndy": "Lutos", + "ndz": "Ndogo", + "ne": "Nepali (macrolanguage)", + "nea": "Eastern Ngad'a", + "neb": "Toura (Côte d'Ivoire)", + "nec": "Nedebang", + "ned": "Nde-Gbite", + "nee": "Nêlêmwa-Nixumwak", + "nef": "Nefamese", + "neg": "Negidal", + "neh": "Nyenkha", + "nei": "Neo-Hittite", + "nej": "Neko", + "nek": "Neku", + "nem": "Nemi", + "nen": "Nengone", + "neo": "Ná-Meo", + "neq": "North Central Mixe", + "ner": "Yahadian", + "nes": "Bhoti Kinnauri", + "net": "Nete", + "neu": "Neo", + "nev": "Nyaheun", + "new": "Newari; Nepal Bhasa", + "nex": "Neme", + "ney": "Neyo", + "nez": "Nez Perce", + "nfa": "Dhao", + "nfd": "Ahwai", + "nfl": "Ayiwo; Äiwoo", + "nfr": "Nafaanra", + "nfu": "Mfumte", + "ng": "Ndonga", + "nga": "Ngbaka", + "ngb": "Northern Ngbandi", + "ngc": "Ngombe (Democratic Republic of Congo)", + "ngd": "Ngando (Central African Republic)", + "nge": "Ngemba", + "ngf": "Trans-New Guinea languages", + "ngg": "Ngbaka Manza", + "ngh": "Nǁng", + "ngi": "Ngizim", + "ngj": "Ngie", + "ngk": "Dalabon", + "ngl": "Lomwe", + "ngm": "Ngatik Men's Creole", + "ngn": "Ngwo", + "ngp": "Ngulu", + "ngq": "Ngurimi; Ngoreme", + "ngr": "Engdewu", + "ngs": "Gvoko", + "ngt": "Kriang; Ngeq", + "ngu": "Guerrero Nahuatl", + "ngv": "Nagumi", + "ngw": "Ngwaba", + "ngx": "Nggwahyi", + "ngy": "Tibea", + "ngz": "Ngungwel", + "nha": "Nhanda", + "nhb": "Beng", + "nhc": "Tabasco Nahuatl", + "nhd": "Chiripá; Ava Guaraní", + "nhe": "Eastern Huasteca Nahuatl", + "nhf": "Nhuwala", + "nhg": "Tetelcingo Nahuatl", + "nhh": "Nahari", + "nhi": "Zacatlán-Ahuacatlán-Tepetzintla Nahuatl", + "nhk": "Isthmus-Cosoleacaque Nahuatl", + "nhm": "Morelos Nahuatl", + "nhn": "Central Nahuatl", + "nho": "Takuu", + "nhp": "Isthmus-Pajapan Nahuatl", + "nhq": "Huaxcaleca Nahuatl", + "nhr": "Naro", + "nht": "Ometepec Nahuatl", + "nhu": "Noone", + "nhv": "Temascaltepec Nahuatl", + "nhw": "Western Huasteca Nahuatl", + "nhx": "Isthmus-Mecayapan Nahuatl", + "nhy": "Northern Oaxaca Nahuatl", + "nhz": "Santa María La Alta Nahuatl", + "nia": "Nias", + "nib": "Nakame", + "nic": "Niger-Kordofanian languages", + "nid": "Ngandi", + "nie": "Niellim", + "nif": "Nek", + "nig": "Ngalakgan", + "nih": "Nyiha (Tanzania)", + "nii": "Nii", + "nij": "Ngaju", + "nik": "Southern Nicobarese", + "nil": "Nila", + "nim": "Nilamba", + "nin": "Ninzo", + "nio": "Nganasan", + "niq": "Nandi", + "nir": "Nimboran", + "nis": "Nimi", + "nit": "Southeastern Kolami", + "niu": "Niuean", + "niv": "Gilyak", + "niw": "Nimo", + "nix": "Hema", + "niy": "Ngiti", + "niz": "Ningil", + "nja": "Nzanyi", + "njb": "Nocte Naga", + "njd": "Ndonde Hamba", + "njh": "Lotha Naga", + "nji": "Gudanji", + "njj": "Njen", + "njl": "Njalgulgule", + "njm": "Angami Naga", + "njn": "Liangmai Naga", + "njo": "Ao Naga", + "njr": "Njerep", + "njs": "Nisa", + "njt": "Ndyuka-Trio Pidgin", + "nju": "Ngadjunmaya", + "njx": "Kunyi", + "njy": "Njyem", + "njz": "Nyishi", + "nka": "Nkoya", + "nkb": "Khoibu Naga", + "nkc": "Nkongho", + "nkd": "Koireng", + "nke": "Duke", + "nkf": "Inpui Naga", + "nkg": "Nekgini", + "nkh": "Khezha Naga", + "nki": "Thangal Naga", + "nkj": "Nakai", + "nkk": "Nokuku", + "nkm": "Namat", + "nkn": "Nkangala", + "nko": "Nkonya", + "nkp": "Niuatoputapu", + "nkq": "Nkami", + "nkr": "Nukuoro", + "nks": "North Asmat", + "nkt": "Nyika (Tanzania)", + "nku": "Bouna Kulango", + "nkv": "Nyika (Malawi and Zambia)", + "nkw": "Nkutu", + "nkx": "Nkoroo", + "nkz": "Nkari", + "nl": "Dutch; Flemish", + "nla": "Ngombale", + "nlc": "Nalca", + "nle": "East Nyala", + "nlg": "Gela", + "nli": "Grangali", + "nlj": "Nyali", + "nlk": "Ninia Yali", + "nll": "Nihali", + "nlm": "Mankiyali", + "nlo": "Ngul", + "nlq": "Lao Naga", + "nlu": "Nchumbulu", + "nlv": "Orizaba Nahuatl", + "nlw": "Walangama", + "nlx": "Nahali", + "nly": "Nyamal", + "nlz": "Nalögo", + "nma": "Maram Naga", + "nmb": "Big Nambas; V'ënen Taut", + "nmc": "Ngam", + "nmd": "Ndumu", + "nme": "Mzieme Naga", + "nmf": "Tangkhul Naga (India)", + "nmg": "Kwasio", + "nmh": "Monsang Naga", + "nmi": "Nyam", + "nmj": "Ngombe (Central African Republic)", + "nmk": "Namakura", + "nml": "Ndemli", + "nmm": "Manangba", + "nmn": "ǃXóõ", + "nmo": "Moyon Naga", + "nmp": "Nimanbur", + "nmq": "Nambya", + "nmr": "Nimbari", + "nms": "Letemboi", + "nmt": "Namonuito", + "nmu": "Northeast Maidu", + "nmv": "Ngamini", + "nmw": "Nimoa; Rifao", + "nmx": "Nama (Papua New Guinea)", + "nmy": "Namuyi", + "nmz": "Nawdm", + "nn": "Norwegian Nynorsk", + "nna": "Nyangumarta", + "nnb": "Nande", + "nnc": "Nancere", + "nnd": "West Ambae", + "nne": "Ngandyera", + "nnf": "Ngaing", + "nng": "Maring Naga", + "nnh": "Ngiemboon", + "nni": "North Nuaulu", + "nnj": "Nyangatom", + "nnk": "Nankina", + "nnl": "Northern Rengma Naga", + "nnm": "Namia", + "nnn": "Ngete", + "nnp": "Wancho Naga", + "nnq": "Ngindo", + "nnr": "Narungga", + "nnt": "Nanticoke", + "nnu": "Dwang", + "nnv": "Nugunu (Australia)", + "nnw": "Southern Nuni", + "nny": "Nyangga", + "nnz": "Nda'nda'", + "no": "Norwegian", + "noa": "Woun Meu", + "noc": "Nuk", + "nod": "Northern Thai", + "noe": "Nimadi", + "nof": "Nomane", + "nog": "Nogai", + "noh": "Nomu", + "noi": "Noiri", + "noj": "Nonuya", + "nok": "Nooksack", + "nol": "Nomlaki", + "nom": "Nocamán", + "non": "Old Norse", + "nop": "Numanggang", + "noq": "Ngongo", + "nos": "Eastern Nisu", + "not": "Nomatsiguenga", + "nou": "Ewage-Notu", + "nov": "Novial", + "now": "Nyambo", + "noy": "Noy", + "noz": "Nayi", + "npa": "Nar Phu", + "npb": "Nupbikha", + "npg": "Ponyo-Gongwang Naga", + "nph": "Phom Naga", + "npi": "Nepali (individual language)", + "npl": "Southeastern Puebla Nahuatl", + "npn": "Mondropolon", + "npo": "Pochuri Naga", + "nps": "Nipsan", + "npu": "Puimei Naga", + "npx": "Noipx", + "npy": "Napu", + "nqg": "Southern Nago", + "nqk": "Kura Ede Nago", + "nql": "Ngendelengo", + "nqm": "Ndom", + "nqn": "Nen", + "nqo": "N'Ko; N’Ko", + "nqq": "Kyan-Karyaw Naga", + "nqt": "Nteng", + "nqy": "Akyaung Ari Naga", + "nr": "South Ndebele", + "nra": "Ngom", + "nrb": "Nara", + "nrc": "Noric", + "nre": "Southern Rengma Naga", + "nrf": "Jèrriais; Guernésiais", + "nrg": "Narango", + "nri": "Chokri Naga", + "nrk": "Ngarla", + "nrl": "Ngarluma", + "nrm": "Narom", + "nrn": "Norn", + "nrp": "North Picene", + "nrr": "Norra; Nora", + "nrt": "Northern Kalapuya", + "nru": "Narua", + "nrx": "Ngurmbur", + "nrz": "Lala", + "nsa": "Sangtam Naga", + "nsb": "Lower Nossob", + "nsc": "Nshi", + "nsd": "Southern Nisu", + "nse": "Nsenga", + "nsf": "Northwestern Nisu", + "nsg": "Ngasa", + "nsh": "Ngoshie", + "nsi": "Nigerian Sign Language", + "nsk": "Naskapi", + "nsl": "Norwegian Sign Language", + "nsm": "Sumi Naga", + "nsn": "Nehan", + "nso": "Pedi; Northern Sotho; Sepedi", + "nsp": "Nepalese Sign Language", + "nsq": "Northern Sierra Miwok", + "nsr": "Maritime Sign Language", + "nss": "Nali", + "nst": "Tase Naga", + "nsu": "Sierra Negra Nahuatl", + "nsv": "Southwestern Nisu", + "nsw": "Navut", + "nsx": "Nsongo", + "nsy": "Nasal", + "nsz": "Nisenan", + "ntd": "Northern Tidung", + "nte": "Nathembo", + "ntg": "Ngantangarra", + "nti": "Natioro", + "ntj": "Ngaanyatjarra", + "ntk": "Ikoma-Nata-Isenye", + "ntm": "Nateni", + "nto": "Ntomba", + "ntp": "Northern Tepehuan", + "ntr": "Delo", + "ntu": "Natügu", + "ntw": "Nottoway", + "ntx": "Tangkhul Naga (Myanmar)", + "nty": "Mantsi", + "ntz": "Natanzi", + "nua": "Yuanga", + "nub": "Nubian languages", + "nuc": "Nukuini", + "nud": "Ngala", + "nue": "Ngundu", + "nuf": "Nusu", + "nug": "Nungali", + "nuh": "Ndunda", + "nui": "Ngumbi", + "nuj": "Nyole", + "nuk": "Nuu-chah-nulth; Nuuchahnulth", + "nul": "Nusa Laut", + "num": "Niuafo'ou", + "nun": "Anong", + "nuo": "Nguôn", + "nup": "Nupe-Nupe-Tako", + "nuq": "Nukumanu", + "nur": "Nukuria", + "nus": "Nuer", + "nut": "Nung (Viet Nam)", + "nuu": "Ngbundu", + "nuv": "Northern Nuni", + "nuw": "Nguluwan", + "nux": "Mehek", + "nuy": "Nunggubuyu", + "nuz": "Tlamacazapa Nahuatl", + "nv": "Navajo; Navaho", + "nvh": "Nasarian", + "nvm": "Namiae", + "nvo": "Nyokon", + "nwa": "Nawathinehena", + "nwb": "Nyabwa", + "nwc": "Classical Newari; Classical Nepal Bhasa; Old Newari", + "nwe": "Ngwe", + "nwg": "Ngayawung", + "nwi": "Southwest Tanna", + "nwm": "Nyamusa-Molo", + "nwo": "Nauo", + "nwr": "Nawaru", + "nww": "Ndwewe", + "nwx": "Middle Newar", + "nwy": "Nottoway-Meherrin", + "nxa": "Nauete", + "nxd": "Ngando (Democratic Republic of Congo)", + "nxe": "Nage", + "nxg": "Ngad'a", + "nxi": "Nindi", + "nxk": "Koki Naga", + "nxl": "South Nuaulu", + "nxm": "Numidian", + "nxn": "Ngawun", + "nxo": "Ndambomo", + "nxq": "Naxi", + "nxr": "Ninggerum", + "nxx": "Nafri", + "ny": "Nyanja; Chewa; Chichewa", + "nyb": "Nyangbo", + "nyc": "Nyanga-li", + "nyd": "Nyore; Olunyole", + "nye": "Nyengo", + "nyf": "Giryama; Kigiryama", + "nyg": "Nyindu", + "nyh": "Nyikina", + "nyi": "Ama (Sudan)", + "nyj": "Nyanga", + "nyk": "Nyaneka", + "nyl": "Nyeu", + "nym": "Nyamwezi", + "nyn": "Nyankole", + "nyo": "Nyoro", + "nyp": "Nyang'i", + "nyq": "Nayini", + "nyr": "Nyiha (Malawi)", + "nys": "Nyungar", + "nyt": "Nyawaygi", + "nyu": "Nyungwe", + "nyv": "Nyulnyul", + "nyw": "Nyaw", + "nyx": "Nganyaywana", + "nyy": "Nyakyusa-Ngonde", + "nza": "Tigon Mbembe", + "nzb": "Njebi", + "nzd": "Nzadi", + "nzi": "Nzima", + "nzk": "Nzakara", + "nzm": "Zeme Naga", + "nzs": "New Zealand Sign Language", + "nzu": "Teke-Nzikou", + "nzy": "Nzakambay", + "nzz": "Nanga Dama Dogon", + "oaa": "Orok", + "oac": "Oroch", + "oar": "Old Aramaic (up to 700 BCE); Ancient Aramaic (up to 700 BCE)", + "oav": "Old Avar", + "obi": "Obispeño", + "obk": "Southern Bontok", + "obl": "Oblo", + "obm": "Moabite", + "obo": "Obo Manobo", + "obr": "Old Burmese", + "obt": "Old Breton", + "obu": "Obulom", + "oc": "Occitan (post 1500)", + "oca": "Ocaina", + "och": "Old Chinese", + "ocm": "Old Cham", + "oco": "Old Cornish", + "ocu": "Atzingo Matlatzinca", + "oda": "Odut", + "odk": "Od", + "odt": "Old Dutch", + "odu": "Odual", + "ofo": "Ofo", + "ofs": "Old Frisian", + "ofu": "Efutop", + "ogb": "Ogbia", + "ogc": "Ogbah", + "oge": "Old Georgian", + "ogg": "Ogbogolo", + "ogo": "Khana", + "ogu": "Ogbronuagum", + "oht": "Old Hittite", + "ohu": "Old Hungarian", + "oia": "Oirata", + "oie": "Okolie", + "oin": "Inebu One", + "oj": "Ojibwa", + "ojb": "Northwestern Ojibwa", + "ojc": "Central Ojibwa", + "ojg": "Eastern Ojibwa", + "ojp": "Old Japanese", + "ojs": "Severn Ojibwa", + "ojv": "Ontong Java", + "ojw": "Western Ojibwa", + "oka": "Okanagan", + "okb": "Okobo", + "okc": "Kobo", + "okd": "Okodia", + "oke": "Okpe (Southwestern Edo)", + "okg": "Koko Babangk", + "okh": "Koresh-e Rostam", + "oki": "Okiek", + "okj": "Oko-Juwoi", + "okk": "Kwamtim One", + "okl": "Old Kentish Sign Language", + "okm": "Middle Korean (10th-16th cent.)", + "okn": "Oki-No-Erabu", + "oko": "Old Korean (3rd-9th cent.)", + "okr": "Kirike", + "oks": "Oko-Eni-Osayen", + "oku": "Oku", + "okv": "Orokaiva", + "okx": "Okpe (Northwestern Edo)", + "okz": "Old Khmer", + "ola": "Walungge", + "old": "Mochi", + "ole": "Olekha", + "olk": "Olkol", + "olm": "Oloma", + "olo": "Livvi", + "olr": "Olrat", + "olt": "Old Lithuanian", + "olu": "Kuvale", + "om": "Oromo", + "oma": "Omaha-Ponca", + "omb": "East Ambae", + "omc": "Mochica", + "omg": "Omagua", + "omi": "Omi", + "omk": "Omok", + "oml": "Ombo", + "omn": "Minoan", + "omo": "Utarmbung", + "omp": "Old Manipuri", + "omq": "Oto-Manguean languages", + "omr": "Old Marathi", + "omt": "Omotik", + "omu": "Omurano", + "omv": "Omotic languages", + "omw": "South Tairora", + "omx": "Old Mon", + "omy": "Old Malay", + "ona": "Ona", + "onb": "Lingao", + "one": "Oneida", + "ong": "Olo", + "oni": "Onin", + "onj": "Onjob", + "onk": "Kabore One", + "onn": "Onobasulu", + "ono": "Onondaga", + "onp": "Sartang", + "onr": "Northern One", + "ons": "Ono", + "ont": "Ontenu", + "onu": "Unua", + "onw": "Old Nubian", + "onx": "Onin Based Pidgin", + "ood": "Tohono O'odham", + "oog": "Ong", + "oon": "Önge", + "oor": "Oorlams", + "oos": "Old Ossetic", + "opa": "Okpamheri", + "opk": "Kopkaka", + "opm": "Oksapmin", + "opo": "Opao", + "opt": "Opata", + "opy": "Ofayé", + "or": "Oriya (macrolanguage); Odia (macrolanguage)", + "ora": "Oroha", + "orc": "Orma", + "ore": "Orejón", + "org": "Oring", + "orh": "Oroqen", + "orn": "Orang Kanaq", + "oro": "Orokolo", + "orr": "Oruma", + "ors": "Orang Seletar", + "ort": "Adivasi Oriya", + "oru": "Ormuri", + "orv": "Old Russian", + "orw": "Oro Win", + "orx": "Oro", + "ory": "Odia (individual language); Oriya (individual language)", + "orz": "Ormu", + "os": "Ossetian; Ossetic", + "osa": "Osage", + "osc": "Oscan", + "osi": "Osing", + "osn": "Old Sundanese", + "oso": "Ososo", + "osp": "Old Spanish", + "ost": "Osatu", + "osu": "Southern One", + "osx": "Old Saxon", + "ota": "Ottoman Turkish (1500-1928)", + "otb": "Old Tibetan", + "otd": "Ot Danum", + "ote": "Mezquital Otomi", + "oti": "Oti", + "otk": "Old Turkish", + "otl": "Tilapa Otomi", + "otm": "Eastern Highland Otomi", + "otn": "Tenango Otomi", + "oto": "Otomian languages", + "otq": "Querétaro Otomi", + "otr": "Otoro", + "ots": "Estado de México Otomi", + "ott": "Temoaya Otomi", + "otu": "Otuke", + "otw": "Ottawa", + "otx": "Texcatepec Otomi", + "oty": "Old Tamil", + "otz": "Ixtenco Otomi", + "oua": "Tagargrent", + "oub": "Glio-Oubi", + "oue": "Oune", + "oui": "Old Uighur", + "oum": "Ouma", + "ovd": "Elfdalian; Övdalian", + "owi": "Owiniga", + "owl": "Old Welsh", + "oyb": "Oy", + "oyd": "Oyda", + "oym": "Wayampi", + "oyy": "Oya'oya", + "ozm": "Koonzime", + "pa": "Panjabi; Punjabi", + "paa": "Papuan languages", + "pab": "Parecís", + "pac": "Pacoh", + "pad": "Paumarí", + "pae": "Pagibete", + "paf": "Paranawát", + "pag": "Pangasinan", + "pah": "Tenharim", + "pai": "Pe", + "pak": "Parakanã", + "pal": "Pahlavi", + "pam": "Pampanga; Kapampangan", + "pao": "Northern Paiute", + "pap": "Papiamento", + "paq": "Parya", + "par": "Panamint; Timbisha", + "pas": "Papasena", + "pau": "Palauan", + "pav": "Pakaásnovos", + "paw": "Pawnee", + "pax": "Pankararé", + "pay": "Pech", + "paz": "Pankararú", + "pbb": "Páez", + "pbc": "Patamona", + "pbe": "Mezontla Popoloca", + "pbf": "Coyotepec Popoloca", + "pbg": "Paraujano", + "pbh": "E'ñapa Woromaipu", + "pbi": "Parkwa", + "pbl": "Mak (Nigeria)", + "pbm": "Puebla Mazatec", + "pbn": "Kpasam", + "pbo": "Papel", + "pbp": "Badyara", + "pbr": "Pangwa", + "pbs": "Central Pame", + "pbt": "Southern Pashto", + "pbu": "Northern Pashto", + "pbv": "Pnar", + "pby": "Pyu (Papua New Guinea)", + "pca": "Santa Inés Ahuatempan Popoloca", + "pcb": "Pear", + "pcc": "Bouyei", + "pcd": "Picard", + "pce": "Ruching Palaung", + "pcf": "Paliyan", + "pcg": "Paniya", + "pch": "Pardhan", + "pci": "Duruwa", + "pcj": "Parenga", + "pck": "Paite Chin", + "pcl": "Pardhi", + "pcm": "Nigerian Pidgin", + "pcn": "Piti", + "pcp": "Pacahuara", + "pcw": "Pyapun", + "pda": "Anam", + "pdc": "Pennsylvania German", + "pdi": "Pa Di", + "pdn": "Podena; Fedan", + "pdo": "Padoe", + "pdt": "Plautdietsch", + "pdu": "Kayan", + "pea": "Peranakan Indonesian", + "peb": "Eastern Pomo", + "ped": "Mala (Papua New Guinea)", + "pee": "Taje", + "pef": "Northeastern Pomo", + "peg": "Pengo", + "peh": "Bonan", + "pei": "Chichimeca-Jonaz", + "pej": "Northern Pomo", + "pek": "Penchal", + "pel": "Pekal", + "pem": "Phende", + "peo": "Old Persian (ca. 600-400 B.C.)", + "pep": "Kunja", + "peq": "Southern Pomo", + "pes": "Iranian Persian", + "pev": "Pémono", + "pex": "Petats", + "pey": "Petjo", + "pez": "Eastern Penan", + "pfa": "Pááfang", + "pfe": "Pere", + "pfl": "Pfaelzisch", + "pga": "Sudanese Creole Arabic", + "pgd": "Gāndhārī", + "pgg": "Pangwali", + "pgi": "Pagi", + "pgk": "Rerep", + "pgl": "Primitive Irish", + "pgn": "Paelignian", + "pgs": "Pangseng", + "pgu": "Pagu", + "pgz": "Papua New Guinean Sign Language", + "pha": "Pa-Hng", + "phd": "Phudagi", + "phg": "Phuong", + "phh": "Phukha", + "phi": "Philippine languages", + "phj": "Pahari", + "phk": "Phake", + "phl": "Phalura; Palula", + "phm": "Phimbi", + "phn": "Phoenician", + "pho": "Phunoi", + "phq": "Phana'", + "phr": "Pahari-Potwari", + "pht": "Phu Thai", + "phu": "Phuan", + "phv": "Pahlavani", + "phw": "Phangduwali", + "pi": "Pali", + "pia": "Pima Bajo", + "pib": "Yine", + "pic": "Pinji", + "pid": "Piaroa", + "pie": "Piro", + "pif": "Pingelapese", + "pig": "Pisabo", + "pih": "Pitcairn-Norfolk", + "pij": "Pijao", + "pil": "Yom", + "pim": "Powhatan", + "pin": "Piame", + "pio": "Piapoco", + "pip": "Pero", + "pir": "Piratapuyo", + "pis": "Pijin", + "pit": "Pitta Pitta", + "piu": "Pintupi-Luritja", + "piv": "Pileni; Vaeakau-Taumako", + "piw": "Pimbwe", + "pix": "Piu", + "piy": "Piya-Kwonci", + "piz": "Pije", + "pjt": "Pitjantjatjara", + "pka": "Ardhamāgadhī Prākrit", + "pkb": "Pokomo; Kipfokomo", + "pkc": "Paekche", + "pkg": "Pak-Tong", + "pkh": "Pankhu", + "pkn": "Pakanha", + "pko": "Pökoot", + "pkp": "Pukapuka", + "pkr": "Attapady Kurumba", + "pks": "Pakistan Sign Language", + "pkt": "Maleng", + "pku": "Paku", + "pl": "Polish", + "pla": "Miani", + "plb": "Polonombauk", + "plc": "Central Palawano", + "pld": "Polari", + "ple": "Palu'e", + "plf": "Central Malayo-Polynesian languages", + "plg": "Pilagá", + "plh": "Paulohi", + "plj": "Polci", + "plk": "Kohistani Shina", + "pll": "Shwe Palaung", + "pln": "Palenquero", + "plo": "Oluta Popoluca", + "plq": "Palaic", + "plr": "Palaka Senoufo", + "pls": "San Marcos Tlacoyalco Popoloca; San Marcos Tlalcoyalco Popoloca", + "plt": "Plateau Malagasy", + "plu": "Palikúr", + "plv": "Southwest Palawano", + "plw": "Brooke's Point Palawano", + "ply": "Bolyu", + "plz": "Paluan", + "pma": "Paama", + "pmb": "Pambia", + "pmd": "Pallanganmiddang", + "pme": "Pwaamei", + "pmf": "Pamona", + "pmh": "Māhārāṣṭri Prākrit", + "pmi": "Northern Pumi", + "pmj": "Southern Pumi", + "pmk": "Pamlico", + "pml": "Lingua Franca", + "pmm": "Pomo", + "pmn": "Pam", + "pmo": "Pom", + "pmq": "Northern Pame", + "pmr": "Paynamar", + "pms": "Piemontese", + "pmt": "Tuamotuan", + "pmw": "Plains Miwok", + "pmx": "Poumei Naga", + "pmy": "Papuan Malay", + "pmz": "Southern Pame", + "pna": "Punan Bah-Biau", + "pnb": "Western Panjabi", + "pnc": "Pannei", + "pnd": "Mpinda", + "pne": "Western Penan", + "png": "Pangu; Pongu", + "pnh": "Penrhyn", + "pni": "Aoheng", + "pnj": "Pinjarup", + "pnk": "Paunaka", + "pnl": "Paleni", + "pnm": "Punan Batu 1", + "pnn": "Pinai-Hagahai", + "pno": "Panobo", + "pnp": "Pancana", + "pnq": "Pana (Burkina Faso)", + "pnr": "Panim", + "pns": "Ponosakan", + "pnt": "Pontic", + "pnu": "Jiongnai Bunu", + "pnv": "Pinigura", + "pnw": "Banyjima; Panytyima", + "pnx": "Phong-Kniang", + "pny": "Pinyin", + "pnz": "Pana (Central African Republic)", + "poc": "Poqomam", + "poe": "San Juan Atzingo Popoloca", + "pof": "Poke", + "pog": "Potiguára", + "poh": "Poqomchi'", + "poi": "Highland Popoluca", + "pok": "Pokangá", + "pom": "Southeastern Pomo", + "pon": "Pohnpeian", + "poo": "Central Pomo", + "pop": "Pwapwâ", + "poq": "Texistepec Popoluca", + "pos": "Sayula Popoluca", + "pot": "Potawatomi", + "pov": "Upper Guinea Crioulo", + "pow": "San Felipe Otlaltepec Popoloca", + "pox": "Polabian", + "poy": "Pogolo", + "poz": "Malayo-Polynesian languages", + "ppe": "Papi", + "ppi": "Paipai", + "ppk": "Uma", + "ppl": "Pipil; Nicarao", + "ppm": "Papuma", + "ppn": "Papapana", + "ppo": "Folopa", + "ppp": "Pelende", + "ppq": "Pei", + "pps": "San Luís Temalacayuca Popoloca", + "ppt": "Pare", + "ppu": "Papora", + "pqa": "Pa'a", + "pqe": "Eastern Malayo-Polynesian languages", + "pqm": "Malecite-Passamaquoddy", + "pqw": "Western Malayo-Polynesian languages", + "pra": "Prakrit languages", + "prc": "Parachi", + "prd": "Parsi-Dari", + "pre": "Principense", + "prf": "Paranan", + "prg": "Prussian", + "prh": "Porohanon", + "pri": "Paicî", + "prk": "Parauk", + "prl": "Peruvian Sign Language", + "prm": "Kibiri", + "prn": "Prasuni", + "pro": "Old Provençal (to 1500); Old Occitan (to 1500)", + "prp": "Parsi", + "prq": "Ashéninka Perené", + "prr": "Puri", + "prs": "Dari; Afghan Persian", + "prt": "Phai", + "pru": "Puragi", + "prw": "Parawen", + "prx": "Purik", + "prz": "Providencia Sign Language", + "ps": "Pushto; Pashto", + "psa": "Asue Awyu", + "psc": "Iranian Sign Language; Persian Sign Language", + "psd": "Plains Indian Sign Language", + "pse": "Central Malay", + "psg": "Penang Sign Language", + "psh": "Southwest Pashai; Southwest Pashayi", + "psi": "Southeast Pashai; Southeast Pashayi", + "psl": "Puerto Rican Sign Language", + "psm": "Pauserna", + "psn": "Panasuan", + "pso": "Polish Sign Language", + "psp": "Philippine Sign Language", + "psq": "Pasi", + "psr": "Portuguese Sign Language", + "pss": "Kaulong", + "pst": "Central Pashto", + "psu": "Sauraseni Prākrit", + "psw": "Port Sandwich", + "psy": "Piscataway", + "pt": "Portuguese", + "pta": "Pai Tavytera", + "pth": "Pataxó Hã-Ha-Hãe", + "pti": "Pindiini; Wangkatha", + "ptn": "Patani", + "pto": "Zo'é", + "ptp": "Patep", + "ptq": "Pattapu", + "ptr": "Piamatsina", + "ptt": "Enrekang", + "ptu": "Bambam", + "ptv": "Port Vato", + "ptw": "Pentlatch", + "pty": "Pathiya", + "pua": "Western Highland Purepecha", + "pub": "Purum", + "puc": "Punan Merap", + "pud": "Punan Aput", + "pue": "Puelche", + "puf": "Punan Merah", + "pug": "Phuie", + "pui": "Puinave", + "puj": "Punan Tubu", + "pum": "Puma", + "puo": "Puoc", + "pup": "Pulabu", + "puq": "Puquina", + "pur": "Puruborá", + "put": "Putoh", + "puu": "Punu", + "puw": "Puluwatese", + "pux": "Puare", + "puy": "Purisimeño", + "pwa": "Pawaia", + "pwb": "Panawa", + "pwg": "Gapapaiwa", + "pwi": "Patwin", + "pwm": "Molbog", + "pwn": "Paiwan", + "pwo": "Pwo Western Karen", + "pwr": "Powari", + "pww": "Pwo Northern Karen", + "pxm": "Quetzaltepec Mixe", + "pye": "Pye Krumen", + "pym": "Fyam", + "pyn": "Poyanáwa", + "pys": "Paraguayan Sign Language; Lengua de Señas del Paraguay", + "pyu": "Puyuma", + "pyx": "Pyu (Myanmar)", + "pyy": "Pyen", + "pzh": "Pazeh", + "pzn": "Jejara Naga; Para Naga", + "qu": "Quechua", + "qua": "Quapaw", + "qub": "Huallaga Huánuco Quechua", + "quc": "K'iche'; Quiché", + "qud": "Calderón Highland Quichua", + "quf": "Lambayeque Quechua", + "qug": "Chimborazo Highland Quichua", + "quh": "South Bolivian Quechua", + "qui": "Quileute", + "quk": "Chachapoyas Quechua", + "qul": "North Bolivian Quechua", + "qum": "Sipacapense", + "qun": "Quinault", + "qup": "Southern Pastaza Quechua", + "quq": "Quinqui", + "qur": "Yanahuanca Pasco Quechua", + "qus": "Santiago del Estero Quichua", + "quv": "Sacapulteco", + "quw": "Tena Lowland Quichua", + "qux": "Yauyos Quechua", + "quy": "Ayacucho Quechua", + "quz": "Cusco Quechua", + "qva": "Ambo-Pasco Quechua", + "qvc": "Cajamarca Quechua", + "qve": "Eastern Apurímac Quechua", + "qvh": "Huamalíes-Dos de Mayo Huánuco Quechua", + "qvi": "Imbabura Highland Quichua", + "qvj": "Loja Highland Quichua", + "qvl": "Cajatambo North Lima Quechua", + "qvm": "Margos-Yarowilca-Lauricocha Quechua", + "qvn": "North Junín Quechua", + "qvo": "Napo Lowland Quechua", + "qvp": "Pacaraos Quechua", + "qvs": "San Martín Quechua", + "qvw": "Huaylla Wanca Quechua", + "qvy": "Queyu", + "qvz": "Northern Pastaza Quichua", + "qwa": "Corongo Ancash Quechua", + "qwc": "Classical Quechua", + "qwe": "Quechuan (family)", + "qwh": "Huaylas Ancash Quechua", + "qwm": "Kuman (Russia)", + "qws": "Sihuas Ancash Quechua", + "qwt": "Kwalhioqua-Tlatskanai", + "qxa": "Chiquián Ancash Quechua", + "qxc": "Chincha Quechua", + "qxh": "Panao Huánuco Quechua", + "qxl": "Salasaca Highland Quichua", + "qxn": "Northern Conchucos Ancash Quechua", + "qxo": "Southern Conchucos Ancash Quechua", + "qxp": "Puno Quechua", + "qxq": "Qashqa'i", + "qxr": "Cañar Highland Quichua", + "qxs": "Southern Qiang", + "qxt": "Santa Ana de Tusi Pasco Quechua", + "qxu": "Arequipa-La Unión Quechua", + "qxw": "Jauja Wanca Quechua", + "qya": "Quenya", + "qyp": "Quiripi", + "raa": "Dungmali", + "rab": "Camling", + "rac": "Rasawa", + "rad": "Rade", + "raf": "Western Meohang", + "rag": "Logooli; Lulogooli", + "rah": "Rabha", + "rai": "Ramoaaina", + "raj": "Rajasthani", + "rak": "Tulu-Bohuai", + "ral": "Ralte", + "ram": "Canela", + "ran": "Riantana", + "rao": "Rao", + "rap": "Rapanui", + "raq": "Saam", + "rar": "Rarotongan; Cook Islands Maori", + "ras": "Tegali", + "rat": "Razajerdi", + "rau": "Raute", + "rav": "Sampang", + "raw": "Rawang", + "rax": "Rang", + "ray": "Rapa", + "raz": "Rahambuu", + "rbb": "Rumai Palaung", + "rbk": "Northern Bontok", + "rbl": "Miraya Bikol", + "rbp": "Barababaraba", + "rcf": "Réunion Creole French", + "rdb": "Rudbari", + "rea": "Rerau", + "reb": "Rembong", + "ree": "Rejang Kayan", + "reg": "Kara (Tanzania)", + "rei": "Reli", + "rej": "Rejang", + "rel": "Rendille", + "rem": "Remo", + "ren": "Rengao", + "rer": "Rer Bare", + "res": "Reshe", + "ret": "Retta", + "rey": "Reyesano", + "rga": "Roria", + "rge": "Romano-Greek", + "rgk": "Rangkas", + "rgn": "Romagnol", + "rgr": "Resígaro", + "rgs": "Southern Roglai", + "rgu": "Ringgou", + "rhg": "Rohingya", + "rhp": "Yahang", + "ria": "Riang (India)", + "rib": "Bribri Sign Language", + "rif": "Tarifit", + "ril": "Riang Lang; Riang (Myanmar)", + "rim": "Nyaturu", + "rin": "Nungu", + "rir": "Ribun", + "rit": "Ritharrngu", + "riu": "Riung", + "rjg": "Rajong", + "rji": "Raji", + "rjs": "Rajbanshi", + "rka": "Kraol", + "rkb": "Rikbaktsa", + "rkh": "Rakahanga-Manihiki", + "rki": "Rakhine", + "rkm": "Marka", + "rkt": "Rangpuri; Kamta", + "rkw": "Arakwal", + "rm": "Romansh", + "rma": "Rama", + "rmb": "Rembarrnga", + "rmc": "Carpathian Romani", + "rmd": "Traveller Danish", + "rme": "Angloromani", + "rmf": "Kalo Finnish Romani", + "rmg": "Traveller Norwegian", + "rmh": "Murkim", + "rmi": "Lomavren", + "rmk": "Romkun", + "rml": "Baltic Romani", + "rmm": "Roma", + "rmn": "Balkan Romani", + "rmo": "Sinte Romani", + "rmp": "Rempi", + "rmq": "Caló", + "rms": "Romanian Sign Language", + "rmt": "Domari", + "rmu": "Tavringer Romani", + "rmv": "Romanova", + "rmw": "Welsh Romani", + "rmx": "Romam", + "rmy": "Vlax Romani", + "rmz": "Marma", + "rn": "Rundi", + "rnb": "Brunca Sign Language", + "rnd": "Ruund", + "rng": "Ronga", + "rnl": "Ranglong", + "rnn": "Roon", + "rnp": "Rongpo", + "rnr": "Nari Nari", + "rnw": "Rungwa", + "ro": "Romanian; Moldavian; Moldovan", + "roa": "Romance languages", + "rob": "Tae'", + "roc": "Cacgia Roglai", + "rod": "Rogo", + "roe": "Ronji", + "rof": "Rombo", + "rog": "Northern Roglai", + "rol": "Romblomanon", + "rom": "Romany", + "roo": "Rotokas", + "rop": "Kriol", + "ror": "Rongga", + "rou": "Runga", + "row": "Dela-Oenale", + "rpn": "Repanbitip", + "rpt": "Rapting", + "rri": "Ririo", + "rro": "Waima", + "rrt": "Arritinngithigh", + "rsb": "Romano-Serbian", + "rsk": "Ruthenian; Rusyn", + "rsl": "Russian Sign Language", + "rsm": "Miriwoong Sign Language", + "rsn": "Rwandan Sign Language", + "rtc": "Rungtu Chin", + "rth": "Ratahan", + "rtm": "Rotuman", + "rts": "Yurats", + "rtw": "Rathawi", + "ru": "Russian", + "rub": "Gungu", + "ruc": "Ruuli", + "rue": "Rusyn", + "ruf": "Luguru", + "rug": "Roviana", + "ruh": "Ruga", + "rui": "Rufiji", + "ruk": "Che", + "ruo": "Istro Romanian", + "rup": "Macedo-Romanian; Aromanian; Arumanian", + "ruq": "Megleno Romanian", + "rut": "Rutul", + "ruu": "Lanas Lobu", + "ruy": "Mala (Nigeria)", + "ruz": "Ruma", + "rw": "Kinyarwanda", + "rwa": "Rawo", + "rwk": "Rwa", + "rwl": "Ruwila", + "rwm": "Amba (Uganda)", + "rwo": "Rawa", + "rwr": "Marwari (India)", + "rxd": "Ngardi", + "rxw": "Karuwali; Garuwali", + "ryn": "Northern Amami-Oshima", + "rys": "Yaeyama", + "ryu": "Central Okinawan", + "rzh": "Rāziḥī", + "sa": "Sanskrit", + "saa": "Saba", + "sab": "Buglere", + "sac": "Meskwaki", + "sad": "Sandawe", + "sae": "Sabanê", + "saf": "Safaliba", + "sah": "Yakut", + "sai": "South American Indian languages", + "saj": "Sahu", + "sak": "Sake", + "sal": "Salishan languages", + "sam": "Samaritan Aramaic", + "sao": "Sause", + "saq": "Samburu", + "sar": "Saraveca", + "sas": "Sasak", + "sat": "Santali", + "sau": "Saleman", + "sav": "Saafi-Saafi", + "saw": "Sawi", + "sax": "Sa", + "say": "Saya", + "saz": "Saurashtra", + "sba": "Ngambay", + "sbb": "Simbo", + "sbc": "Kele (Papua New Guinea)", + "sbd": "Southern Samo", + "sbe": "Saliba", + "sbf": "Chabu; Shabo", + "sbg": "Seget", + "sbh": "Sori-Harengan", + "sbi": "Seti", + "sbj": "Surbakhal", + "sbk": "Safwa", + "sbl": "Botolan Sambal", + "sbm": "Sagala", + "sbn": "Sindhi Bhil", + "sbo": "Sabüm", + "sbp": "Sangu (Tanzania)", + "sbq": "Sileibi", + "sbr": "Sembakung Murut", + "sbs": "Subiya", + "sbt": "Kimki", + "sbu": "Stod Bhoti", + "sbv": "Sabine", + "sbw": "Simba", + "sbx": "Seberuang", + "sby": "Soli", + "sbz": "Sara Kaba", + "sc": "Sardinian", + "scb": "Chut", + "sce": "Dongxiang", + "scf": "San Miguel Creole French", + "scg": "Sanggau", + "sch": "Sakachep", + "sci": "Sri Lankan Creole Malay", + "sck": "Sadri", + "scl": "Shina", + "scn": "Sicilian", + "sco": "Scots", + "scp": "Hyolmo; Helambu Sherpa", + "scq": "Sa'och", + "scs": "North Slavey", + "sct": "Southern Katang", + "scu": "Shumcho", + "scv": "Sheni", + "scw": "Sha", + "scx": "Sicel", + "sd": "Sindhi", + "sda": "Toraja-Sa'dan", + "sdb": "Shabak", + "sdc": "Sassarese Sardinian", + "sde": "Surubu", + "sdf": "Sarli", + "sdg": "Savi", + "sdh": "Southern Kurdish", + "sdj": "Suundi", + "sdk": "Sos Kundi", + "sdl": "Saudi Arabian Sign Language", + "sdn": "Gallurese Sardinian", + "sdo": "Bukar-Sadung Bidayuh", + "sdp": "Sherdukpen", + "sdq": "Semandang", + "sdr": "Oraon Sadri", + "sds": "Sened", + "sdt": "Shuadit", + "sdu": "Sarudu", + "sdv": "Eastern Sudanic languages", + "sdx": "Sibu Melanau", + "sdz": "Sallands", + "se": "Northern Sami", + "sea": "Semai", + "seb": "Shempire Senoufo", + "sec": "Sechelt", + "sed": "Sedang", + "see": "Seneca", + "sef": "Cebaara Senoufo", + "seg": "Segeju", + "seh": "Sena", + "sei": "Seri", + "sej": "Sene", + "sek": "Sekani", + "sel": "Selkup", + "sem": "Semitic languages", + "sen": "Nanerigé Sénoufo", + "seo": "Suarmin", + "sep": "Sìcìté Sénoufo", + "seq": "Senara Sénoufo", + "ser": "Serrano", + "ses": "Koyraboro Senni Songhai", + "set": "Sentani", + "seu": "Serui-Laut", + "sev": "Nyarafolo Senoufo", + "sew": "Sewa Bay", + "sey": "Secoya", + "sez": "Senthang Chin", + "sfb": "Langue des signes de Belgique Francophone; French Belgian Sign Language", + "sfe": "Eastern Subanen", + "sfm": "Small Flowery Miao", + "sfs": "South African Sign Language", + "sfw": "Sehwi", + "sg": "Sango", + "sga": "Old Irish (to 900)", + "sgb": "Mag-antsi Ayta", + "sgc": "Kipsigis", + "sgd": "Surigaonon", + "sge": "Segai", + "sgg": "Swiss-German Sign Language", + "sgh": "Shughni", + "sgi": "Suga", + "sgj": "Surgujia", + "sgk": "Sangkong", + "sgm": "Singa", + "sgn": "Sign languages", + "sgp": "Singpho", + "sgr": "Sangisari", + "sgs": "Samogitian", + "sgt": "Brokpake", + "sgu": "Salas", + "sgw": "Sebat Bet Gurage", + "sgx": "Sierra Leone Sign Language", + "sgy": "Sanglechi", + "sgz": "Sursurunga", + "sh": "Serbo-Croatian", + "sha": "Shall-Zwall", + "shb": "Ninam", + "shc": "Sonde", + "shd": "Kundal Shahi", + "she": "Sheko", + "shg": "Shua", + "shh": "Shoshoni", + "shi": "Tachelhit", + "shj": "Shatt", + "shk": "Shilluk", + "shl": "Shendu", + "shm": "Shahrudi", + "shn": "Shan", + "sho": "Shanga", + "shp": "Shipibo-Conibo", + "shq": "Sala", + "shr": "Shi", + "shs": "Shuswap", + "sht": "Shasta", + "shu": "Chadian Arabic", + "shv": "Shehri", + "shw": "Shwai", + "shx": "She", + "shy": "Tachawit", + "shz": "Syenara Senoufo", + "si": "Sinhala; Sinhalese", + "sia": "Akkala Sami", + "sib": "Sebop", + "sid": "Sidamo", + "sie": "Simaa", + "sif": "Siamou", + "sig": "Paasaal", + "sih": "Zire; Sîshëë", + "sii": "Shom Peng", + "sij": "Numbami", + "sik": "Sikiana", + "sil": "Tumulung Sisaala", + "sim": "Mende (Papua New Guinea)", + "sio": "Siouan languages", + "sip": "Sikkimese", + "siq": "Sonia", + "sir": "Siri", + "sis": "Siuslaw", + "sit": "Sino-Tibetan languages", + "siu": "Sinagen", + "siv": "Sumariup", + "siw": "Siwai", + "six": "Sumau", + "siy": "Sivandi", + "siz": "Siwi", + "sja": "Epena", + "sjb": "Sajau Basap", + "sjd": "Kildin Sami", + "sje": "Pite Sami", + "sjg": "Assangori", + "sjk": "Kemi Sami", + "sjl": "Sajalong; Miji", + "sjm": "Mapun", + "sjn": "Sindarin", + "sjo": "Xibe", + "sjp": "Surjapuri", + "sjr": "Siar-Lak", + "sjs": "Senhaja De Srair", + "sjt": "Ter Sami", + "sju": "Ume Sami", + "sjw": "Shawnee", + "sk": "Slovak", + "ska": "Skagit", + "skb": "Saek", + "skc": "Ma Manda", + "skd": "Southern Sierra Miwok", + "ske": "Seke (Vanuatu)", + "skf": "Sakirabiá", + "skg": "Sakalava Malagasy", + "skh": "Sikule", + "ski": "Sika", + "skj": "Seke (Nepal)", + "skm": "Kutong", + "skn": "Kolibugan Subanon", + "sko": "Seko Tengah", + "skp": "Sekapan", + "skq": "Sininkere", + "skr": "Saraiki; Seraiki", + "sks": "Maia", + "skt": "Sakata", + "sku": "Sakao", + "skv": "Skou", + "skw": "Skepi Creole Dutch", + "skx": "Seko Padang", + "sky": "Sikaiana", + "skz": "Sekar", + "sl": "Slovenian", + "sla": "Slavic languages", + "slc": "Sáliba", + "sld": "Sissala", + "sle": "Sholaga", + "slf": "Swiss-Italian Sign Language", + "slg": "Selungai Murut", + "slh": "Southern Puget Sound Salish", + "sli": "Lower Silesian", + "slj": "Salumá", + "sll": "Salt-Yui", + "slm": "Pangutaran Sama", + "sln": "Salinan", + "slp": "Lamaholot", + "slq": "Salchuq", + "slr": "Salar", + "sls": "Singapore Sign Language", + "slt": "Sila", + "slu": "Selaru", + "slw": "Sialum", + "slx": "Salampasu", + "sly": "Selayar", + "slz": "Ma'ya", + "sm": "Samoan", + "sma": "Southern Sami", + "smb": "Simbari", + "smc": "Som", + "smf": "Auwe", + "smg": "Simbali", + "smh": "Samei", + "smi": "Sami languages", + "smj": "Lule Sami", + "smk": "Bolinao", + "sml": "Central Sama", + "smm": "Musasa", + "smn": "Inari Sami", + "smp": "Samaritan", + "smq": "Samo", + "smr": "Simeulue", + "sms": "Skolt Sami", + "smt": "Simte", + "smu": "Somray", + "smv": "Samvedi", + "smw": "Sumbawa", + "smx": "Samba", + "smy": "Semnani", + "smz": "Simeku", + "sn": "Shona", + "snc": "Sinaugoro", + "sne": "Bau Bidayuh", + "snf": "Noon", + "sng": "Sanga (Democratic Republic of Congo)", + "sni": "Sensi", + "snj": "Riverain Sango", + "snk": "Soninke", + "snl": "Sangil", + "snm": "Southern Ma'di", + "snn": "Siona", + "sno": "Snohomish", + "snp": "Siane", + "snq": "Sangu (Gabon)", + "snr": "Sihan", + "sns": "South West Bay; Nahavaq", + "snu": "Senggi; Viid", + "snv": "Sa'ban", + "snw": "Selee", + "snx": "Sam", + "sny": "Saniyo-Hiyewe", + "snz": "Kou", + "so": "Somali", + "soa": "Thai Song", + "sob": "Sobei", + "soc": "So (Democratic Republic of Congo)", + "sod": "Songoora", + "soe": "Songomeno", + "sog": "Sogdian", + "soh": "Aka", + "soi": "Sonha", + "soj": "Soi", + "sok": "Sokoro", + "sol": "Solos", + "son": "Songhai languages", + "soo": "Songo", + "sop": "Songe", + "soq": "Kanasi", + "sor": "Somrai", + "sos": "Seeku", + "sou": "Southern Thai", + "sov": "Sonsorol", + "sow": "Sowanda", + "sox": "Swo", + "soy": "Miyobe", + "soz": "Temi", + "spb": "Sepa (Indonesia)", + "spc": "Sapé", + "spd": "Saep", + "spe": "Sepa (Papua New Guinea)", + "spg": "Sian", + "spi": "Saponi", + "spk": "Sengo", + "spl": "Selepet", + "spm": "Akukem", + "spn": "Sanapaná", + "spo": "Spokane", + "spp": "Supyire Senoufo", + "spq": "Loreto-Ucayali Spanish", + "spr": "Saparua", + "sps": "Saposa", + "spt": "Spiti Bhoti", + "spu": "Sapuan", + "spv": "Sambalpuri; Kosli", + "spx": "South Picene", + "spy": "Sabaot", + "sq": "Albanian", + "sqa": "Shama-Sambuga", + "sqh": "Shau", + "sqj": "Albanian languages", + "sqk": "Albanian Sign Language", + "sqm": "Suma", + "sqn": "Susquehannock", + "sqo": "Sorkhei", + "sqq": "Sou", + "sqr": "Siculo Arabic", + "sqs": "Sri Lankan Sign Language", + "sqt": "Soqotri", + "squ": "Squamish", + "sqx": "Kufr Qassem Sign Language (KQSL)", + "sr": "Serbian", + "sra": "Saruga", + "srb": "Sora", + "src": "Logudorese Sardinian", + "sre": "Sara", + "srf": "Nafi", + "srg": "Sulod", + "srh": "Sarikoli", + "sri": "Siriano", + "srk": "Serudung Murut", + "srl": "Isirawa", + "srm": "Saramaccan", + "srn": "Sranan Tongo", + "sro": "Campidanese Sardinian", + "srq": "Sirionó", + "srr": "Serer", + "srs": "Sarsi", + "srt": "Sauri", + "sru": "Suruí", + "srv": "Southern Sorsoganon", + "srw": "Serua", + "srx": "Sirmauri", + "sry": "Sera", + "srz": "Shahmirzadi", + "ss": "Swati", + "ssa": "Nilo-Saharan languages", + "ssb": "Southern Sama", + "ssc": "Suba-Simbiti", + "ssd": "Siroi", + "sse": "Balangingi; Bangingih Sama", + "ssf": "Thao", + "ssg": "Seimat", + "ssh": "Shihhi Arabic", + "ssi": "Sansi", + "ssj": "Sausi", + "ssk": "Sunam", + "ssl": "Western Sisaala", + "ssm": "Semnam", + "ssn": "Waata", + "sso": "Sissano", + "ssp": "Spanish Sign Language", + "ssq": "So'a", + "ssr": "Swiss-French Sign Language", + "sss": "Sô", + "sst": "Sinasina", + "ssu": "Susuami", + "ssv": "Shark Bay", + "ssx": "Samberigi", + "ssy": "Saho", + "ssz": "Sengseng", + "st": "Southern Sotho", + "sta": "Settla", + "stb": "Northern Subanen", + "std": "Sentinel", + "ste": "Liana-Seti", + "stf": "Seta", + "stg": "Trieng", + "sth": "Shelta", + "sti": "Bulo Stieng", + "stj": "Matya Samo", + "stk": "Arammba", + "stl": "Stellingwerfs", + "stm": "Setaman", + "stn": "Owa", + "sto": "Stoney", + "stp": "Southeastern Tepehuan", + "stq": "Saterfriesisch", + "str": "Straits Salish", + "sts": "Shumashti", + "stt": "Budeh Stieng", + "stu": "Samtao", + "stv": "Silt'e", + "stw": "Satawalese", + "sty": "Siberian Tatar", + "su": "Sundanese", + "sua": "Sulka", + "sub": "Suku", + "suc": "Western Subanon", + "sue": "Suena", + "sug": "Suganga", + "sui": "Suki", + "suj": "Shubi", + "suk": "Sukuma", + "suo": "Bouni", + "suq": "Tirmaga-Chai Suri; Suri", + "sur": "Mwaghavul", + "sus": "Susu", + "sut": "Subtiaba", + "suv": "Puroik", + "suw": "Sumbwa", + "sux": "Sumerian", + "suy": "Suyá", + "suz": "Sunwar", + "sv": "Swedish", + "sva": "Svan", + "svb": "Ulau-Suain", + "svc": "Vincentian Creole English", + "sve": "Serili", + "svk": "Slovakian Sign Language", + "svm": "Slavomolisano", + "svs": "Savosavo", + "svx": "Skalvian", + "sw": "Swahili (macrolanguage)", + "swb": "Maore Comorian", + "swc": "Congo Swahili", + "swf": "Sere", + "swg": "Swabian", + "swh": "Swahili (individual language); Kiswahili", + "swi": "Sui", + "swj": "Sira", + "swk": "Malawi Sena", + "swl": "Swedish Sign Language", + "swm": "Samosa", + "swn": "Sawknah", + "swo": "Shanenawa", + "swp": "Suau", + "swq": "Sharwa", + "swr": "Saweru", + "sws": "Seluwasan", + "swt": "Sawila", + "swu": "Suwawa", + "swv": "Shekhawati", + "sww": "Sowa", + "swx": "Suruahá", + "swy": "Sarua", + "sxb": "Suba", + "sxc": "Sicanian", + "sxe": "Sighu", + "sxg": "Shuhi; Shixing", + "sxk": "Southern Kalapuya", + "sxl": "Selian", + "sxm": "Samre", + "sxn": "Sangir", + "sxo": "Sorothaptic", + "sxr": "Saaroa", + "sxs": "Sasaru", + "sxu": "Upper Saxon", + "sxw": "Saxwe Gbe", + "sya": "Siang", + "syb": "Central Subanen", + "syc": "Classical Syriac", + "syd": "Samoyedic languages", + "syi": "Seki", + "syk": "Sukur", + "syl": "Sylheti", + "sym": "Maya Samo", + "syn": "Senaya", + "syo": "Suoy", + "syr": "Syriac", + "sys": "Sinyar", + "syw": "Kagate", + "syx": "Samay", + "syy": "Al-Sayyid Bedouin Sign Language", + "sza": "Semelai", + "szb": "Ngalum", + "szc": "Semaq Beri", + "szd": "Seru", + "sze": "Seze", + "szg": "Sengele", + "szl": "Silesian", + "szn": "Sula", + "szp": "Suabo", + "szs": "Solomon Islands Sign Language", + "szv": "Isu (Fako Division)", + "szw": "Sawai", + "szy": "Sakizaya", + "ta": "Tamil", + "taa": "Lower Tanana", + "tab": "Tabassaran", + "tac": "Lowland Tarahumara", + "tad": "Tause", + "tae": "Tariana", + "taf": "Tapirapé", + "tag": "Tagoi", + "tai": "Tai languages", + "taj": "Eastern Tamang", + "tak": "Tala", + "tal": "Tal", + "tan": "Tangale", + "tao": "Yami", + "tap": "Taabwa", + "taq": "Tamasheq", + "tar": "Central Tarahumara", + "tas": "Tay Boi", + "tau": "Upper Tanana", + "tav": "Tatuyo", + "taw": "Tai", + "tax": "Tamki", + "tay": "Atayal", + "taz": "Tocho", + "tba": "Aikanã", + "tbc": "Takia", + "tbd": "Kaki Ae", + "tbe": "Tanimbili", + "tbf": "Mandara", + "tbg": "North Tairora", + "tbh": "Dharawal; Thurawal", + "tbi": "Gaam", + "tbj": "Tiang", + "tbk": "Calamian Tagbanwa", + "tbl": "Tboli", + "tbm": "Tagbu", + "tbn": "Barro Negro Tunebo", + "tbo": "Tawala", + "tbp": "Taworta; Diebroud", + "tbq": "Tibeto-Burman languages", + "tbr": "Tumtum", + "tbs": "Tanguat", + "tbt": "Tembo (Kitembo)", + "tbu": "Tubar", + "tbv": "Tobo", + "tbw": "Tagbanwa", + "tbx": "Kapin", + "tby": "Tabaru", + "tbz": "Ditammari", + "tca": "Ticuna", + "tcb": "Tanacross", + "tcc": "Datooga", + "tcd": "Tafi", + "tce": "Southern Tutchone", + "tcf": "Malinaltepec Me'phaa; Malinaltepec Tlapanec", + "tcg": "Tamagario", + "tch": "Turks And Caicos Creole English", + "tci": "Wára", + "tck": "Tchitchege", + "tcl": "Taman (Myanmar)", + "tcm": "Tanahmerah", + "tcn": "Tichurong", + "tco": "Taungyo", + "tcp": "Tawr Chin", + "tcq": "Kaiy", + "tcs": "Torres Strait Creole; Yumplatok", + "tct": "T'en", + "tcu": "Southeastern Tarahumara", + "tcw": "Tecpatlán Totonac", + "tcx": "Toda", + "tcy": "Tulu", + "tcz": "Thado Chin", + "tda": "Tagdal", + "tdb": "Panchpargania", + "tdc": "Emberá-Tadó", + "tdd": "Tai Nüa", + "tde": "Tiranige Diga Dogon", + "tdf": "Talieng", + "tdg": "Western Tamang", + "tdh": "Thulung", + "tdi": "Tomadino", + "tdj": "Tajio", + "tdk": "Tambas", + "tdl": "Sur", + "tdm": "Taruma", + "tdn": "Tondano", + "tdo": "Teme", + "tdq": "Tita", + "tdr": "Todrah", + "tds": "Doutai", + "tdt": "Tetun Dili", + "tdv": "Toro", + "tdx": "Tandroy-Mahafaly Malagasy", + "tdy": "Tadyawan", + "te": "Telugu", + "tea": "Temiar", + "teb": "Tetete", + "tec": "Terik", + "ted": "Tepo Krumen", + "tee": "Huehuetla Tepehua", + "tef": "Teressa", + "teg": "Teke-Tege", + "teh": "Tehuelche", + "tei": "Torricelli", + "tek": "Ibali Teke", + "tem": "Timne", + "ten": "Tama (Colombia)", + "teo": "Teso", + "tep": "Tepecano", + "teq": "Temein", + "ter": "Tereno", + "tes": "Tengger", + "tet": "Tetum", + "teu": "Soo", + "tev": "Teor", + "tew": "Tewa (USA)", + "tex": "Tennet", + "tey": "Tulishi", + "tez": "Tetserret", + "tfi": "Tofin Gbe", + "tfn": "Tanaina", + "tfo": "Tefaro", + "tfr": "Teribe", + "tft": "Ternate", + "tg": "Tajik", + "tga": "Sagalla", + "tgb": "Tobilung", + "tgc": "Tigak", + "tgd": "Ciwogai", + "tge": "Eastern Gorkha Tamang", + "tgf": "Chalikha", + "tgh": "Tobagonian Creole English", + "tgi": "Lawunuia", + "tgj": "Tagin", + "tgn": "Tandaganon", + "tgo": "Sudest", + "tgp": "Tangoa", + "tgq": "Tring", + "tgr": "Tareng", + "tgs": "Nume", + "tgt": "Central Tagbanwa", + "tgu": "Tanggu", + "tgv": "Tingui-Boto", + "tgw": "Tagwana Senoufo", + "tgx": "Tagish", + "tgy": "Togoyo", + "tgz": "Tagalaka", + "th": "Thai", + "thd": "Kuuk Thaayorre; Thayore", + "the": "Chitwania Tharu", + "thf": "Thangmi", + "thh": "Northern Tarahumara", + "thi": "Tai Long", + "thk": "Tharaka; Kitharaka", + "thl": "Dangaura Tharu", + "thm": "Aheu", + "thn": "Thachanadan", + "thp": "Thompson", + "thq": "Kochila Tharu", + "thr": "Rana Tharu", + "ths": "Thakali", + "tht": "Tahltan", + "thu": "Thuri", + "thv": "Tahaggart Tamahaq", + "thy": "Tha", + "thz": "Tayart Tamajeq", + "ti": "Tigrinya", + "tia": "Tidikelt Tamazight", + "tic": "Tira", + "tif": "Tifal", + "tig": "Tigre", + "tih": "Timugon Murut", + "tii": "Tiene", + "tij": "Tilung", + "tik": "Tikar", + "til": "Tillamook", + "tim": "Timbe", + "tin": "Tindi", + "tio": "Teop", + "tip": "Trimuris", + "tiq": "Tiéfo", + "tis": "Masadiit Itneg", + "tit": "Tinigua", + "tiu": "Adasen", + "tiv": "Tiv", + "tiw": "Tiwi", + "tix": "Southern Tiwa", + "tiy": "Tiruray", + "tiz": "Tai Hongjin", + "tja": "Tajuasohn", + "tjg": "Tunjung", + "tji": "Northern Tujia", + "tjj": "Tjungundji", + "tjl": "Tai Laing", + "tjm": "Timucua", + "tjn": "Tonjon", + "tjo": "Temacine Tamazight", + "tjp": "Tjupany", + "tjs": "Southern Tujia", + "tju": "Tjurruru", + "tjw": "Djabwurrung", + "tk": "Turkmen", + "tka": "Truká", + "tkb": "Buksa", + "tkd": "Tukudede", + "tke": "Takwane", + "tkf": "Tukumanféd", + "tkg": "Tesaka Malagasy", + "tkl": "Tokelau", + "tkm": "Takelma", + "tkn": "Toku-No-Shima", + "tkp": "Tikopia", + "tkq": "Tee", + "tkr": "Tsakhur", + "tks": "Takestani", + "tkt": "Kathoriya Tharu", + "tku": "Upper Necaxa Totonac", + "tkv": "Mur Pano", + "tkw": "Teanu", + "tkx": "Tangko", + "tkz": "Takua", + "tl": "Tagalog", + "tla": "Southwestern Tepehuan", + "tlb": "Tobelo", + "tlc": "Yecuatla Totonac", + "tld": "Talaud", + "tlf": "Telefol", + "tlg": "Tofanma", + "tlh": "Klingon; tlhIngan Hol", + "tli": "Tlingit", + "tlj": "Talinga-Bwisi", + "tlk": "Taloki", + "tll": "Tetela", + "tlm": "Tolomako", + "tln": "Talondo'", + "tlo": "Talodi", + "tlp": "Filomena Mata-Coahuitlán Totonac", + "tlq": "Tai Loi", + "tlr": "Talise", + "tls": "Tambotalo", + "tlt": "Sou Nama; Teluti", + "tlu": "Tulehu", + "tlv": "Taliabu", + "tlx": "Khehek", + "tly": "Talysh", + "tma": "Tama (Chad)", + "tmb": "Katbol; Avava", + "tmc": "Tumak", + "tmd": "Haruai", + "tme": "Tremembé", + "tmf": "Toba-Maskoy", + "tmg": "Ternateño", + "tmh": "Tamashek", + "tmi": "Tutuba", + "tmj": "Samarokena", + "tmk": "Northwestern Tamang", + "tml": "Tamnim Citak", + "tmm": "Tai Thanh", + "tmn": "Taman (Indonesia)", + "tmo": "Temoq", + "tmq": "Tumleo", + "tmr": "Jewish Babylonian Aramaic (ca. 200-1200 CE)", + "tms": "Tima", + "tmt": "Tasmate", + "tmu": "Iau", + "tmv": "Tembo (Motembo)", + "tmw": "Temuan", + "tmy": "Tami", + "tmz": "Tamanaku", + "tn": "Tswana", + "tna": "Tacana", + "tnb": "Western Tunebo", + "tnc": "Tanimuca-Retuarã", + "tnd": "Angosturas Tunebo", + "tng": "Tobanga", + "tnh": "Maiani", + "tni": "Tandia", + "tnk": "Kwamera", + "tnl": "Lenakel", + "tnm": "Tabla", + "tnn": "North Tanna", + "tno": "Toromono", + "tnp": "Whitesands", + "tnq": "Taino", + "tnr": "Ménik", + "tns": "Tenis", + "tnt": "Tontemboan", + "tnu": "Tay Khang", + "tnv": "Tangchangya", + "tnw": "Tonsawang", + "tnx": "Tanema", + "tny": "Tongwe", + "tnz": "Ten'edn", + "to": "Tonga (Tonga Islands)", + "tob": "Toba", + "toc": "Coyutla Totonac", + "tod": "Toma", + "tof": "Gizrra", + "tog": "Tonga (Nyasa)", + "toh": "Gitonga", + "toi": "Tonga (Zambia)", + "toj": "Tojolabal", + "tok": "Toki Pona", + "tol": "Tolowa", + "tom": "Tombulu", + "too": "Xicotepec De Juárez Totonac", + "top": "Papantla Totonac", + "toq": "Toposa", + "tor": "Togbo-Vara Banda", + "tos": "Highland Totonac", + "tou": "Tho", + "tov": "Upper Taromi", + "tow": "Jemez", + "tox": "Tobian", + "toy": "Topoiyo", + "toz": "To", + "tpa": "Taupota", + "tpc": "Azoyú Me'phaa; Azoyú Tlapanec", + "tpe": "Tippera", + "tpf": "Tarpia", + "tpg": "Kula", + "tpi": "Tok Pisin", + "tpj": "Tapieté", + "tpk": "Tupinikin", + "tpl": "Tlacoapa Me'phaa; Tlacoapa Tlapanec", + "tpm": "Tampulma", + "tpn": "Tupinambá", + "tpo": "Tai Pao", + "tpp": "Pisaflores Tepehua", + "tpq": "Tukpa", + "tpr": "Tuparí", + "tpt": "Tlachichilco Tepehua", + "tpu": "Tampuan", + "tpv": "Tanapag", + "tpw": "Tupí", + "tpx": "Acatepec Me'phaa; Acatepec Tlapanec", + "tpy": "Trumai", + "tpz": "Tinputz", + "tqb": "Tembé", + "tql": "Lehali", + "tqm": "Turumsa", + "tqn": "Tenino", + "tqo": "Toaripi", + "tqp": "Tomoip", + "tqq": "Tunni", + "tqr": "Torona", + "tqt": "Western Totonac", + "tqu": "Touo", + "tqw": "Tonkawa", + "tr": "Turkish", + "tra": "Tirahi", + "trb": "Terebu", + "trc": "Copala Triqui", + "trd": "Turi", + "tre": "East Tarangan", + "trf": "Trinidadian Creole English", + "trg": "Lishán Didán", + "trh": "Turaka", + "tri": "Trió", + "trj": "Toram", + "trk": "Turkic languages", + "trl": "Traveller Scottish", + "trm": "Tregami", + "trn": "Trinitario", + "tro": "Tarao Naga", + "trp": "Kok Borok", + "trq": "San Martín Itunyoso Triqui", + "trr": "Taushiro", + "trs": "Chicahuaxtla Triqui", + "trt": "Tunggare", + "tru": "Turoyo; Surayt", + "trv": "Sediq; Seediq; Taroko", + "trw": "Torwali", + "trx": "Tringgus-Sembaan Bidayuh", + "try": "Turung", + "trz": "Torá", + "ts": "Tsonga", + "tsa": "Tsaangi", + "tsb": "Tsamai", + "tsc": "Tswa", + "tsd": "Tsakonian", + "tse": "Tunisian Sign Language", + "tsg": "Tausug", + "tsh": "Tsuvan", + "tsi": "Tsimshian", + "tsj": "Tshangla", + "tsk": "Tseku", + "tsl": "Ts'ün-Lao", + "tsm": "Turkish Sign Language; Türk İşaret Dili", + "tsp": "Northern Toussian", + "tsq": "Thai Sign Language", + "tsr": "Akei", + "tss": "Taiwan Sign Language", + "tst": "Tondi Songway Kiini", + "tsu": "Tsou", + "tsv": "Tsogo", + "tsw": "Tsishingini", + "tsx": "Mubami", + "tsy": "Tebul Sign Language", + "tsz": "Purepecha", + "tt": "Tatar", + "tta": "Tutelo", + "ttb": "Gaa", + "ttc": "Tektiteko", + "ttd": "Tauade", + "tte": "Bwanabwana", + "ttf": "Tuotomb", + "ttg": "Tutong", + "tth": "Upper Ta'oih", + "tti": "Tobati", + "ttj": "Tooro", + "ttk": "Totoro", + "ttl": "Totela", + "ttm": "Northern Tutchone", + "ttn": "Towei", + "tto": "Lower Ta'oih", + "ttp": "Tombelala", + "ttq": "Tawallammat Tamajaq", + "ttr": "Tera", + "tts": "Northeastern Thai", + "ttt": "Muslim Tat", + "ttu": "Torau", + "ttv": "Titan", + "ttw": "Long Wat", + "tty": "Sikaritai", + "ttz": "Tsum", + "tua": "Wiarumus", + "tub": "Tübatulabal", + "tuc": "Mutu", + "tud": "Tuxá", + "tue": "Tuyuca", + "tuf": "Central Tunebo", + "tug": "Tunia", + "tuh": "Taulil", + "tui": "Tupuri", + "tuj": "Tugutil", + "tul": "Tula", + "tum": "Tumbuka", + "tun": "Tunica", + "tuo": "Tucano", + "tup": "Tupi languages", + "tuq": "Tedaga", + "tus": "Tuscarora", + "tut": "Altaic languages", + "tuu": "Tututni", + "tuv": "Turkana", + "tuw": "Tungus languages", + "tux": "Tuxináwa", + "tuy": "Tugen", + "tuz": "Turka", + "tva": "Vaghua", + "tvd": "Tsuvadi", + "tve": "Te'un", + "tvk": "Southeast Ambrym", + "tvl": "Tuvalu", + "tvm": "Tela-Masbuar", + "tvn": "Tavoyan", + "tvo": "Tidore", + "tvs": "Taveta", + "tvt": "Tutsa Naga", + "tvu": "Tunen", + "tvw": "Sedoa", + "tvx": "Taivoan", + "tvy": "Timor Pidgin", + "tw": "Twi", + "twa": "Twana", + "twb": "Western Tawbuid", + "twc": "Teshenawa", + "twd": "Twents", + "twe": "Tewa (Indonesia)", + "twf": "Northern Tiwa", + "twg": "Tereweng", + "twh": "Tai Dón", + "twl": "Tawara", + "twm": "Tawang Monpa", + "twn": "Twendi", + "two": "Tswapong", + "twp": "Ere", + "twq": "Tasawaq", + "twr": "Southwestern Tarahumara", + "twt": "Turiwára", + "twu": "Termanu", + "tww": "Tuwari", + "twx": "Tewe", + "twy": "Tawoyan", + "txa": "Tombonuo", + "txb": "Tokharian B", + "txc": "Tsetsaut", + "txe": "Totoli", + "txg": "Tangut", + "txh": "Thracian", + "txi": "Ikpeng", + "txj": "Tarjumo", + "txm": "Tomini", + "txn": "West Tarangan", + "txo": "Toto", + "txq": "Tii", + "txr": "Tartessian", + "txs": "Tonsea", + "txt": "Citak", + "txu": "Kayapó", + "txx": "Tatana", + "txy": "Tanosy Malagasy", + "ty": "Tahitian", + "tya": "Tauya", + "tye": "Kyanga", + "tyh": "O'du", + "tyi": "Teke-Tsaayi", + "tyj": "Tai Do; Tai Yo", + "tyl": "Thu Lao", + "tyn": "Kombai", + "typ": "Thaypan", + "tyr": "Tai Daeng", + "tys": "Tày Sa Pa", + "tyt": "Tày Tac", + "tyu": "Kua", + "tyv": "Tuvinian", + "tyx": "Teke-Tyee", + "tyy": "Tiyaa", + "tyz": "Tày", + "tza": "Tanzanian Sign Language", + "tzh": "Tzeltal", + "tzj": "Tz'utujil", + "tzl": "Talossan", + "tzm": "Central Atlas Tamazight", + "tzn": "Tugun", + "tzo": "Tzotzil", + "tzx": "Tabriak", + "uam": "Uamué", + "uan": "Kuan", + "uar": "Tairuma", + "uba": "Ubang", + "ubi": "Ubi", + "ubl": "Buhi'non Bikol", + "ubr": "Ubir", + "ubu": "Umbu-Ungu", + "uby": "Ubykh", + "uda": "Uda", + "ude": "Udihe", + "udg": "Muduga", + "udi": "Udi", + "udj": "Ujir", + "udl": "Wuzlam", + "udm": "Udmurt", + "udu": "Uduk", + "ues": "Kioko", + "ufi": "Ufim", + "ug": "Uighur; Uyghur", + "uga": "Ugaritic", + "ugb": "Kuku-Ugbanh", + "uge": "Ughele", + "ugh": "Kubachi", + "ugn": "Ugandan Sign Language", + "ugo": "Ugong", + "ugy": "Uruguayan Sign Language", + "uha": "Uhami", + "uhn": "Damal", + "uis": "Uisai", + "uiv": "Iyive", + "uji": "Tanjijili", + "uk": "Ukrainian", + "uka": "Kaburi", + "ukg": "Ukuriguma", + "ukh": "Ukhwejo", + "uki": "Kui (India)", + "ukk": "Muak Sa-aak", + "ukl": "Ukrainian Sign Language", + "ukp": "Ukpe-Bayobiri", + "ukq": "Ukwa", + "uks": "Urubú-Kaapor Sign Language; Kaapor Sign Language", + "uku": "Ukue", + "ukv": "Kuku", + "ukw": "Ukwuani-Aboh-Ndoni", + "uky": "Kuuk-Yak", + "ula": "Fungwa", + "ulb": "Ulukwumi", + "ulc": "Ulch", + "ule": "Lule", + "ulf": "Usku; Afra", + "uli": "Ulithian", + "ulk": "Meriam Mir", + "ull": "Ullatan", + "ulm": "Ulumanda'", + "uln": "Unserdeutsch", + "ulu": "Uma' Lung", + "ulw": "Ulwa", + "uma": "Umatilla", + "umb": "Umbundu", + "umc": "Marrucinian", + "umd": "Umbindhamu", + "umg": "Morrobalama; Umbuygamu", + "umi": "Ukit", + "umm": "Umon", + "umn": "Makyan Naga", + "umo": "Umotína", + "ump": "Umpila", + "umr": "Umbugarla", + "ums": "Pendau", + "umu": "Munsee", + "una": "North Watut", + "und": "Undetermined", + "une": "Uneme", + "ung": "Ngarinyin", + "uni": "Uni", + "unk": "Enawené-Nawé", + "unm": "Unami", + "unn": "Kurnai", + "unr": "Mundari", + "unu": "Unubahe", + "unx": "Munda", + "unz": "Unde Kaili", + "uon": "Kulon", + "upi": "Umeda", + "upv": "Uripiv-Wala-Rano-Atchin", + "ur": "Urdu", + "ura": "Urarina", + "urb": "Urubú-Kaapor; Kaapor", + "urc": "Urningangg", + "ure": "Uru", + "urf": "Uradhi", + "urg": "Urigina", + "urh": "Urhobo", + "uri": "Urim", + "urj": "Uralic languages", + "urk": "Urak Lawoi'", + "url": "Urali", + "urm": "Urapmin", + "urn": "Uruangnirin", + "uro": "Ura (Papua New Guinea)", + "urp": "Uru-Pa-In", + "urr": "Lehalurup; Löyöp", + "urt": "Urat", + "uru": "Urumi", + "urv": "Uruava", + "urw": "Sop", + "urx": "Urimo", + "ury": "Orya", + "urz": "Uru-Eu-Wau-Wau", + "usa": "Usarufa", + "ush": "Ushojo", + "usi": "Usui", + "usk": "Usaghade", + "usp": "Uspanteco", + "uss": "us-Saare", + "usu": "Uya", + "uta": "Otank", + "ute": "Ute-Southern Paiute", + "uth": "ut-Hun", + "utp": "Amba (Solomon Islands)", + "utr": "Etulo", + "utu": "Utu", + "uum": "Urum", + "uur": "Ura (Vanuatu)", + "uuu": "U", + "uve": "West Uvean; Fagauvea", + "uvh": "Uri", + "uvl": "Lote", + "uwa": "Kuku-Uwanh", + "uya": "Doko-Uyanga", + "uz": "Uzbek", + "uzn": "Northern Uzbek", + "uzs": "Southern Uzbek", + "vaa": "Vaagri Booli", + "vae": "Vale", + "vaf": "Vafsi", + "vag": "Vagla", + "vah": "Varhadi-Nagpuri", + "vai": "Vai", + "vaj": "Sekele; Northwestern ǃKung; Vasekele", + "val": "Vehes", + "vam": "Vanimo", + "van": "Valman", + "vao": "Vao", + "vap": "Vaiphei", + "var": "Huarijio", + "vas": "Vasavi", + "vau": "Vanuma", + "vav": "Varli", + "vay": "Wayu", + "vbb": "Southeast Babar", + "vbk": "Southwestern Bontok", + "ve": "Venda", + "vec": "Venetian", + "ved": "Veddah", + "vel": "Veluws", + "vem": "Vemgo-Mabas", + "veo": "Ventureño", + "vep": "Veps", + "ver": "Mom Jango", + "vgr": "Vaghri", + "vgt": "Vlaamse Gebarentaal; Flemish Sign Language", + "vi": "Vietnamese", + "vic": "Virgin Islands Creole English", + "vid": "Vidunda", + "vif": "Vili", + "vig": "Viemo", + "vil": "Vilela", + "vin": "Vinza", + "vis": "Vishavan", + "vit": "Viti", + "viv": "Iduna", + "vka": "Kariyarra", + "vkj": "Kujarge", + "vkk": "Kaur", + "vkl": "Kulisusu", + "vkm": "Kamakan", + "vkn": "Koro Nulu", + "vko": "Kodeoha", + "vkp": "Korlai Creole Portuguese", + "vkt": "Tenggarong Kutai Malay", + "vku": "Kurrama", + "vkz": "Koro Zuba", + "vlp": "Valpei", + "vls": "Vlaams", + "vma": "Martuyhunira", + "vmb": "Barbaram", + "vmc": "Juxtlahuaca Mixtec", + "vmd": "Mudu Koraga", + "vme": "East Masela", + "vmf": "Mainfränkisch", + "vmg": "Lungalunga", + "vmh": "Maraghei", + "vmi": "Miwa", + "vmj": "Ixtayutla Mixtec", + "vmk": "Makhuwa-Shirima", + "vml": "Malgana", + "vmm": "Mitlatongo Mixtec", + "vmp": "Soyaltepec Mazatec", + "vmq": "Soyaltepec Mixtec", + "vmr": "Marenje", + "vms": "Moksela", + "vmu": "Muluridyi", + "vmv": "Valley Maidu", + "vmw": "Makhuwa", + "vmx": "Tamazola Mixtec", + "vmy": "Ayautla Mazatec", + "vmz": "Mazatlán Mazatec", + "vnk": "Vano; Lovono", + "vnm": "Vinmavis; Neve'ei", + "vnp": "Vunapu", + "vo": "Volapük", + "vor": "Voro", + "vot": "Votic", + "vra": "Vera'a", + "vro": "Võro", + "vrs": "Varisi", + "vrt": "Burmbar; Banam Bay", + "vsi": "Moldova Sign Language", + "vsl": "Venezuelan Sign Language", + "vsv": "Valencian Sign Language; Llengua de signes valenciana", + "vto": "Vitou", + "vum": "Vumbu", + "vun": "Vunjo", + "vut": "Vute", + "vwa": "Awa (China)", + "wa": "Walloon", + "waa": "Walla Walla", + "wab": "Wab", + "wac": "Wasco-Wishram", + "wad": "Wamesa; Wondama", + "wae": "Walser", + "waf": "Wakoná", + "wag": "Wa'ema", + "wah": "Watubela", + "wai": "Wares", + "waj": "Waffa", + "wak": "Wakashan languages", + "wal": "Wolaytta; Wolaitta", + "wam": "Wampanoag", + "wan": "Wan", + "wao": "Wappo", + "wap": "Wapishana", + "waq": "Wagiman", + "war": "Waray (Philippines)", + "was": "Washo", + "wat": "Kaninuwa", + "wau": "Waurá", + "wav": "Waka", + "waw": "Waiwai", + "wax": "Watam; Marangis", + "way": "Wayana", + "waz": "Wampur", + "wba": "Warao", + "wbb": "Wabo", + "wbe": "Waritai", + "wbf": "Wara", + "wbh": "Wanda", + "wbi": "Vwanji", + "wbj": "Alagwa", + "wbk": "Waigali", + "wbl": "Wakhi", + "wbm": "Wa", + "wbp": "Warlpiri", + "wbq": "Waddar", + "wbr": "Wagdi", + "wbs": "West Bengal Sign Language", + "wbt": "Warnman", + "wbv": "Wajarri", + "wbw": "Woi", + "wca": "Yanomámi", + "wci": "Waci Gbe", + "wdd": "Wandji", + "wdg": "Wadaginam", + "wdj": "Wadjiginy", + "wdk": "Wadikali", + "wdt": "Wendat", + "wdu": "Wadjigu", + "wdy": "Wadjabangayi", + "wea": "Wewaw", + "wec": "Wè Western", + "wed": "Wedau", + "weg": "Wergaia", + "weh": "Weh", + "wei": "Kiunum", + "wem": "Weme Gbe", + "wen": "Sorbian languages", + "weo": "Wemale", + "wep": "Westphalien", + "wer": "Weri", + "wes": "Cameroon Pidgin", + "wet": "Perai", + "weu": "Rawngtu Chin", + "wew": "Wejewa", + "wfg": "Yafi; Zorop", + "wga": "Wagaya", + "wgb": "Wagawaga", + "wgg": "Wangkangurru; Wangganguru", + "wgi": "Wahgi", + "wgo": "Waigeo", + "wgu": "Wirangu", + "wgy": "Warrgamay", + "wha": "Sou Upaa; Manusela", + "whg": "North Wahgi", + "whk": "Wahau Kenyah", + "whu": "Wahau Kayan", + "wib": "Southern Toussian", + "wic": "Wichita", + "wie": "Wik-Epa", + "wif": "Wik-Keyangan", + "wig": "Wik Ngathan", + "wih": "Wik-Me'anha", + "wii": "Minidien", + "wij": "Wik-Iiyanh", + "wik": "Wikalkan", + "wil": "Wilawila", + "wim": "Wik-Mungkan", + "win": "Ho-Chunk", + "wir": "Wiraféd", + "wiu": "Wiru", + "wiv": "Vitu", + "wiy": "Wiyot", + "wja": "Waja", + "wji": "Warji", + "wka": "Kw'adza", + "wkb": "Kumbaran", + "wkd": "Wakde; Mo", + "wkl": "Kalanadi", + "wkr": "Keerray-Woorroong", + "wku": "Kunduvadi", + "wkw": "Wakawaka", + "wky": "Wangkayutyuru", + "wla": "Walio", + "wlc": "Mwali Comorian", + "wle": "Wolane", + "wlg": "Kunbarlang", + "wlh": "Welaun", + "wli": "Waioli", + "wlk": "Wailaki", + "wll": "Wali (Sudan)", + "wlm": "Middle Welsh", + "wlo": "Wolio", + "wlr": "Wailapa", + "wls": "Wallisian", + "wlu": "Wuliwuli", + "wlv": "Wichí Lhamtés Vejoz", + "wlw": "Walak", + "wlx": "Wali (Ghana)", + "wly": "Waling", + "wma": "Mawa (Nigeria)", + "wmb": "Wambaya", + "wmc": "Wamas", + "wmd": "Mamaindé", + "wme": "Wambule", + "wmg": "Western Minyag", + "wmh": "Waima'a", + "wmi": "Wamin", + "wmm": "Maiwa (Indonesia)", + "wmn": "Waamwang", + "wmo": "Wom (Papua New Guinea)", + "wms": "Wambon", + "wmt": "Walmajarri", + "wmw": "Mwani", + "wmx": "Womo", + "wnb": "Wanambre", + "wnc": "Wantoat", + "wnd": "Wandarang", + "wne": "Waneci", + "wng": "Wanggom", + "wni": "Ndzwani Comorian", + "wnk": "Wanukaka", + "wnm": "Wanggamala", + "wnn": "Wunumara", + "wno": "Wano", + "wnp": "Wanap", + "wnu": "Usan", + "wnw": "Wintu", + "wny": "Wanyi; Waanyi", + "wo": "Wolof", + "woa": "Kuwema; Tyaraity", + "wob": "Wè Northern", + "woc": "Wogeo", + "wod": "Wolani", + "woe": "Woleaian", + "wof": "Gambian Wolof", + "wog": "Wogamusin", + "woi": "Kamang", + "wok": "Longto", + "wom": "Wom (Nigeria)", + "won": "Wongo", + "woo": "Manombai", + "wor": "Woria", + "wos": "Hanga Hundi", + "wow": "Wawonii", + "woy": "Weyto", + "wpc": "Maco", + "wrb": "Waluwarra; Warluwara", + "wrg": "Warungu; Gudjal", + "wrh": "Wiradjuri", + "wri": "Wariyangga", + "wrk": "Garrwa", + "wrl": "Warlmanpa", + "wrm": "Warumungu", + "wrn": "Warnang", + "wro": "Worrorra", + "wrp": "Waropen", + "wrr": "Wardaman", + "wrs": "Waris", + "wru": "Waru", + "wrv": "Waruna", + "wrw": "Gugu Warra", + "wrx": "Wae Rana", + "wry": "Merwari", + "wrz": "Waray (Australia)", + "wsa": "Warembori", + "wsg": "Adilabad Gondi", + "wsi": "Wusi", + "wsk": "Waskia", + "wsr": "Owenia", + "wss": "Wasa", + "wsu": "Wasu", + "wsv": "Wotapuri-Katarqalai", + "wtf": "Watiwa", + "wth": "Wathawurrung", + "wti": "Berta", + "wtk": "Watakataui", + "wtm": "Mewati", + "wtw": "Wotu", + "wua": "Wikngenchera", + "wub": "Wunambal", + "wud": "Wudu", + "wuh": "Wutunhua", + "wul": "Silimo", + "wum": "Wumbvu", + "wun": "Bungu", + "wur": "Wurrugu", + "wut": "Wutung", + "wuu": "Wu Chinese", + "wuv": "Wuvulu-Aua", + "wux": "Wulna", + "wuy": "Wauyai", + "wwa": "Waama", + "wwb": "Wakabunga", + "wwo": "Wetamut; Dorig", + "wwr": "Warrwa", + "www": "Wawa", + "wxa": "Waxianghua", + "wxw": "Wardandi", + "wyb": "Wangaaybuwan-Ngiyambaa", + "wyi": "Woiwurrung", + "wym": "Wymysorys", + "wyn": "Wyandot", + "wyr": "Wayoró", + "wyy": "Western Fijian", + "xaa": "Andalusian Arabic", + "xab": "Sambe", + "xac": "Kachari", + "xad": "Adai", + "xae": "Aequian", + "xag": "Aghwan", + "xai": "Kaimbé", + "xaj": "Ararandewára", + "xak": "Máku", + "xal": "Kalmyk; Oirat", + "xam": "ǀXam", + "xan": "Xamtanga", + "xao": "Khao", + "xap": "Apalachee", + "xaq": "Aquitanian", + "xar": "Karami", + "xas": "Kamas", + "xat": "Katawixi", + "xau": "Kauwera", + "xav": "Xavánte", + "xaw": "Kawaiisu", + "xay": "Kayan Mahakam", + "xbb": "Lower Burdekin", + "xbc": "Bactrian", + "xbd": "Bindal", + "xbe": "Bigambal", + "xbg": "Bunganditj", + "xbi": "Kombio", + "xbj": "Birrpayi", + "xbm": "Middle Breton", + "xbn": "Kenaboi", + "xbo": "Bolgarian", + "xbp": "Bibbulman", + "xbr": "Kambera", + "xbw": "Kambiwá", + "xby": "Batjala; Batyala", + "xcb": "Cumbric", + "xcc": "Camunic", + "xce": "Celtiberian", + "xcg": "Cisalpine Gaulish", + "xch": "Chemakum; Chimakum", + "xcl": "Classical Armenian", + "xcm": "Comecrudo", + "xcn": "Cotoname", + "xco": "Chorasmian", + "xcr": "Carian", + "xct": "Classical Tibetan", + "xcu": "Curonian", + "xcv": "Chuvantsy", + "xcw": "Coahuilteco", + "xcy": "Cayuse", + "xda": "Darkinyung", + "xdc": "Dacian", + "xdk": "Dharuk", + "xdm": "Edomite", + "xdo": "Kwandu", + "xdq": "Kaitag", + "xdy": "Malayic Dayak", + "xeb": "Eblan", + "xed": "Hdi", + "xeg": "ǁXegwi", + "xel": "Kelo", + "xem": "Kembayan", + "xep": "Epi-Olmec", + "xer": "Xerénte", + "xes": "Kesawai", + "xet": "Xetá", + "xeu": "Keoru-Ahia", + "xfa": "Faliscan", + "xga": "Galatian", + "xgb": "Gbin", + "xgd": "Gudang", + "xgf": "Gabrielino-Fernandeño", + "xgg": "Goreng", + "xgi": "Garingbal", + "xgl": "Galindan", + "xgm": "Dharumbal; Guwinmal", + "xgn": "Mongolian languages", + "xgr": "Garza", + "xgu": "Unggumi", + "xgw": "Guwa", + "xh": "Xhosa", + "xha": "Harami", + "xhc": "Hunnic", + "xhd": "Hadrami", + "xhe": "Khetrani", + "xhm": "Middle Khmer (1400 to 1850 CE)", + "xhr": "Hernican", + "xht": "Hattic", + "xhu": "Hurrian", + "xhv": "Khua", + "xib": "Iberian", + "xii": "Xiri", + "xil": "Illyrian", + "xin": "Xinca", + "xir": "Xiriâna", + "xis": "Kisan", + "xiv": "Indus Valley Language", + "xiy": "Xipaya", + "xjb": "Minjungbal", + "xjt": "Jaitmatang", + "xka": "Kalkoti", + "xkb": "Northern Nago", + "xkc": "Kho'ini", + "xkd": "Mendalam Kayan", + "xke": "Kereho", + "xkf": "Khengkha", + "xkg": "Kagoro", + "xki": "Kenyan Sign Language", + "xkj": "Kajali", + "xkk": "Kachok; Kaco'", + "xkl": "Mainstream Kenyah", + "xkn": "Kayan River Kayan", + "xko": "Kiorr", + "xkp": "Kabatei", + "xkq": "Koroni", + "xkr": "Xakriabá", + "xks": "Kumbewaha", + "xkt": "Kantosi", + "xku": "Kaamba", + "xkv": "Kgalagadi", + "xkw": "Kembra", + "xkx": "Karore", + "xky": "Uma' Lasan", + "xkz": "Kurtokha", + "xla": "Kamula", + "xlb": "Loup B", + "xlc": "Lycian", + "xld": "Lydian", + "xle": "Lemnian", + "xlg": "Ligurian (Ancient)", + "xli": "Liburnian", + "xln": "Alanic", + "xlo": "Loup A", + "xlp": "Lepontic", + "xls": "Lusitanian", + "xlu": "Cuneiform Luwian", + "xly": "Elymian", + "xma": "Mushungulu", + "xmb": "Mbonga", + "xmc": "Makhuwa-Marrevone", + "xmd": "Mbudum", + "xme": "Median", + "xmf": "Mingrelian", + "xmg": "Mengaka", + "xmh": "Kugu-Muminh", + "xmj": "Majera", + "xmk": "Ancient Macedonian", + "xml": "Malaysian Sign Language", + "xmm": "Manado Malay", + "xmn": "Manichaean Middle Persian", + "xmo": "Morerebi", + "xmp": "Kuku-Mu'inh", + "xmq": "Kuku-Mangk", + "xmr": "Meroitic", + "xms": "Moroccan Sign Language", + "xmt": "Matbat", + "xmu": "Kamu", + "xmv": "Antankarana Malagasy; Tankarana Malagasy", + "xmw": "Tsimihety Malagasy", + "xmx": "Salawati; Maden", + "xmy": "Mayaguduna", + "xmz": "Mori Bawah", + "xna": "Ancient North Arabian", + "xnb": "Kanakanabu", + "xnd": "Na-Dene languages", + "xng": "Middle Mongolian", + "xnh": "Kuanhua", + "xni": "Ngarigu", + "xnj": "Ngoni (Tanzania)", + "xnk": "Nganakarti", + "xnm": "Ngumbarl", + "xnn": "Northern Kankanay", + "xno": "Anglo-Norman", + "xnq": "Ngoni (Mozambique)", + "xnr": "Kangri", + "xns": "Kanashi", + "xnt": "Narragansett", + "xnu": "Nukunul", + "xny": "Nyiyaparli", + "xnz": "Kenzi; Mattoki", + "xoc": "O'chi'chi'", + "xod": "Kokoda", + "xog": "Soga", + "xoi": "Kominimung", + "xok": "Xokleng", + "xom": "Komo (Sudan)", + "xon": "Konkomba", + "xoo": "Xukurú", + "xop": "Kopar", + "xor": "Korubo", + "xow": "Kowaki", + "xpa": "Pirriya", + "xpb": "Northeastern Tasmanian; Pyemmairrener", + "xpc": "Pecheneg", + "xpd": "Oyster Bay Tasmanian", + "xpe": "Liberia Kpelle", + "xpf": "Southeast Tasmanian; Nuenonne", + "xpg": "Phrygian", + "xph": "North Midlands Tasmanian; Tyerrenoterpanner", + "xpi": "Pictish", + "xpj": "Mpalitjanh", + "xpk": "Kulina Pano", + "xpl": "Port Sorell Tasmanian", + "xpm": "Pumpokol", + "xpn": "Kapinawá", + "xpo": "Pochutec", + "xpp": "Puyo-Paekche", + "xpq": "Mohegan-Pequot", + "xpr": "Parthian", + "xps": "Pisidian", + "xpt": "Punthamara", + "xpu": "Punic", + "xpv": "Northern Tasmanian; Tommeginne", + "xpw": "Northwestern Tasmanian; Peerapper", + "xpx": "Southwestern Tasmanian; Toogee", + "xpy": "Puyo", + "xpz": "Bruny Island Tasmanian", + "xqa": "Karakhanid", + "xqt": "Qatabanian", + "xra": "Krahô", + "xrb": "Eastern Karaboro", + "xrd": "Gundungurra", + "xre": "Kreye", + "xrg": "Minang", + "xri": "Krikati-Timbira", + "xrm": "Armazic", + "xrn": "Arin", + "xrr": "Raetic", + "xrt": "Aranama-Tamique", + "xru": "Marriammu", + "xrw": "Karawa", + "xsa": "Sabaean", + "xsb": "Sambal", + "xsc": "Scythian", + "xsd": "Sidetic", + "xse": "Sempan", + "xsh": "Shamang", + "xsi": "Sio", + "xsj": "Subi", + "xsl": "South Slavey", + "xsm": "Kasem", + "xsn": "Sanga (Nigeria)", + "xso": "Solano", + "xsp": "Silopi", + "xsq": "Makhuwa-Saka", + "xsr": "Sherpa", + "xss": "Assan", + "xsu": "Sanumá", + "xsv": "Sudovian", + "xsy": "Saisiyat", + "xta": "Alcozauca Mixtec", + "xtb": "Chazumba Mixtec", + "xtc": "Katcha-Kadugli-Miri", + "xtd": "Diuxi-Tilantongo Mixtec", + "xte": "Ketengban", + "xtg": "Transalpine Gaulish", + "xth": "Yitha Yitha", + "xti": "Sinicahua Mixtec", + "xtj": "San Juan Teita Mixtec", + "xtl": "Tijaltepec Mixtec", + "xtm": "Magdalena Peñasco Mixtec", + "xtn": "Northern Tlaxiaco Mixtec", + "xto": "Tokharian A", + "xtp": "San Miguel Piedras Mixtec", + "xtq": "Tumshuqese", + "xtr": "Early Tripuri", + "xts": "Sindihui Mixtec", + "xtt": "Tacahua Mixtec", + "xtu": "Cuyamecalco Mixtec", + "xtv": "Thawa", + "xtw": "Tawandê", + "xty": "Yoloxochitl Mixtec", + "xua": "Alu Kurumba", + "xub": "Betta Kurumba", + "xud": "Umiida", + "xug": "Kunigami", + "xuj": "Jennu Kurumba", + "xul": "Ngunawal; Nunukul", + "xum": "Umbrian", + "xun": "Unggaranggu", + "xuo": "Kuo", + "xup": "Upper Umpqua", + "xur": "Urartian", + "xut": "Kuthant", + "xuu": "Kxoe; Khwedam", + "xve": "Venetic", + "xvi": "Kamviri", + "xvn": "Vandalic", + "xvo": "Volscian", + "xvs": "Vestinian", + "xwa": "Kwaza", + "xwc": "Woccon", + "xwd": "Wadi Wadi", + "xwe": "Xwela Gbe", + "xwg": "Kwegu", + "xwj": "Wajuk", + "xwk": "Wangkumara", + "xwl": "Western Xwla Gbe", + "xwo": "Written Oirat", + "xwr": "Kwerba Mamberamo", + "xwt": "Wotjobaluk", + "xww": "Wemba Wemba", + "xxb": "Boro (Ghana)", + "xxk": "Ke'o", + "xxm": "Minkin", + "xxr": "Koropó", + "xxt": "Tambora", + "xya": "Yaygir", + "xyb": "Yandjibara", + "xyj": "Mayi-Yapi", + "xyk": "Mayi-Kulan", + "xyl": "Yalakalore", + "xyt": "Mayi-Thakurti", + "xyy": "Yorta Yorta", + "xzh": "Zhang-Zhung", + "xzm": "Zemgalian", + "xzp": "Ancient Zapotec", + "yaa": "Yaminahua", + "yab": "Yuhup", + "yac": "Pass Valley Yali", + "yad": "Yagua", + "yae": "Pumé", + "yaf": "Yaka (Democratic Republic of Congo)", + "yag": "Yámana", + "yah": "Yazgulyam", + "yai": "Yagnobi", + "yaj": "Banda-Yangere", + "yak": "Yakama", + "yal": "Yalunka", + "yam": "Yamba", + "yan": "Mayangna", + "yao": "Yao", + "yap": "Yapese", + "yaq": "Yaqui", + "yar": "Yabarana", + "yas": "Nugunu (Cameroon)", + "yat": "Yambeta", + "yau": "Yuwana", + "yav": "Yangben", + "yaw": "Yawalapití", + "yax": "Yauma", + "yay": "Agwagwune", + "yaz": "Lokaa", + "yba": "Yala", + "ybb": "Yemba", + "ybe": "West Yugur", + "ybh": "Yakha", + "ybi": "Yamphu", + "ybj": "Hasha", + "ybk": "Bokha", + "ybl": "Yukuben", + "ybm": "Yaben", + "ybn": "Yabaâna", + "ybo": "Yabong", + "ybx": "Yawiyo", + "yby": "Yaweyuha", + "ych": "Chesu", + "ycl": "Lolopo", + "ycn": "Yucuna", + "ycp": "Chepya", + "yda": "Yanda", + "ydd": "Eastern Yiddish", + "yde": "Yangum Dey", + "ydg": "Yidgha", + "ydk": "Yoidik", + "yea": "Ravula", + "yec": "Yeniche", + "yee": "Yimas", + "yei": "Yeni", + "yej": "Yevanic", + "yel": "Yela", + "yer": "Tarok", + "yes": "Nyankpa", + "yet": "Yetfa", + "yeu": "Yerukula", + "yev": "Yapunda", + "yey": "Yeyi", + "yga": "Malyangapa", + "ygi": "Yiningayi", + "ygl": "Yangum Gel", + "ygm": "Yagomi", + "ygp": "Gepo", + "ygr": "Yagaria", + "ygs": "Yolŋu Sign Language", + "ygu": "Yugul", + "ygw": "Yagwoia", + "yha": "Baha Buyang", + "yhd": "Judeo-Iraqi Arabic", + "yhl": "Hlepho Phowa", + "yhs": "Yan-nhaŋu Sign Language", + "yi": "Yiddish", + "yia": "Yinggarda", + "yif": "Ache", + "yig": "Wusa Nasu", + "yih": "Western Yiddish", + "yii": "Yidiny", + "yij": "Yindjibarndi", + "yik": "Dongshanba Lalo", + "yil": "Yindjilandji", + "yim": "Yimchungru Naga", + "yin": "Riang Lai; Yinchia", + "yip": "Pholo", + "yiq": "Miqie", + "yir": "North Awyu", + "yis": "Yis", + "yit": "Eastern Lalu", + "yiu": "Awu", + "yiv": "Northern Nisu", + "yix": "Axi Yi", + "yiz": "Azhe", + "yka": "Yakan", + "ykg": "Northern Yukaghir", + "yki": "Yoke", + "ykk": "Yakaikeke", + "ykl": "Khlula", + "ykm": "Kap", + "ykn": "Kua-nsi", + "yko": "Yasa", + "ykr": "Yekora", + "ykt": "Kathu", + "yku": "Kuamasi", + "yky": "Yakoma", + "yla": "Yaul", + "ylb": "Yaleba", + "yle": "Yele", + "ylg": "Yelogu", + "yli": "Angguruk Yali", + "yll": "Yil", + "ylm": "Limi", + "yln": "Langnian Buyang", + "ylo": "Naluo Yi", + "ylr": "Yalarnnga", + "ylu": "Aribwaung", + "yly": "Nyâlayu; Nyelâyu", + "ymb": "Yambes", + "ymc": "Southern Muji", + "ymd": "Muda", + "yme": "Yameo", + "ymg": "Yamongeri", + "ymh": "Mili", + "ymi": "Moji", + "ymk": "Makwe", + "yml": "Iamalele", + "ymm": "Maay", + "ymn": "Yamna; Sunum", + "ymo": "Yangum Mon", + "ymp": "Yamap", + "ymq": "Qila Muji", + "ymr": "Malasar", + "yms": "Mysian", + "ymx": "Northern Muji", + "ymz": "Muzi", + "yna": "Aluo", + "ynd": "Yandruwandha", + "yne": "Lang'e", + "yng": "Yango", + "ynk": "Naukan Yupik", + "ynl": "Yangulam", + "ynn": "Yana", + "yno": "Yong", + "ynq": "Yendang", + "yns": "Yansi", + "ynu": "Yahuna", + "yo": "Yoruba", + "yob": "Yoba", + "yog": "Yogad", + "yoi": "Yonaguni", + "yok": "Yokuts", + "yol": "Yola", + "yom": "Yombe", + "yon": "Yongkom", + "yot": "Yotti", + "yox": "Yoron", + "yoy": "Yoy", + "ypa": "Phala", + "ypb": "Labo Phowa", + "ypg": "Phola", + "yph": "Phupha", + "ypk": "Yupik languages", + "ypm": "Phuma", + "ypn": "Ani Phowa", + "ypo": "Alo Phola", + "ypp": "Phupa", + "ypz": "Phuza", + "yra": "Yerakai", + "yrb": "Yareba", + "yre": "Yaouré", + "yrk": "Nenets", + "yrl": "Nhengatu", + "yrm": "Yirrk-Mel", + "yrn": "Yerong", + "yro": "Yaroamë", + "yrs": "Yarsun", + "yrw": "Yarawata", + "yry": "Yarluyandi", + "ysc": "Yassic", + "ysd": "Samatao", + "ysg": "Sonaga", + "ysl": "Yugoslavian Sign Language", + "ysm": "Myanmar Sign Language", + "ysn": "Sani", + "yso": "Nisi (China)", + "ysp": "Southern Lolopo", + "ysr": "Sirenik Yupik", + "yss": "Yessan-Mayo", + "ysy": "Sanie", + "yta": "Talu", + "ytl": "Tanglang", + "ytp": "Thopho", + "ytw": "Yout Wam", + "yty": "Yatay", + "yua": "Yucateco; Yucatec Maya", + "yub": "Yugambal", + "yuc": "Yuchi", + "yud": "Judeo-Tripolitanian Arabic", + "yue": "Yue Chinese; Cantonese", + "yuf": "Havasupai-Walapai-Yavapai", + "yug": "Yug", + "yui": "Yurutí", + "yuj": "Karkar-Yuri", + "yuk": "Yuki", + "yul": "Yulu", + "yum": "Quechan", + "yun": "Bena (Nigeria)", + "yup": "Yukpa", + "yuq": "Yuqui", + "yur": "Yurok", + "yut": "Yopno", + "yuw": "Yau (Morobe Province)", + "yux": "Southern Yukaghir", + "yuy": "East Yugur", + "yuz": "Yuracare", + "yva": "Yawa", + "yvt": "Yavitero", + "ywa": "Kalou", + "ywg": "Yinhawangka", + "ywl": "Western Lalu", + "ywn": "Yawanawa", + "ywq": "Wuding-Luquan Yi", + "ywr": "Yawuru", + "ywt": "Xishanba Lalo; Central Lalo", + "ywu": "Wumeng Nasu", + "yww": "Yawarawarga", + "yxa": "Mayawali", + "yxg": "Yagara", + "yxl": "Yardliyawarra", + "yxm": "Yinwum", + "yxu": "Yuyu", + "yxy": "Yabula Yabula", + "yyr": "Yir Yoront", + "yyu": "Yau (Sandaun Province)", + "yyz": "Ayizi", + "yzg": "E'ma Buyang", + "yzk": "Zokhuo", + "za": "Zhuang; Chuang", + "zaa": "Sierra de Juárez Zapotec", + "zab": "Western Tlacolula Valley Zapotec; San Juan Guelavía Zapotec", + "zac": "Ocotlán Zapotec", + "zad": "Cajonos Zapotec", + "zae": "Yareni Zapotec", + "zaf": "Ayoquesco Zapotec", + "zag": "Zaghawa", + "zah": "Zangwal", + "zai": "Isthmus Zapotec", + "zaj": "Zaramo", + "zak": "Zanaki", + "zal": "Zauzou", + "zam": "Miahuatlán Zapotec", + "zao": "Ozolotepec Zapotec", + "zap": "Zapotec", + "zaq": "Aloápam Zapotec", + "zar": "Rincón Zapotec", + "zas": "Santo Domingo Albarradas Zapotec", + "zat": "Tabaa Zapotec", + "zau": "Zangskari", + "zav": "Yatzachi Zapotec", + "zaw": "Mitla Zapotec", + "zax": "Xadani Zapotec", + "zay": "Zayse-Zergulla; Zaysete", + "zaz": "Zari", + "zba": "Balaibalan", + "zbc": "Central Berawan", + "zbe": "East Berawan", + "zbl": "Blissymbols; Bliss; Blissymbolics", + "zbt": "Batui", + "zbu": "Bu (Bauchi State)", + "zbw": "West Berawan", + "zca": "Coatecas Altas Zapotec", + "zcd": "Las Delicias Zapotec", + "zch": "Central Hongshuihe Zhuang", + "zdj": "Ngazidja Comorian", + "zea": "Zeeuws", + "zeg": "Zenag", + "zeh": "Eastern Hongshuihe Zhuang", + "zen": "Zenaga", + "zga": "Kinga", + "zgb": "Guibei Zhuang", + "zgh": "Standard Moroccan Tamazight", + "zgm": "Minz Zhuang", + "zgn": "Guibian Zhuang", + "zgr": "Magori", + "zh": "Chinese", + "zhb": "Zhaba", + "zhd": "Dai Zhuang", + "zhi": "Zhire", + "zhn": "Nong Zhuang", + "zhw": "Zhoa", + "zhx": "Chinese (family)", + "zia": "Zia", + "zib": "Zimbabwe Sign Language", + "zik": "Zimakani", + "zil": "Zialo", + "zim": "Mesme", + "zin": "Zinza", + "ziw": "Zigula", + "ziz": "Zizilivakan", + "zka": "Kaimbulawa", + "zkb": "Koibal", + "zkd": "Kadu", + "zkg": "Koguryo", + "zkh": "Khorezmian", + "zkk": "Karankawa", + "zkn": "Kanan", + "zko": "Kott", + "zkp": "São Paulo Kaingáng", + "zkr": "Zakhring", + "zkt": "Kitan", + "zku": "Kaurna", + "zkv": "Krevinian", + "zkz": "Khazar", + "zla": "Zula", + "zle": "East Slavic languages", + "zlj": "Liujiang Zhuang", + "zlm": "Malay (individual language)", + "zln": "Lianshan Zhuang", + "zlq": "Liuqian Zhuang", + "zls": "South Slavic languages", + "zlw": "West Slavic languages", + "zma": "Manda (Australia)", + "zmb": "Zimba", + "zmc": "Margany", + "zmd": "Maridan", + "zme": "Mangerr", + "zmf": "Mfinu", + "zmg": "Marti Ke", + "zmh": "Makolkol", + "zmi": "Negeri Sembilan Malay", + "zmj": "Maridjabin", + "zmk": "Mandandanyi", + "zml": "Matngala", + "zmm": "Marimanindji; Marramaninyshi", + "zmn": "Mbangwe", + "zmo": "Molo", + "zmp": "Mpuono", + "zmq": "Mituku", + "zmr": "Maranunggu", + "zms": "Mbesa", + "zmt": "Maringarr", + "zmu": "Muruwari", + "zmv": "Mbariman-Gudhinma", + "zmw": "Mbo (Democratic Republic of Congo)", + "zmx": "Bomitaba", + "zmy": "Mariyedi", + "zmz": "Mbandja", + "zna": "Zan Gula", + "znd": "Zande languages", + "zne": "Zande (individual language)", + "zng": "Mang", + "znk": "Manangkari", + "zns": "Mangas", + "zoc": "Copainalá Zoque", + "zoh": "Chimalapa Zoque", + "zom": "Zou", + "zoo": "Asunción Mixtepec Zapotec", + "zoq": "Tabasco Zoque", + "zor": "Rayón Zoque", + "zos": "Francisco León Zoque", + "zpa": "Lachiguiri Zapotec", + "zpb": "Yautepec Zapotec", + "zpc": "Choapan Zapotec", + "zpd": "Southeastern Ixtlán Zapotec", + "zpe": "Petapa Zapotec", + "zpf": "San Pedro Quiatoni Zapotec", + "zpg": "Guevea De Humboldt Zapotec", + "zph": "Totomachapan Zapotec", + "zpi": "Santa María Quiegolani Zapotec", + "zpj": "Quiavicuzas Zapotec", + "zpk": "Tlacolulita Zapotec", + "zpl": "Lachixío Zapotec", + "zpm": "Mixtepec Zapotec", + "zpn": "Santa Inés Yatzechi Zapotec", + "zpo": "Amatlán Zapotec", + "zpp": "El Alto Zapotec", + "zpq": "Zoogocho Zapotec", + "zpr": "Santiago Xanica Zapotec", + "zps": "Coatlán Zapotec", + "zpt": "San Vicente Coatlán Zapotec", + "zpu": "Yalálag Zapotec", + "zpv": "Chichicapan Zapotec", + "zpw": "Zaniza Zapotec", + "zpx": "San Baltazar Loxicha Zapotec", + "zpy": "Mazaltepec Zapotec", + "zpz": "Texmelucan Zapotec", + "zqe": "Qiubei Zhuang", + "zra": "Kara (Korea)", + "zrg": "Mirgan", + "zrn": "Zerenkel", + "zro": "Záparo", + "zrp": "Zarphatic", + "zrs": "Mairasi", + "zsa": "Sarasira", + "zsk": "Kaskean", + "zsl": "Zambian Sign Language", + "zsm": "Standard Malay", + "zsr": "Southern Rincon Zapotec", + "zsu": "Sukurum", + "zte": "Elotepec Zapotec", + "ztg": "Xanaguía Zapotec", + "ztl": "Lapaguía-Guivini Zapotec", + "ztm": "San Agustín Mixtepec Zapotec", + "ztn": "Santa Catarina Albarradas Zapotec", + "ztp": "Loxicha Zapotec", + "ztq": "Quioquitani-Quierí Zapotec", + "zts": "Tilquiapan Zapotec", + "ztt": "Tejalapan Zapotec", + "ztu": "Güilá Zapotec", + "ztx": "Zaachila Zapotec", + "zty": "Yatee Zapotec", + "zu": "Zulu", + "zua": "Zeem", + "zuh": "Tokano", + "zum": "Kumzari", + "zun": "Zuni", + "zuy": "Zumaya", + "zwa": "Zay", + "zyb": "Yongbei Zhuang", + "zyg": "Yang Zhuang", + "zyj": "Youjiang Zhuang", + "zyn": "Yongnan Zhuang", + "zyp": "Zyphe Chin", + "zza": "Zaza; Dimili; Dimli (macrolanguage); Kirdki; Kirmanjki (macrolanguage); Zazaki", + "zzj": "Zuojiang Zhuang" +} \ No newline at end of file diff --git a/lib/python3.12/site-packages/datasets/utils/resources/multilingualities.json b/lib/python3.12/site-packages/datasets/utils/resources/multilingualities.json new file mode 100644 index 0000000000000000000000000000000000000000..a35c79f03dfcf7c8a116b7fc8ee1b383ab5022fa --- /dev/null +++ b/lib/python3.12/site-packages/datasets/utils/resources/multilingualities.json @@ -0,0 +1,6 @@ +{ + "monolingual": "contains a single language", + "multilingual": "contains multiple languages", + "translation": "contains translated or aligned text", + "other": "other type of language distribution" +} diff --git a/lib/python3.12/site-packages/datasets/utils/resources/readme_structure.yaml b/lib/python3.12/site-packages/datasets/utils/resources/readme_structure.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5b781e11e8258a446874ebf96104f642d0c190cf --- /dev/null +++ b/lib/python3.12/site-packages/datasets/utils/resources/readme_structure.yaml @@ -0,0 +1,116 @@ +name: "" # Filename comes here +allow_empty: false +allow_empty_text: true +subsections: + - name: "Dataset Card for X" # First-level markdown heading + allow_empty: false + allow_empty_text: true + subsections: + - name: "Table of Contents" + allow_empty: false + allow_empty_text: false + subsections: null # meaning it should not be checked. + - name: "Dataset Description" + allow_empty: false + allow_empty_text: false + subsections: + - name: "Dataset Summary" + allow_empty: false + allow_empty_text: false + subsections: null + - name: "Supported Tasks and Leaderboards" + allow_empty: true + allow_empty_text: true + subsections: null + - name: Languages + allow_empty: true + allow_empty_text: true + subsections: null + - name: "Dataset Structure" + allow_empty: false + allow_empty_text: true + subsections: + - name: "Data Instances" + allow_empty: false + allow_empty_text: true + subsections: null + - name: "Data Fields" + allow_empty: false + allow_empty_text: true + subsections: null + - name: "Data Splits" + allow_empty: false + allow_empty_text: true + subsections: null + - name: "Dataset Creation" + allow_empty: false + allow_empty_text: true + subsections: + - name: "Curation Rationale" + allow_empty: true + allow_empty_text: true + subsections: null + - name: "Source Data" + allow_empty: false + allow_empty_text: true + subsections: + - name: "Initial Data Collection and Normalization" + allow_empty: true + allow_empty_text: true + subsections: null + - name: "Who are the source language producers?" + allow_empty: true + allow_empty_text: true + subsections: null + - name: "Annotations" + allow_empty: false + allow_empty_text: true + subsections: + - name: "Annotation process" + allow_empty: true + allow_empty_text: true + subsections: null + - name: "Who are the annotators?" + allow_empty: true + allow_empty_text: true + subsections: null + - name: "Personal and Sensitive Information" + allow_empty: true + allow_empty_text: true + subsections: null + - name: "Considerations for Using the Data" + allow_empty: true + allow_empty_text: true + subsections: + - name: "Social Impact of Dataset" + allow_empty: true + allow_empty_text: true + subsections: null + - name: "Discussion of Biases" + allow_empty: true + allow_empty_text: true + subsections: null + - name: "Other Known Limitations" + allow_empty: true + allow_empty_text: true + subsections: null + - name: "Additional Information" + allow_empty: true + allow_empty_text: true + subsections: + - name: "Dataset Curators" + allow_empty: true + allow_empty_text: true + subsections: null + - name: "Licensing Information" + allow_empty: true + allow_empty_text: true + subsections: null + - name: "Citation Information" + allow_empty: false + allow_empty_text: true + subsections: null + - name: "Contributions" + allow_empty: false + allow_empty_text: false + subsections: null diff --git a/lib/python3.12/site-packages/datasets/utils/resources/size_categories.json b/lib/python3.12/site-packages/datasets/utils/resources/size_categories.json new file mode 100644 index 0000000000000000000000000000000000000000..983ce0c10dbb2e2245f90ae47e9de4c1025d5bb1 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/utils/resources/size_categories.json @@ -0,0 +1,14 @@ +[ + "unknown", + "n<1K", + "1K1T" +] diff --git a/lib/python3.12/site-packages/datasets/utils/sharding.py b/lib/python3.12/site-packages/datasets/utils/sharding.py new file mode 100644 index 0000000000000000000000000000000000000000..753bdf60c2a5b0b23f70893de5ba57a51462d126 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/utils/sharding.py @@ -0,0 +1,92 @@ +import numpy as np + + +def _number_of_shards_in_gen_kwargs(gen_kwargs: dict) -> int: + """Return the number of possible shards according to the input gen_kwargs""" + # Having lists of different sizes makes sharding ambigious, raise an error in this case + # until we decide how to define sharding without ambiguity for users + lists_lengths = {key: len(value) for key, value in gen_kwargs.items() if isinstance(value, list)} + if len(set(lists_lengths.values())) > 1: + raise RuntimeError( + "Sharding is ambiguous for this dataset: " + + "we found several data sources lists of different lengths, and we don't know over which list we should parallelize:\n" + + "\n".join(f"\t- key {key} has length {length}" for key, length in lists_lengths.items()) + + "\nTo fix this, check the 'gen_kwargs' and make sure to use lists only for data sources, " + + "and use tuples otherwise. In the end there should only be one single list, or several lists with the same length." + ) + max_length = max(lists_lengths.values(), default=0) + return max(1, max_length) + + +def _distribute_shards(num_shards: int, max_num_jobs: int) -> list[range]: + """ + Get the range of shard indices per job. + If num_shards>> _distribute_shards(2, max_num_jobs=4) + [range(0, 1), range(1, 2)] + >>> _distribute_shards(10, max_num_jobs=3) + [range(0, 4), range(4, 7), range(7, 10)] + ``` + """ + shards_indices_per_group = [] + for group_idx in range(max_num_jobs): + num_shards_to_add = num_shards // max_num_jobs + (group_idx < (num_shards % max_num_jobs)) + if num_shards_to_add == 0: + break + start = shards_indices_per_group[-1].stop if shards_indices_per_group else 0 + shard_indices = range(start, start + num_shards_to_add) + shards_indices_per_group.append(shard_indices) + return shards_indices_per_group + + +def _split_gen_kwargs(gen_kwargs: dict, max_num_jobs: int) -> list[dict]: + """Split the gen_kwargs into `max_num_job` gen_kwargs""" + # Having lists of different sizes makes sharding ambigious, raise an error in this case + num_shards = _number_of_shards_in_gen_kwargs(gen_kwargs) + if num_shards == 1: + return [dict(gen_kwargs)] + else: + shard_indices_per_group = _distribute_shards(num_shards=num_shards, max_num_jobs=max_num_jobs) + return [ + { + key: [value[shard_idx] for shard_idx in shard_indices_per_group[group_idx]] + if isinstance(value, list) + else value + for key, value in gen_kwargs.items() + } + for group_idx in range(len(shard_indices_per_group)) + ] + + +def _merge_gen_kwargs(gen_kwargs_list: list[dict]) -> dict: + return { + key: [value for gen_kwargs in gen_kwargs_list for value in gen_kwargs[key]] + if isinstance(gen_kwargs_list[0][key], list) + else gen_kwargs_list[0][key] + for key in gen_kwargs_list[0] + } + + +def _shuffle_gen_kwargs(rng: np.random.Generator, gen_kwargs: dict) -> dict: + """Return a shuffled copy of the input gen_kwargs""" + # We must shuffle all the lists, and lists of the same size must have the same shuffling. + # This way entangled lists of (shard, shard_metadata) are still in the right order. + + # First, let's generate the shuffled indices per list size + list_sizes = {len(value) for value in gen_kwargs.values() if isinstance(value, list)} + indices_per_size = {} + for size in list_sizes: + indices_per_size[size] = list(range(size)) + rng.shuffle(indices_per_size[size]) + # Now let's copy the gen_kwargs and shuffle the lists based on their sizes + shuffled_kwargs = dict(gen_kwargs) + for key, value in shuffled_kwargs.items(): + if isinstance(value, list): + shuffled_kwargs[key] = [value[i] for i in indices_per_size[len(value)]] + return shuffled_kwargs diff --git a/lib/python3.12/site-packages/datasets/utils/stratify.py b/lib/python3.12/site-packages/datasets/utils/stratify.py new file mode 100644 index 0000000000000000000000000000000000000000..6dab0469e8168148a81eb2c13f7c103b98d061e7 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/utils/stratify.py @@ -0,0 +1,107 @@ +import numpy as np + + +def approximate_mode(class_counts, n_draws, rng): + """Computes approximate mode of multivariate hypergeometric. + This is an approximation to the mode of the multivariate + hypergeometric given by class_counts and n_draws. + It shouldn't be off by more than one. + It is the mostly likely outcome of drawing n_draws many + samples from the population given by class_counts. + Args + ---------- + class_counts : ndarray of int + Population per class. + n_draws : int + Number of draws (samples to draw) from the overall population. + rng : random state + Used to break ties. + Returns + ------- + sampled_classes : ndarray of int + Number of samples drawn from each class. + np.sum(sampled_classes) == n_draws + + """ + # this computes a bad approximation to the mode of the + # multivariate hypergeometric given by class_counts and n_draws + continuous = n_draws * class_counts / class_counts.sum() + # floored means we don't overshoot n_samples, but probably undershoot + floored = np.floor(continuous) + # we add samples according to how much "left over" probability + # they had, until we arrive at n_samples + need_to_add = int(n_draws - floored.sum()) + if need_to_add > 0: + remainder = continuous - floored + values = np.sort(np.unique(remainder))[::-1] + # add according to remainder, but break ties + # randomly to avoid biases + for value in values: + (inds,) = np.where(remainder == value) + # if we need_to_add less than what's in inds + # we draw randomly from them. + # if we need to add more, we add them all and + # go to the next value + add_now = min(len(inds), need_to_add) + inds = rng.choice(inds, size=add_now, replace=False) + floored[inds] += 1 + need_to_add -= add_now + if need_to_add == 0: + break + return floored.astype(np.int64) + + +def stratified_shuffle_split_generate_indices(y, n_train, n_test, rng, n_splits=10): + """ + + Provides train/test indices to split data in train/test sets. + It's reference is taken from StratifiedShuffleSplit implementation + of scikit-learn library. + + Args + ---------- + + n_train : int, + represents the absolute number of train samples. + + n_test : int, + represents the absolute number of test samples. + + random_state : int or RandomState instance, default=None + Controls the randomness of the training and testing indices produced. + Pass an int for reproducible output across multiple function calls. + + n_splits : int, default=10 + Number of re-shuffling & splitting iterations. + """ + classes, y_indices = np.unique(y, return_inverse=True) + n_classes = classes.shape[0] + class_counts = np.bincount(y_indices) + if np.min(class_counts) < 2: + raise ValueError("Minimum class count error") + if n_train < n_classes: + raise ValueError( + "The train_size = %d should be greater or equal to the number of classes = %d" % (n_train, n_classes) + ) + if n_test < n_classes: + raise ValueError( + "The test_size = %d should be greater or equal to the number of classes = %d" % (n_test, n_classes) + ) + class_indices = np.split(np.argsort(y_indices, kind="mergesort"), np.cumsum(class_counts)[:-1]) + for _ in range(n_splits): + n_i = approximate_mode(class_counts, n_train, rng) + class_counts_remaining = class_counts - n_i + t_i = approximate_mode(class_counts_remaining, n_test, rng) + + train = [] + test = [] + + for i in range(n_classes): + permutation = rng.permutation(class_counts[i]) + perm_indices_class_i = class_indices[i].take(permutation, mode="clip") + train.extend(perm_indices_class_i[: n_i[i]]) + test.extend(perm_indices_class_i[n_i[i] : n_i[i] + t_i[i]]) + train = rng.permutation(train) + test = rng.permutation(test) + + yield train, test diff --git a/lib/python3.12/site-packages/datasets/utils/tf_utils.py b/lib/python3.12/site-packages/datasets/utils/tf_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..10ae3ef64ae5e14339bf0e855c010253b92a7e02 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/utils/tf_utils.py @@ -0,0 +1,564 @@ +# Copyright 2022 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""TF-specific utils import.""" + +import os +import warnings +from functools import partial +from math import ceil +from uuid import uuid4 + +import numpy as np +import pyarrow as pa +from multiprocess import get_context + + +try: + from multiprocess.shared_memory import SharedMemory +except ImportError: + SharedMemory = None # Version checks should prevent this being called on older Python versions + +from .. import config + + +def minimal_tf_collate_fn(features): + if isinstance(features, dict): # case batch_size=None: nothing to collate + return features + elif config.TF_AVAILABLE: + import tensorflow as tf + else: + raise ImportError("Called a Tensorflow-specific function but Tensorflow is not installed.") + + first = features[0] + batch = {} + for k, v in first.items(): + if isinstance(v, np.ndarray): + batch[k] = np.stack([f[k] for f in features]) + elif isinstance(v, tf.Tensor): + batch[k] = tf.stack([f[k] for f in features]) + else: + batch[k] = np.array([f[k] for f in features]) + return batch + + +def minimal_tf_collate_fn_with_renaming(features): + batch = minimal_tf_collate_fn(features) + if "label" in batch: + batch["labels"] = batch["label"] + del batch["label"] + return batch + + +def is_numeric_pa_type(pa_type): + if pa.types.is_list(pa_type): + return is_numeric_pa_type(pa_type.value_type) + return pa.types.is_integer(pa_type) or pa.types.is_floating(pa_type) or pa.types.is_decimal(pa_type) + + +def np_get_batch( + indices, dataset, cols_to_retain, collate_fn, collate_fn_args, columns_to_np_types, return_dict=False +): + if not isinstance(indices, np.ndarray): + indices = indices.numpy() + + is_batched = True + # Optimization - if we're loading a sequential batch, do it with slicing instead of a list of indices + if isinstance(indices, np.integer): + batch = dataset[indices.item()] + is_batched = False + elif np.all(np.diff(indices) == 1): + batch = dataset[indices[0] : indices[-1] + 1] + elif isinstance(indices, np.ndarray): + batch = dataset[indices] + else: + raise RuntimeError(f"Unexpected type for indices: {type(indices)}") + + if cols_to_retain is not None: + batch = { + key: value + for key, value in batch.items() + if key in cols_to_retain or key in ("label", "label_ids", "labels") + } + + if is_batched: + actual_size = len(list(batch.values())[0]) # Get the length of one of the arrays, assume all same + # Our collators expect a list of dicts, not a dict of lists/arrays, so we invert + batch = [{key: value[i] for key, value in batch.items()} for i in range(actual_size)] + batch = collate_fn(batch, **collate_fn_args) + + if return_dict: + out_batch = {} + for col, cast_dtype in columns_to_np_types.items(): + # In case the collate_fn returns something strange + array = np.array(batch[col]) + array = array.astype(cast_dtype) + out_batch[col] = array + else: + out_batch = [] + for col, cast_dtype in columns_to_np_types.items(): + # In case the collate_fn returns something strange + array = np.array(batch[col]) + array = array.astype(cast_dtype) + out_batch.append(array) + return out_batch + + +def dataset_to_tf( + dataset, + cols_to_retain, + collate_fn, + collate_fn_args, + columns_to_np_types, + output_signature, + shuffle, + batch_size, + drop_remainder, +): + """Create a tf.data.Dataset from the underlying Dataset. This is a single-process method - the multiprocess + equivalent is multiprocess_dataset_to_tf. + + Args: + dataset (`Dataset`): Dataset to wrap with tf.data.Dataset. + cols_to_retain (`List[str]`): Dataset column(s) to load in the + tf.data.Dataset. It is acceptable to include column names that are created by the `collate_fn` and + that do not exist in the original dataset. + collate_fn(`Callable`): A function or callable object (such as a `DataCollator`) that will collate + lists of samples into a batch. + collate_fn_args (`Dict`): A `dict` of keyword arguments to be passed to the + `collate_fn`. Can be empty. + columns_to_np_types (`Dict[str, np.dtype]`): A `dict` mapping column names to numpy dtypes. + output_signature (`Dict[str, tf.TensorSpec]`): A `dict` mapping column names to + `tf.TensorSpec` objects. + shuffle(`bool`): Shuffle the dataset order when loading. Recommended True for training, False for + validation/evaluation. + batch_size (`int`, default `None`): Size of batches to load from the dataset. Defaults to `None`, which implies that + the dataset won't be batched, but the returned dataset can be batched later with `tf_dataset.batch(batch_size)`. + drop_remainder(`bool`, default `None`): Drop the last incomplete batch when loading. If not provided, + defaults to the same setting as shuffle. + + Returns: + `tf.data.Dataset` + """ + if config.TF_AVAILABLE: + import tensorflow as tf + else: + raise ImportError("Called a Tensorflow-specific function but Tensorflow is not installed.") + + # TODO Matt: When our minimum Python version is 3.8 or higher, we can delete all of this and move everything + # to the NumPy multiprocessing path. + if hasattr(tf, "random_index_shuffle"): + random_index_shuffle = tf.random_index_shuffle + elif hasattr(tf.random.experimental, "index_shuffle"): + random_index_shuffle = tf.random.experimental.index_shuffle + else: + if len(dataset) > 10_000_000: + warnings.warn( + "to_tf_dataset() can be memory-inefficient on versions of TensorFlow older than 2.9. " + "If you are iterating over a dataset with a very large number of samples, consider " + "upgrading to TF >= 2.9." + ) + random_index_shuffle = None + + getter_fn = partial( + np_get_batch, + dataset=dataset, + cols_to_retain=cols_to_retain, + collate_fn=collate_fn, + collate_fn_args=collate_fn_args, + columns_to_np_types=columns_to_np_types, + return_dict=False, + ) + + # This works because dictionaries always output in the same order + tout = [tf.dtypes.as_dtype(dtype) for dtype in columns_to_np_types.values()] + + @tf.function(input_signature=[tf.TensorSpec(None, tf.int64)]) + def fetch_function(indices): + output = tf.py_function( + getter_fn, + inp=[indices], + Tout=tout, + ) + return {key: output[i] for i, key in enumerate(columns_to_np_types.keys())} + + tf_dataset = tf.data.Dataset.range(len(dataset)) + + if shuffle and random_index_shuffle is not None: + base_seed = tf.fill((3,), value=tf.cast(-1, dtype=tf.int64)) + + def scan_random_index(state, index): + if tf.reduce_all(state == -1): + # This generates a new random seed once per epoch only, + # to ensure that we iterate over each sample exactly once per epoch + state = tf.random.uniform(shape=(3,), maxval=2**62, dtype=tf.int64) + shuffled_index = random_index_shuffle(index=index, seed=state, max_index=len(dataset) - 1) + return state, shuffled_index + + tf_dataset = tf_dataset.scan(base_seed, scan_random_index) + elif shuffle: + tf_dataset = tf_dataset.shuffle(tf_dataset.cardinality()) + + if batch_size is not None: + tf_dataset = tf_dataset.batch(batch_size, drop_remainder=drop_remainder) + + tf_dataset = tf_dataset.map(fetch_function) + + if batch_size is not None: + + def ensure_shapes(input_dict): + return {key: tf.ensure_shape(val, output_signature[key].shape) for key, val in input_dict.items()} + + else: + # Ensure shape but remove batch dimension of output_signature[key].shape + def ensure_shapes(input_dict): + return {key: tf.ensure_shape(val, output_signature[key].shape[1:]) for key, val in input_dict.items()} + + return tf_dataset.map(ensure_shapes) + + +class SharedMemoryContext: + # This is a context manager for creating shared memory that ensures cleanup happens even if a process is interrupted + # The process that creates shared memory is always the one responsible for unlinking it in the end + def __init__(self): + self.created_shms = [] + self.opened_shms = [] + + def get_shm(self, name, size, create): + shm = SharedMemory(size=int(size), name=name, create=create) + if create: + # We only unlink the ones we created in this context + self.created_shms.append(shm) + else: + # If we didn't create it, we only close it when done, we don't unlink it + self.opened_shms.append(shm) + return shm + + def get_array(self, name, shape, dtype, create): + shm = self.get_shm(name=name, size=np.prod(shape) * np.dtype(dtype).itemsize, create=create) + return np.ndarray(shape, dtype=dtype, buffer=shm.buf) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + for shm in self.created_shms: + shm.close() + shm.unlink() + for shm in self.opened_shms: + shm.close() + + +class NumpyMultiprocessingGenerator: + def __init__( + self, + dataset, + cols_to_retain, + collate_fn, + collate_fn_args, + columns_to_np_types, + output_signature, + shuffle, + batch_size, + drop_remainder, + num_workers, + ): + self.dataset = dataset + self.cols_to_retain = cols_to_retain + self.collate_fn = collate_fn + self.collate_fn_args = collate_fn_args + self.string_columns = [col for col, dtype in columns_to_np_types.items() if dtype is np.str_] + # Strings will be converted to arrays of single unicode chars, so that we can have a constant itemsize + self.columns_to_np_types = { + col: dtype if col not in self.string_columns else np.dtype("U1") + for col, dtype in columns_to_np_types.items() + } + self.output_signature = output_signature + self.shuffle = shuffle + self.batch_size = batch_size + self.drop_remainder = drop_remainder + self.num_workers = num_workers + # Because strings are converted to characters, we need to add one extra dimension to the shape + self.columns_to_ranks = { + col: int(spec.shape.rank) if col not in self.string_columns else int(spec.shape.rank) + 1 + for col, spec in output_signature.items() + } + + def __iter__(self): + # Make sure we only spawn workers if they have work to do + num_workers = min(self.num_workers, int(ceil(len(self.dataset) / self.batch_size))) + # Do the shuffling in iter so that it's done at the start of each epoch + per_worker_batches, final_batch, final_batch_worker = self.distribute_batches( + self.dataset, self.batch_size, self.drop_remainder, num_workers, self.shuffle + ) + ctx = get_context("spawn") + names = [] + shape_arrays = [] + workers = [] + array_ready_events = [ctx.Event() for _ in range(num_workers)] + array_loaded_events = [ctx.Event() for _ in range(num_workers)] + + base_args = { + "dataset": self.dataset, + "cols_to_retain": self.cols_to_retain, + "collate_fn": self.collate_fn, + "collate_fn_args": self.collate_fn_args, + "columns_to_np_types": self.columns_to_np_types, + "columns_to_ranks": self.columns_to_ranks, + "string_columns": self.string_columns, + } + with SharedMemoryContext() as shm_ctx: + for i in range(num_workers): + worker_random_id = str(uuid4()) + worker_name = f"dw_{i}_{worker_random_id}"[:10] + names.append(worker_name) + + worker_shape_arrays = { + col: shm_ctx.get_array(f"{worker_name}_{col}_shape", shape=(rank,), dtype=np.int64, create=True) + for col, rank in self.columns_to_ranks.items() + } + shape_arrays.append(worker_shape_arrays) + + worker_indices = per_worker_batches[i] + if i == final_batch_worker and final_batch is not None: + final_batch_arg = final_batch + else: + final_batch_arg = None + worker_kwargs = { + "worker_name": worker_name, + "indices": worker_indices, + "extra_batch": final_batch_arg, + "array_ready_event": array_ready_events[i], + "array_loaded_event": array_loaded_events[i], + **base_args, + } + worker = ctx.Process(target=self.worker_loop, kwargs=worker_kwargs, daemon=True) + worker.start() + workers.append(worker) + + end_signal_received = False + while not end_signal_received: + for i in range(num_workers): + if not array_ready_events[i].wait(timeout=60): + raise TimeoutError("Data loading worker timed out!") + array_ready_events[i].clear() + array_shapes = shape_arrays[i] + if any(np.any(shape < 0) for shape in array_shapes.values()): + # Child processes send negative array shapes to indicate + # that no more data is going to be sent + end_signal_received = True + break + # Matt: Because array shapes are variable we recreate the shared memory each iteration. + # I suspect repeatedly opening lots of shared memory is the bottleneck for the parent process. + # A future optimization, at the cost of some code complexity, could be to reuse shared memory + # between iterations, but this would require knowing in advance the maximum size, or having + # a system to only create a new memory block when a new maximum size is seen. + # Another potential optimization would be to figure out which memory copies are necessary, + # or whether we can yield objects straight out of shared memory. + with SharedMemoryContext() as batch_shm_ctx: + # This memory context only lasts long enough to copy everything out of the batch + arrays = { + col: batch_shm_ctx.get_array( + f"{names[i]}_{col}", + shape=shape, + dtype=self.columns_to_np_types[col], + create=False, + ) + for col, shape in array_shapes.items() + } + # Copy everything out of shm because the memory + # will be unlinked by the child process at some point + arrays = {col: np.copy(arr) for col, arr in arrays.items()} + # Now we convert any unicode char arrays to strings + for string_col in self.string_columns: + arrays[string_col] = ( + arrays[string_col].view(f"U{arrays[string_col].shape[-1]}").squeeze(-1) + ) + yield arrays + array_loaded_events[i].set() + # Now we just do some cleanup + # Shared memory is cleaned up by the context manager, so we just make sure workers finish + for worker in workers: + worker.join() + + def __call__(self): + return self + + @staticmethod + def worker_loop( + dataset, + cols_to_retain, + collate_fn, + collate_fn_args, + columns_to_np_types, + columns_to_ranks, + string_columns, + indices, + extra_batch, + worker_name, + array_ready_event, + array_loaded_event, + ): + os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3" + + if config.TF_AVAILABLE: + import tensorflow as tf + else: + raise ImportError("Called a Tensorflow-specific function but Tensorflow is not installed.") + + tf.config.set_visible_devices([], "GPU") # Make sure workers don't try to allocate GPU memory + + def send_batch_to_parent(indices): + batch = np_get_batch( + indices=indices, + dataset=dataset, + cols_to_retain=cols_to_retain, + collate_fn=collate_fn, + collate_fn_args=collate_fn_args, + columns_to_np_types=columns_to_np_types, + return_dict=True, + ) + + # Now begins the fun part where we start shovelling shared memory at the parent process + out_arrays = {} + with SharedMemoryContext() as batch_shm_ctx: + # The batch shared memory context exists only as long as it takes for the parent process + # to read everything, after which it cleans everything up again + for col, cast_dtype in columns_to_np_types.items(): + # Everything has to be np.array for this to work, even if the collate_fn is giving us tf.Tensor + array = batch[col] + if col in string_columns: + # We can't send unicode arrays over shared memory, so we convert to single chars ("U1") + # which have a fixed width of 4 bytes. The parent process will convert these back to strings. + array = array.view("U1").reshape(array.shape + (-1,)) + shape_arrays[col][:] = array.shape + out_arrays[col] = batch_shm_ctx.get_array( + f"{worker_name}_{col}", shape=array.shape, dtype=cast_dtype, create=True + ) + out_arrays[col][:] = array + + array_ready_event.set() + array_loaded_event.wait() + array_loaded_event.clear() + + with SharedMemoryContext() as shm_ctx: + shape_arrays = { + col: shm_ctx.get_array(f"{worker_name}_{col}_shape", shape=(rank,), dtype=np.int64, create=False) + for col, rank in columns_to_ranks.items() + } + + for batch in indices: + send_batch_to_parent(batch) + if extra_batch is not None: + send_batch_to_parent(extra_batch) + # Now we send a batsignal to the parent process that we're done + for col, array in shape_arrays.items(): + array[:] = -1 + array_ready_event.set() + + @staticmethod + def distribute_batches(dataset, batch_size, drop_remainder, num_workers, shuffle): + indices = np.arange(len(dataset)) + if shuffle: + np.random.shuffle(indices) + num_samples = len(indices) + # We distribute the batches so that reading from the workers in round-robin order yields the exact + # order specified in indices. This is only important when shuffle is False, but we do it regardless. + incomplete_batch_cutoff = num_samples - (num_samples % batch_size) + indices, last_incomplete_batch = np.split(indices, [incomplete_batch_cutoff]) + if drop_remainder or len(last_incomplete_batch) == 0: + last_incomplete_batch = None + + indices = indices.reshape(-1, batch_size) + num_batches = len(indices) + final_batches_cutoff = num_batches - (num_batches % num_workers) + indices, final_batches = np.split(indices, [final_batches_cutoff]) + indices = indices.reshape(-1, num_workers, batch_size) + + per_worker_indices = np.split(indices, indices.shape[1], axis=1) + per_worker_indices = [np.squeeze(worker_indices, 1) for worker_indices in per_worker_indices] + # Distribute the final batches to the first workers + for i in range(len(final_batches)): + # len(final_batches) can be zero, and is always less than num_workers + per_worker_indices[i] = np.concatenate([per_worker_indices[i], final_batches[i].reshape(1, -1)], axis=0) + # Add the last incomplete batch to the next worker, which might be the first worker + if last_incomplete_batch is not None: + incomplete_batch_worker_idx = len(final_batches) + else: + incomplete_batch_worker_idx = None + return per_worker_indices, last_incomplete_batch, incomplete_batch_worker_idx + + +def multiprocess_dataset_to_tf( + dataset, + cols_to_retain, + collate_fn, + collate_fn_args, + columns_to_np_types, + output_signature, + shuffle, + batch_size, + drop_remainder, + num_workers, +): + """Create a tf.data.Dataset from the underlying Dataset. This is a multi-process method - the single-process + equivalent is dataset_to_tf. + + Args: + dataset (`Dataset`): Dataset to wrap with tf.data.Dataset. + cols_to_retain (`List[str]`): Dataset column(s) to load in the + tf.data.Dataset. It is acceptable to include column names that are created by the `collate_fn` and + that do not exist in the original dataset. + collate_fn(`Callable`): A function or callable object (such as a `DataCollator`) that will collate + lists of samples into a batch. + collate_fn_args (`Dict`): A `dict` of keyword arguments to be passed to the + `collate_fn`. Can be empty. + columns_to_np_types (`Dict[str, np.dtype]`): A `dict` mapping column names to numpy dtypes. + output_signature (`Dict[str, tf.TensorSpec]`): A `dict` mapping column names to + `tf.TensorSpec` objects. + shuffle(`bool`): Shuffle the dataset order when loading. Recommended True for training, False for + validation/evaluation. + batch_size (`int`, default `None`): Size of batches to load from the dataset. Defaults to `None`, which implies that + the dataset won't be batched, but the returned dataset can be batched later with `tf_dataset.batch(batch_size)`. + drop_remainder(`bool`, default `None`): Drop the last incomplete batch when loading. If not provided, + defaults to the same setting as shuffle. + num_workers (`int`): Number of workers to use for loading the dataset. Should be >= 1. + + Returns: + `tf.data.Dataset` + """ + if config.TF_AVAILABLE: + import tensorflow as tf + else: + raise ImportError("Called a Tensorflow-specific function but Tensorflow is not installed.") + + data_generator = NumpyMultiprocessingGenerator( + dataset=dataset, + cols_to_retain=cols_to_retain, + collate_fn=collate_fn, + collate_fn_args=collate_fn_args, + columns_to_np_types=columns_to_np_types, + output_signature=output_signature, + shuffle=shuffle, + batch_size=batch_size, + drop_remainder=drop_remainder, + num_workers=num_workers, + ) + + tf_dataset = tf.data.Dataset.from_generator(data_generator, output_signature=output_signature) + if drop_remainder: + dataset_length = int(len(dataset) // batch_size) + else: + dataset_length = int(ceil(len(dataset) / batch_size)) + return tf_dataset.apply(tf.data.experimental.assert_cardinality(dataset_length)) diff --git a/lib/python3.12/site-packages/datasets/utils/tqdm.py b/lib/python3.12/site-packages/datasets/utils/tqdm.py new file mode 100644 index 0000000000000000000000000000000000000000..e28a8ff7ccfda7d4fd9a6195636d181f285ceb65 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/utils/tqdm.py @@ -0,0 +1,131 @@ +"""Utility helpers to handle progress bars in `datasets`. + +Example: + 1. Use `datasets.utils.tqdm` as you would use `tqdm.tqdm` or `tqdm.auto.tqdm`. + 2. To disable progress bars, either use `disable_progress_bars()` helper or set the + environment variable `HF_DATASETS_DISABLE_PROGRESS_BARS` to 1. + 3. To re-enable progress bars, use `enable_progress_bars()`. + 4. To check whether progress bars are disabled, use `are_progress_bars_disabled()`. + +NOTE: Environment variable `HF_DATASETS_DISABLE_PROGRESS_BARS` has the priority. + +Example: + ```py + from datasets.utils import ( + are_progress_bars_disabled, + disable_progress_bars, + enable_progress_bars, + tqdm, + ) + + # Disable progress bars globally + disable_progress_bars() + + # Use as normal `tqdm` + for _ in tqdm(range(5)): + do_something() + + # Still not showing progress bars, as `disable=False` is overwritten to `True`. + for _ in tqdm(range(5), disable=False): + do_something() + + are_progress_bars_disabled() # True + + # Re-enable progress bars globally + enable_progress_bars() + + # Progress bar will be shown ! + for _ in tqdm(range(5)): + do_something() + ``` +""" + +import warnings + +from tqdm.auto import tqdm as old_tqdm + +from ..config import HF_DATASETS_DISABLE_PROGRESS_BARS + + +# `HF_DATASETS_DISABLE_PROGRESS_BARS` is `Optional[bool]` while `_hf_datasets_progress_bars_disabled` +# is a `bool`. If `HF_DATASETS_DISABLE_PROGRESS_BARS` is set to True or False, it has priority. +# If `HF_DATASETS_DISABLE_PROGRESS_BARS` is None, it means the user have not set the +# environment variable and is free to enable/disable progress bars programmatically. +# TL;DR: env variable has priority over code. +# +# By default, progress bars are enabled. +_hf_datasets_progress_bars_disabled: bool = HF_DATASETS_DISABLE_PROGRESS_BARS or False + + +def disable_progress_bars() -> None: + """ + Disable globally progress bars used in `datasets` except if `HF_DATASETS_DISABLE_PROGRESS_BAR` environment + variable has been set. + + Use [`~utils.enable_progress_bars`] to re-enable them. + """ + if HF_DATASETS_DISABLE_PROGRESS_BARS is False: + warnings.warn( + "Cannot disable progress bars: environment variable `HF_DATASETS_DISABLE_PROGRESS_BAR=0` is set and has" + " priority." + ) + return + global _hf_datasets_progress_bars_disabled + _hf_datasets_progress_bars_disabled = True + + +def enable_progress_bars() -> None: + """ + Enable globally progress bars used in `datasets` except if `HF_DATASETS_DISABLE_PROGRESS_BAR` environment + variable has been set. + + Use [`~utils.disable_progress_bars`] to disable them. + """ + if HF_DATASETS_DISABLE_PROGRESS_BARS is True: + warnings.warn( + "Cannot enable progress bars: environment variable `HF_DATASETS_DISABLE_PROGRESS_BAR=1` is set and has" + " priority." + ) + return + global _hf_datasets_progress_bars_disabled + _hf_datasets_progress_bars_disabled = False + + +def are_progress_bars_disabled() -> bool: + """Return whether progress bars are globally disabled or not. + + Progress bars used in `datasets` can be enable or disabled globally using [`~utils.enable_progress_bars`] + and [`~utils.disable_progress_bars`] or by setting `HF_DATASETS_DISABLE_PROGRESS_BAR` as environment variable. + """ + global _hf_datasets_progress_bars_disabled + return _hf_datasets_progress_bars_disabled + + +class tqdm(old_tqdm): + """ + Class to override `disable` argument in case progress bars are globally disabled. + + Taken from https://github.com/tqdm/tqdm/issues/619#issuecomment-619639324. + """ + + def __init__(self, *args, **kwargs): + if are_progress_bars_disabled(): + kwargs["disable"] = True + super().__init__(*args, **kwargs) + + def __delattr__(self, attr: str) -> None: + """Fix for https://github.com/huggingface/datasets/issues/6066""" + try: + super().__delattr__(attr) + except AttributeError: + if attr != "_lock": + raise + + +# backward compatibility +enable_progress_bar = enable_progress_bars +disable_progress_bar = disable_progress_bars + + +def is_progress_bar_enabled(): + return not are_progress_bars_disabled() diff --git a/lib/python3.12/site-packages/datasets/utils/track.py b/lib/python3.12/site-packages/datasets/utils/track.py new file mode 100644 index 0000000000000000000000000000000000000000..ffd3434af790de63fdd046ad8f93c3067c6e6da8 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/utils/track.py @@ -0,0 +1,61 @@ +from collections.abc import Iterable, Iterator + + +class tracked_str(str): + origins = {} + + def set_origin(self, origin: str): + if super().__repr__() not in self.origins: + self.origins[super().__repr__()] = origin + + def get_origin(self): + return self.origins.get(super().__repr__(), str(self)) + + def __repr__(self) -> str: + if super().__repr__() not in self.origins or self.origins[super().__repr__()] == self: + return super().__repr__() + else: + return f"{str(self)} (origin={self.origins[super().__repr__()]})" + + +class tracked_list(list): + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self.last_item = None + + def __iter__(self) -> Iterator: + for x in super().__iter__(): + self.last_item = x + yield x + self.last_item = None + + def __repr__(self) -> str: + if self.last_item is None: + return super().__repr__() + else: + return f"{self.__class__.__name__}(current={self.last_item})" + + +class TrackedIterableFromGenerator(Iterable): + """Utility class to create an iterable from a generator function, in order to reset the generator when needed.""" + + def __init__(self, generator, *args): + super().__init__() + self.generator = generator + self.args = args + self.last_item = None + + def __iter__(self): + for x in self.generator(*self.args): + self.last_item = x + yield x + self.last_item = None + + def __repr__(self) -> str: + if self.last_item is None: + return super().__repr__() + else: + return f"{self.__class__.__name__}(current={self.last_item})" + + def __reduce__(self): + return (self.__class__, (self.generator, *self.args)) diff --git a/lib/python3.12/site-packages/datasets/utils/typing.py b/lib/python3.12/site-packages/datasets/utils/typing.py new file mode 100644 index 0000000000000000000000000000000000000000..4736d74cbb575bd0e12f625c520202a8c74b3ddc --- /dev/null +++ b/lib/python3.12/site-packages/datasets/utils/typing.py @@ -0,0 +1,9 @@ +import os +from typing import TypeVar, Union + + +T = TypeVar("T") + +ListLike = Union[list[T], tuple[T, ...]] +NestedDataStructureLike = Union[T, list[T], dict[str, T]] +PathLike = Union[str, bytes, os.PathLike] diff --git a/lib/python3.12/site-packages/datasets/utils/version.py b/lib/python3.12/site-packages/datasets/utils/version.py new file mode 100644 index 0000000000000000000000000000000000000000..75cf4c39d5f9b916ade8f8d88a0f0ca9e5769217 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/utils/version.py @@ -0,0 +1,106 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +"""Version utils.""" + +import dataclasses +import re +from dataclasses import dataclass +from functools import total_ordering +from typing import Optional, Union + + +_VERSION_REG = re.compile(r"^(?P\d+)" r"\.(?P\d+)" r"\.(?P\d+)$") + + +@total_ordering +@dataclass +class Version: + """Dataset version `MAJOR.MINOR.PATCH`. + + Args: + version_str (`str`): + The dataset version. + description (`str`): + A description of what is new in this version. + major (`str`): + minor (`str`): + patch (`str`): + + Example: + + ```py + >>> VERSION = datasets.Version("1.0.0") + ``` + """ + + version_str: str + description: Optional[str] = None + major: Optional[Union[str, int]] = None + minor: Optional[Union[str, int]] = None + patch: Optional[Union[str, int]] = None + + def __post_init__(self): + self.major, self.minor, self.patch = _str_to_version_tuple(self.version_str) + + def __repr__(self): + return f"{self.tuple[0]}.{self.tuple[1]}.{self.tuple[2]}" + + @property + def tuple(self): + return self.major, self.minor, self.patch + + def _validate_operand(self, other): + if isinstance(other, str): + return Version(other) + elif isinstance(other, Version): + return other + raise TypeError(f"{other} (type {type(other)}) cannot be compared to version.") + + def __eq__(self, other): + try: + other = self._validate_operand(other) + except (TypeError, ValueError): + return False + else: + return self.tuple == other.tuple + + def __lt__(self, other): + other = self._validate_operand(other) + return self.tuple < other.tuple + + def __hash__(self): + return hash(_version_tuple_to_str(self.tuple)) + + @classmethod + def from_dict(cls, dic): + field_names = {f.name for f in dataclasses.fields(cls)} + return cls(**{k: v for k, v in dic.items() if k in field_names}) + + def _to_yaml_string(self) -> str: + return self.version_str + + +def _str_to_version_tuple(version_str): + """Return the tuple (major, minor, patch) version extracted from the str.""" + res = _VERSION_REG.match(version_str) + if not res: + raise ValueError(f"Invalid version '{version_str}'. Format should be x.y.z with {{x,y,z}} being digits.") + return tuple(int(v) for v in [res.group("major"), res.group("minor"), res.group("patch")]) + + +def _version_tuple_to_str(version_tuple): + """Return the str version from the version tuple (major, minor, patch).""" + return ".".join(str(v) for v in version_tuple) diff --git a/lib/python3.12/site-packages/openai/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/openai/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4db43ae1fe043b8bd6994592d51e83111a90ab2e Binary files /dev/null and b/lib/python3.12/site-packages/openai/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/__pycache__/__main__.cpython-312.pyc b/lib/python3.12/site-packages/openai/__pycache__/__main__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..064ed3536e23f9f2a8f72fc9c28ff8841e4b9162 Binary files /dev/null and b/lib/python3.12/site-packages/openai/__pycache__/__main__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/__pycache__/_base_client.cpython-312.pyc b/lib/python3.12/site-packages/openai/__pycache__/_base_client.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..30913c57653b265861d7d8aec8e536054b2589b5 Binary files /dev/null and b/lib/python3.12/site-packages/openai/__pycache__/_base_client.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/__pycache__/_client.cpython-312.pyc b/lib/python3.12/site-packages/openai/__pycache__/_client.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..85a6dd27e9cb8450fa26eabf9258543882b7d1a2 Binary files /dev/null and b/lib/python3.12/site-packages/openai/__pycache__/_client.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/__pycache__/_compat.cpython-312.pyc b/lib/python3.12/site-packages/openai/__pycache__/_compat.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7a86c1ef1122b15dead407b878b73e64828147f0 Binary files /dev/null and b/lib/python3.12/site-packages/openai/__pycache__/_compat.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/__pycache__/_constants.cpython-312.pyc b/lib/python3.12/site-packages/openai/__pycache__/_constants.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d260529b7f9fbfc6ce37135c59e5a55a4e122027 Binary files /dev/null and b/lib/python3.12/site-packages/openai/__pycache__/_constants.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/__pycache__/_exceptions.cpython-312.pyc b/lib/python3.12/site-packages/openai/__pycache__/_exceptions.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..25c89d2df2e3b580cf638af1bd692b2472bec842 Binary files /dev/null and b/lib/python3.12/site-packages/openai/__pycache__/_exceptions.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/__pycache__/_files.cpython-312.pyc b/lib/python3.12/site-packages/openai/__pycache__/_files.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5c3b6b10d3cbeaf7d690065874c43ecf38029b5d Binary files /dev/null and b/lib/python3.12/site-packages/openai/__pycache__/_files.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/__pycache__/_legacy_response.cpython-312.pyc b/lib/python3.12/site-packages/openai/__pycache__/_legacy_response.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3fa1e876c1a4560c5b8d30c56879b26332578e65 Binary files /dev/null and b/lib/python3.12/site-packages/openai/__pycache__/_legacy_response.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/__pycache__/_models.cpython-312.pyc b/lib/python3.12/site-packages/openai/__pycache__/_models.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ba6c7edf730ef00d26deb40374a19ddc6a7a83d6 Binary files /dev/null and b/lib/python3.12/site-packages/openai/__pycache__/_models.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/__pycache__/_module_client.cpython-312.pyc b/lib/python3.12/site-packages/openai/__pycache__/_module_client.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..25151eba98d2ca4a7c7c8fcf4ed6d33da539f5f2 Binary files /dev/null and b/lib/python3.12/site-packages/openai/__pycache__/_module_client.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/__pycache__/_qs.cpython-312.pyc b/lib/python3.12/site-packages/openai/__pycache__/_qs.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..872707c77c467691baa25c1f5c756d7d45fae594 Binary files /dev/null and b/lib/python3.12/site-packages/openai/__pycache__/_qs.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/__pycache__/_resource.cpython-312.pyc b/lib/python3.12/site-packages/openai/__pycache__/_resource.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..656d1553f3c3d7c2f391d0574fba4417d277e2a6 Binary files /dev/null and b/lib/python3.12/site-packages/openai/__pycache__/_resource.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/__pycache__/_response.cpython-312.pyc b/lib/python3.12/site-packages/openai/__pycache__/_response.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9496aafd600138b10f9ad5aca89fb86f30be3888 Binary files /dev/null and b/lib/python3.12/site-packages/openai/__pycache__/_response.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/__pycache__/_streaming.cpython-312.pyc b/lib/python3.12/site-packages/openai/__pycache__/_streaming.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e898cb855e6f58bc7af706b86a2db0043b32c401 Binary files /dev/null and b/lib/python3.12/site-packages/openai/__pycache__/_streaming.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/__pycache__/_types.cpython-312.pyc b/lib/python3.12/site-packages/openai/__pycache__/_types.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bc9441b76f0d8b425b591ef21663a663ce01d72e Binary files /dev/null and b/lib/python3.12/site-packages/openai/__pycache__/_types.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/__pycache__/_version.cpython-312.pyc b/lib/python3.12/site-packages/openai/__pycache__/_version.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5d8fe63b8ecc80fb49aed1a6cc932a051b71b463 Binary files /dev/null and b/lib/python3.12/site-packages/openai/__pycache__/_version.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/__pycache__/pagination.cpython-312.pyc b/lib/python3.12/site-packages/openai/__pycache__/pagination.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c5fb9d7128aed927646abd6cee58f287d26235ed Binary files /dev/null and b/lib/python3.12/site-packages/openai/__pycache__/pagination.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/__pycache__/version.cpython-312.pyc b/lib/python3.12/site-packages/openai/__pycache__/version.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3c0ad31bb8a19787e7c30f66413d3e79d03f1076 Binary files /dev/null and b/lib/python3.12/site-packages/openai/__pycache__/version.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/_utils/__init__.py b/lib/python3.12/site-packages/openai/_utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..bd01c088dc935b6f8ac159647793438c5369d9df --- /dev/null +++ b/lib/python3.12/site-packages/openai/_utils/__init__.py @@ -0,0 +1,60 @@ +from ._logs import SensitiveHeadersFilter as SensitiveHeadersFilter +from ._sync import asyncify as asyncify +from ._proxy import LazyProxy as LazyProxy +from ._utils import ( + flatten as flatten, + is_dict as is_dict, + is_list as is_list, + is_given as is_given, + is_tuple as is_tuple, + json_safe as json_safe, + lru_cache as lru_cache, + is_mapping as is_mapping, + is_tuple_t as is_tuple_t, + parse_date as parse_date, + is_iterable as is_iterable, + is_sequence as is_sequence, + coerce_float as coerce_float, + is_mapping_t as is_mapping_t, + removeprefix as removeprefix, + removesuffix as removesuffix, + extract_files as extract_files, + is_sequence_t as is_sequence_t, + required_args as required_args, + coerce_boolean as coerce_boolean, + coerce_integer as coerce_integer, + file_from_path as file_from_path, + parse_datetime as parse_datetime, + is_azure_client as is_azure_client, + strip_not_given as strip_not_given, + deepcopy_minimal as deepcopy_minimal, + get_async_library as get_async_library, + maybe_coerce_float as maybe_coerce_float, + get_required_header as get_required_header, + maybe_coerce_boolean as maybe_coerce_boolean, + maybe_coerce_integer as maybe_coerce_integer, + is_async_azure_client as is_async_azure_client, +) +from ._typing import ( + is_list_type as is_list_type, + is_union_type as is_union_type, + extract_type_arg as extract_type_arg, + is_iterable_type as is_iterable_type, + is_required_type as is_required_type, + is_annotated_type as is_annotated_type, + is_type_alias_type as is_type_alias_type, + strip_annotated_type as strip_annotated_type, + extract_type_var_from_base as extract_type_var_from_base, +) +from ._streams import consume_sync_iterator as consume_sync_iterator, consume_async_iterator as consume_async_iterator +from ._transform import ( + PropertyInfo as PropertyInfo, + transform as transform, + async_transform as async_transform, + maybe_transform as maybe_transform, + async_maybe_transform as async_maybe_transform, +) +from ._reflection import ( + function_has_argument as function_has_argument, + assert_signatures_in_sync as assert_signatures_in_sync, +) diff --git a/lib/python3.12/site-packages/openai/_utils/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/openai/_utils/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9b1976ad2d53380c6edc8e04c36edcb24671c8c6 Binary files /dev/null and b/lib/python3.12/site-packages/openai/_utils/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/_utils/__pycache__/_logs.cpython-312.pyc b/lib/python3.12/site-packages/openai/_utils/__pycache__/_logs.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..458e4635042f6f23efef7fd0612e21c72c0f94ed Binary files /dev/null and b/lib/python3.12/site-packages/openai/_utils/__pycache__/_logs.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/_utils/__pycache__/_proxy.cpython-312.pyc b/lib/python3.12/site-packages/openai/_utils/__pycache__/_proxy.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cc56f9ba7e282fd2c0912ef72db536e9c27d7222 Binary files /dev/null and b/lib/python3.12/site-packages/openai/_utils/__pycache__/_proxy.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/_utils/__pycache__/_reflection.cpython-312.pyc b/lib/python3.12/site-packages/openai/_utils/__pycache__/_reflection.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bc86911759d33229c0b1feb3bca9508213f275c1 Binary files /dev/null and b/lib/python3.12/site-packages/openai/_utils/__pycache__/_reflection.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/_utils/__pycache__/_streams.cpython-312.pyc b/lib/python3.12/site-packages/openai/_utils/__pycache__/_streams.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a7859231e429f419283f9e8917830aa754f47603 Binary files /dev/null and b/lib/python3.12/site-packages/openai/_utils/__pycache__/_streams.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/_utils/__pycache__/_sync.cpython-312.pyc b/lib/python3.12/site-packages/openai/_utils/__pycache__/_sync.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e0116f6c04eaaaa2063718052ab9d9109d3c94ae Binary files /dev/null and b/lib/python3.12/site-packages/openai/_utils/__pycache__/_sync.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/_utils/__pycache__/_transform.cpython-312.pyc b/lib/python3.12/site-packages/openai/_utils/__pycache__/_transform.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6413a27ba6d9ad3b8525f7c79cca7e256d5df51e Binary files /dev/null and b/lib/python3.12/site-packages/openai/_utils/__pycache__/_transform.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/_utils/__pycache__/_typing.cpython-312.pyc b/lib/python3.12/site-packages/openai/_utils/__pycache__/_typing.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1ddfb23e7879e6476a2a0d106f7962a412fc723b Binary files /dev/null and b/lib/python3.12/site-packages/openai/_utils/__pycache__/_typing.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/_utils/__pycache__/_utils.cpython-312.pyc b/lib/python3.12/site-packages/openai/_utils/__pycache__/_utils.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e218b4d371643c8c8fbe90dc2ec772085c21bf6c Binary files /dev/null and b/lib/python3.12/site-packages/openai/_utils/__pycache__/_utils.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/_utils/_logs.py b/lib/python3.12/site-packages/openai/_utils/_logs.py new file mode 100644 index 0000000000000000000000000000000000000000..376946933cb7fd220a8192350a8948206aae3c11 --- /dev/null +++ b/lib/python3.12/site-packages/openai/_utils/_logs.py @@ -0,0 +1,42 @@ +import os +import logging +from typing_extensions import override + +from ._utils import is_dict + +logger: logging.Logger = logging.getLogger("openai") +httpx_logger: logging.Logger = logging.getLogger("httpx") + + +SENSITIVE_HEADERS = {"api-key", "authorization"} + + +def _basic_config() -> None: + # e.g. [2023-10-05 14:12:26 - openai._base_client:818 - DEBUG] HTTP Request: POST http://127.0.0.1:4010/foo/bar "200 OK" + logging.basicConfig( + format="[%(asctime)s - %(name)s:%(lineno)d - %(levelname)s] %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + ) + + +def setup_logging() -> None: + env = os.environ.get("OPENAI_LOG") + if env == "debug": + _basic_config() + logger.setLevel(logging.DEBUG) + httpx_logger.setLevel(logging.DEBUG) + elif env == "info": + _basic_config() + logger.setLevel(logging.INFO) + httpx_logger.setLevel(logging.INFO) + + +class SensitiveHeadersFilter(logging.Filter): + @override + def filter(self, record: logging.LogRecord) -> bool: + if is_dict(record.args) and "headers" in record.args and is_dict(record.args["headers"]): + headers = record.args["headers"] = {**record.args["headers"]} + for header in headers: + if str(header).lower() in SENSITIVE_HEADERS: + headers[header] = "" + return True diff --git a/lib/python3.12/site-packages/openai/_utils/_proxy.py b/lib/python3.12/site-packages/openai/_utils/_proxy.py new file mode 100644 index 0000000000000000000000000000000000000000..ffd883e9dded36a0643196b491bfd197dce6c5e3 --- /dev/null +++ b/lib/python3.12/site-packages/openai/_utils/_proxy.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import Generic, TypeVar, Iterable, cast +from typing_extensions import override + +T = TypeVar("T") + + +class LazyProxy(Generic[T], ABC): + """Implements data methods to pretend that an instance is another instance. + + This includes forwarding attribute access and other methods. + """ + + # Note: we have to special case proxies that themselves return proxies + # to support using a proxy as a catch-all for any random access, e.g. `proxy.foo.bar.baz` + + def __getattr__(self, attr: str) -> object: + proxied = self.__get_proxied__() + if isinstance(proxied, LazyProxy): + return proxied # pyright: ignore + return getattr(proxied, attr) + + @override + def __repr__(self) -> str: + proxied = self.__get_proxied__() + if isinstance(proxied, LazyProxy): + return proxied.__class__.__name__ + return repr(self.__get_proxied__()) + + @override + def __str__(self) -> str: + proxied = self.__get_proxied__() + if isinstance(proxied, LazyProxy): + return proxied.__class__.__name__ + return str(proxied) + + @override + def __dir__(self) -> Iterable[str]: + proxied = self.__get_proxied__() + if isinstance(proxied, LazyProxy): + return [] + return proxied.__dir__() + + @property # type: ignore + @override + def __class__(self) -> type: # pyright: ignore + proxied = self.__get_proxied__() + if issubclass(type(proxied), LazyProxy): + return type(proxied) + return proxied.__class__ + + def __get_proxied__(self) -> T: + return self.__load__() + + def __as_proxied__(self) -> T: + """Helper method that returns the current proxy, typed as the loaded object""" + return cast(T, self) + + @abstractmethod + def __load__(self) -> T: ... diff --git a/lib/python3.12/site-packages/openai/_utils/_reflection.py b/lib/python3.12/site-packages/openai/_utils/_reflection.py new file mode 100644 index 0000000000000000000000000000000000000000..bdaca29e4a16c66db7689826be68213348d54da1 --- /dev/null +++ b/lib/python3.12/site-packages/openai/_utils/_reflection.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +import inspect +from typing import Any, Callable + + +def function_has_argument(func: Callable[..., Any], arg_name: str) -> bool: + """Returns whether or not the given function has a specific parameter""" + sig = inspect.signature(func) + return arg_name in sig.parameters + + +def assert_signatures_in_sync( + source_func: Callable[..., Any], + check_func: Callable[..., Any], + *, + exclude_params: set[str] = set(), + description: str = "", +) -> None: + """Ensure that the signature of the second function matches the first.""" + + check_sig = inspect.signature(check_func) + source_sig = inspect.signature(source_func) + + errors: list[str] = [] + + for name, source_param in source_sig.parameters.items(): + if name in exclude_params: + continue + + custom_param = check_sig.parameters.get(name) + if not custom_param: + errors.append(f"the `{name}` param is missing") + continue + + if custom_param.annotation != source_param.annotation: + errors.append( + f"types for the `{name}` param are do not match; source={repr(source_param.annotation)} checking={repr(custom_param.annotation)}" + ) + continue + + if errors: + raise AssertionError( + f"{len(errors)} errors encountered when comparing signatures{description}:\n\n" + "\n\n".join(errors) + ) diff --git a/lib/python3.12/site-packages/openai/_utils/_streams.py b/lib/python3.12/site-packages/openai/_utils/_streams.py new file mode 100644 index 0000000000000000000000000000000000000000..f4a0208f01c069616000eeed319029fc92c03b56 --- /dev/null +++ b/lib/python3.12/site-packages/openai/_utils/_streams.py @@ -0,0 +1,12 @@ +from typing import Any +from typing_extensions import Iterator, AsyncIterator + + +def consume_sync_iterator(iterator: Iterator[Any]) -> None: + for _ in iterator: + ... + + +async def consume_async_iterator(iterator: AsyncIterator[Any]) -> None: + async for _ in iterator: + ... diff --git a/lib/python3.12/site-packages/openai/_utils/_sync.py b/lib/python3.12/site-packages/openai/_utils/_sync.py new file mode 100644 index 0000000000000000000000000000000000000000..ad7ec71b76bd31b9adbdaef523710be1f98202e7 --- /dev/null +++ b/lib/python3.12/site-packages/openai/_utils/_sync.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +import sys +import asyncio +import functools +import contextvars +from typing import Any, TypeVar, Callable, Awaitable +from typing_extensions import ParamSpec + +import anyio +import sniffio +import anyio.to_thread + +T_Retval = TypeVar("T_Retval") +T_ParamSpec = ParamSpec("T_ParamSpec") + + +if sys.version_info >= (3, 9): + _asyncio_to_thread = asyncio.to_thread +else: + # backport of https://docs.python.org/3/library/asyncio-task.html#asyncio.to_thread + # for Python 3.8 support + async def _asyncio_to_thread( + func: Callable[T_ParamSpec, T_Retval], /, *args: T_ParamSpec.args, **kwargs: T_ParamSpec.kwargs + ) -> Any: + """Asynchronously run function *func* in a separate thread. + + Any *args and **kwargs supplied for this function are directly passed + to *func*. Also, the current :class:`contextvars.Context` is propagated, + allowing context variables from the main thread to be accessed in the + separate thread. + + Returns a coroutine that can be awaited to get the eventual result of *func*. + """ + loop = asyncio.events.get_running_loop() + ctx = contextvars.copy_context() + func_call = functools.partial(ctx.run, func, *args, **kwargs) + return await loop.run_in_executor(None, func_call) + + +async def to_thread( + func: Callable[T_ParamSpec, T_Retval], /, *args: T_ParamSpec.args, **kwargs: T_ParamSpec.kwargs +) -> T_Retval: + if sniffio.current_async_library() == "asyncio": + return await _asyncio_to_thread(func, *args, **kwargs) + + return await anyio.to_thread.run_sync( + functools.partial(func, *args, **kwargs), + ) + + +# inspired by `asyncer`, https://github.com/tiangolo/asyncer +def asyncify(function: Callable[T_ParamSpec, T_Retval]) -> Callable[T_ParamSpec, Awaitable[T_Retval]]: + """ + Take a blocking function and create an async one that receives the same + positional and keyword arguments. For python version 3.9 and above, it uses + asyncio.to_thread to run the function in a separate thread. For python version + 3.8, it uses locally defined copy of the asyncio.to_thread function which was + introduced in python 3.9. + + Usage: + + ```python + def blocking_func(arg1, arg2, kwarg1=None): + # blocking code + return result + + + result = asyncify(blocking_function)(arg1, arg2, kwarg1=value1) + ``` + + ## Arguments + + `function`: a blocking regular callable (e.g. a function) + + ## Return + + An async function that takes the same positional and keyword arguments as the + original one, that when called runs the same original function in a thread worker + and returns the result. + """ + + async def wrapper(*args: T_ParamSpec.args, **kwargs: T_ParamSpec.kwargs) -> T_Retval: + return await to_thread(function, *args, **kwargs) + + return wrapper diff --git a/lib/python3.12/site-packages/openai/_utils/_transform.py b/lib/python3.12/site-packages/openai/_utils/_transform.py new file mode 100644 index 0000000000000000000000000000000000000000..b0cc20a73552d51db0a0ce891296e6ad9ba1cf4e --- /dev/null +++ b/lib/python3.12/site-packages/openai/_utils/_transform.py @@ -0,0 +1,447 @@ +from __future__ import annotations + +import io +import base64 +import pathlib +from typing import Any, Mapping, TypeVar, cast +from datetime import date, datetime +from typing_extensions import Literal, get_args, override, get_type_hints as _get_type_hints + +import anyio +import pydantic + +from ._utils import ( + is_list, + is_given, + lru_cache, + is_mapping, + is_iterable, +) +from .._files import is_base64_file_input +from ._typing import ( + is_list_type, + is_union_type, + extract_type_arg, + is_iterable_type, + is_required_type, + is_annotated_type, + strip_annotated_type, +) +from .._compat import get_origin, model_dump, is_typeddict + +_T = TypeVar("_T") + + +# TODO: support for drilling globals() and locals() +# TODO: ensure works correctly with forward references in all cases + + +PropertyFormat = Literal["iso8601", "base64", "custom"] + + +class PropertyInfo: + """Metadata class to be used in Annotated types to provide information about a given type. + + For example: + + class MyParams(TypedDict): + account_holder_name: Annotated[str, PropertyInfo(alias='accountHolderName')] + + This means that {'account_holder_name': 'Robert'} will be transformed to {'accountHolderName': 'Robert'} before being sent to the API. + """ + + alias: str | None + format: PropertyFormat | None + format_template: str | None + discriminator: str | None + + def __init__( + self, + *, + alias: str | None = None, + format: PropertyFormat | None = None, + format_template: str | None = None, + discriminator: str | None = None, + ) -> None: + self.alias = alias + self.format = format + self.format_template = format_template + self.discriminator = discriminator + + @override + def __repr__(self) -> str: + return f"{self.__class__.__name__}(alias='{self.alias}', format={self.format}, format_template='{self.format_template}', discriminator='{self.discriminator}')" + + +def maybe_transform( + data: object, + expected_type: object, +) -> Any | None: + """Wrapper over `transform()` that allows `None` to be passed. + + See `transform()` for more details. + """ + if data is None: + return None + return transform(data, expected_type) + + +# Wrapper over _transform_recursive providing fake types +def transform( + data: _T, + expected_type: object, +) -> _T: + """Transform dictionaries based off of type information from the given type, for example: + + ```py + class Params(TypedDict, total=False): + card_id: Required[Annotated[str, PropertyInfo(alias="cardID")]] + + + transformed = transform({"card_id": ""}, Params) + # {'cardID': ''} + ``` + + Any keys / data that does not have type information given will be included as is. + + It should be noted that the transformations that this function does are not represented in the type system. + """ + transformed = _transform_recursive(data, annotation=cast(type, expected_type)) + return cast(_T, transformed) + + +@lru_cache(maxsize=8096) +def _get_annotated_type(type_: type) -> type | None: + """If the given type is an `Annotated` type then it is returned, if not `None` is returned. + + This also unwraps the type when applicable, e.g. `Required[Annotated[T, ...]]` + """ + if is_required_type(type_): + # Unwrap `Required[Annotated[T, ...]]` to `Annotated[T, ...]` + type_ = get_args(type_)[0] + + if is_annotated_type(type_): + return type_ + + return None + + +def _maybe_transform_key(key: str, type_: type) -> str: + """Transform the given `data` based on the annotations provided in `type_`. + + Note: this function only looks at `Annotated` types that contain `PropertyInfo` metadata. + """ + annotated_type = _get_annotated_type(type_) + if annotated_type is None: + # no `Annotated` definition for this type, no transformation needed + return key + + # ignore the first argument as it is the actual type + annotations = get_args(annotated_type)[1:] + for annotation in annotations: + if isinstance(annotation, PropertyInfo) and annotation.alias is not None: + return annotation.alias + + return key + + +def _no_transform_needed(annotation: type) -> bool: + return annotation == float or annotation == int + + +def _transform_recursive( + data: object, + *, + annotation: type, + inner_type: type | None = None, +) -> object: + """Transform the given data against the expected type. + + Args: + annotation: The direct type annotation given to the particular piece of data. + This may or may not be wrapped in metadata types, e.g. `Required[T]`, `Annotated[T, ...]` etc + + inner_type: If applicable, this is the "inside" type. This is useful in certain cases where the outside type + is a container type such as `List[T]`. In that case `inner_type` should be set to `T` so that each entry in + the list can be transformed using the metadata from the container type. + + Defaults to the same value as the `annotation` argument. + """ + if inner_type is None: + inner_type = annotation + + stripped_type = strip_annotated_type(inner_type) + origin = get_origin(stripped_type) or stripped_type + if is_typeddict(stripped_type) and is_mapping(data): + return _transform_typeddict(data, stripped_type) + + if origin == dict and is_mapping(data): + items_type = get_args(stripped_type)[1] + return {key: _transform_recursive(value, annotation=items_type) for key, value in data.items()} + + if ( + # List[T] + (is_list_type(stripped_type) and is_list(data)) + # Iterable[T] + or (is_iterable_type(stripped_type) and is_iterable(data) and not isinstance(data, str)) + ): + # dicts are technically iterable, but it is an iterable on the keys of the dict and is not usually + # intended as an iterable, so we don't transform it. + if isinstance(data, dict): + return cast(object, data) + + inner_type = extract_type_arg(stripped_type, 0) + if _no_transform_needed(inner_type): + # for some types there is no need to transform anything, so we can get a small + # perf boost from skipping that work. + # + # but we still need to convert to a list to ensure the data is json-serializable + if is_list(data): + return data + return list(data) + + return [_transform_recursive(d, annotation=annotation, inner_type=inner_type) for d in data] + + if is_union_type(stripped_type): + # For union types we run the transformation against all subtypes to ensure that everything is transformed. + # + # TODO: there may be edge cases where the same normalized field name will transform to two different names + # in different subtypes. + for subtype in get_args(stripped_type): + data = _transform_recursive(data, annotation=annotation, inner_type=subtype) + return data + + if isinstance(data, pydantic.BaseModel): + return model_dump(data, exclude_unset=True, mode="json") + + annotated_type = _get_annotated_type(annotation) + if annotated_type is None: + return data + + # ignore the first argument as it is the actual type + annotations = get_args(annotated_type)[1:] + for annotation in annotations: + if isinstance(annotation, PropertyInfo) and annotation.format is not None: + return _format_data(data, annotation.format, annotation.format_template) + + return data + + +def _format_data(data: object, format_: PropertyFormat, format_template: str | None) -> object: + if isinstance(data, (date, datetime)): + if format_ == "iso8601": + return data.isoformat() + + if format_ == "custom" and format_template is not None: + return data.strftime(format_template) + + if format_ == "base64" and is_base64_file_input(data): + binary: str | bytes | None = None + + if isinstance(data, pathlib.Path): + binary = data.read_bytes() + elif isinstance(data, io.IOBase): + binary = data.read() + + if isinstance(binary, str): # type: ignore[unreachable] + binary = binary.encode() + + if not isinstance(binary, bytes): + raise RuntimeError(f"Could not read bytes from {data}; Received {type(binary)}") + + return base64.b64encode(binary).decode("ascii") + + return data + + +def _transform_typeddict( + data: Mapping[str, object], + expected_type: type, +) -> Mapping[str, object]: + result: dict[str, object] = {} + annotations = get_type_hints(expected_type, include_extras=True) + for key, value in data.items(): + if not is_given(value): + # we don't need to include `NotGiven` values here as they'll + # be stripped out before the request is sent anyway + continue + + type_ = annotations.get(key) + if type_ is None: + # we do not have a type annotation for this field, leave it as is + result[key] = value + else: + result[_maybe_transform_key(key, type_)] = _transform_recursive(value, annotation=type_) + return result + + +async def async_maybe_transform( + data: object, + expected_type: object, +) -> Any | None: + """Wrapper over `async_transform()` that allows `None` to be passed. + + See `async_transform()` for more details. + """ + if data is None: + return None + return await async_transform(data, expected_type) + + +async def async_transform( + data: _T, + expected_type: object, +) -> _T: + """Transform dictionaries based off of type information from the given type, for example: + + ```py + class Params(TypedDict, total=False): + card_id: Required[Annotated[str, PropertyInfo(alias="cardID")]] + + + transformed = transform({"card_id": ""}, Params) + # {'cardID': ''} + ``` + + Any keys / data that does not have type information given will be included as is. + + It should be noted that the transformations that this function does are not represented in the type system. + """ + transformed = await _async_transform_recursive(data, annotation=cast(type, expected_type)) + return cast(_T, transformed) + + +async def _async_transform_recursive( + data: object, + *, + annotation: type, + inner_type: type | None = None, +) -> object: + """Transform the given data against the expected type. + + Args: + annotation: The direct type annotation given to the particular piece of data. + This may or may not be wrapped in metadata types, e.g. `Required[T]`, `Annotated[T, ...]` etc + + inner_type: If applicable, this is the "inside" type. This is useful in certain cases where the outside type + is a container type such as `List[T]`. In that case `inner_type` should be set to `T` so that each entry in + the list can be transformed using the metadata from the container type. + + Defaults to the same value as the `annotation` argument. + """ + if inner_type is None: + inner_type = annotation + + stripped_type = strip_annotated_type(inner_type) + origin = get_origin(stripped_type) or stripped_type + if is_typeddict(stripped_type) and is_mapping(data): + return await _async_transform_typeddict(data, stripped_type) + + if origin == dict and is_mapping(data): + items_type = get_args(stripped_type)[1] + return {key: _transform_recursive(value, annotation=items_type) for key, value in data.items()} + + if ( + # List[T] + (is_list_type(stripped_type) and is_list(data)) + # Iterable[T] + or (is_iterable_type(stripped_type) and is_iterable(data) and not isinstance(data, str)) + ): + # dicts are technically iterable, but it is an iterable on the keys of the dict and is not usually + # intended as an iterable, so we don't transform it. + if isinstance(data, dict): + return cast(object, data) + + inner_type = extract_type_arg(stripped_type, 0) + if _no_transform_needed(inner_type): + # for some types there is no need to transform anything, so we can get a small + # perf boost from skipping that work. + # + # but we still need to convert to a list to ensure the data is json-serializable + if is_list(data): + return data + return list(data) + + return [await _async_transform_recursive(d, annotation=annotation, inner_type=inner_type) for d in data] + + if is_union_type(stripped_type): + # For union types we run the transformation against all subtypes to ensure that everything is transformed. + # + # TODO: there may be edge cases where the same normalized field name will transform to two different names + # in different subtypes. + for subtype in get_args(stripped_type): + data = await _async_transform_recursive(data, annotation=annotation, inner_type=subtype) + return data + + if isinstance(data, pydantic.BaseModel): + return model_dump(data, exclude_unset=True, mode="json") + + annotated_type = _get_annotated_type(annotation) + if annotated_type is None: + return data + + # ignore the first argument as it is the actual type + annotations = get_args(annotated_type)[1:] + for annotation in annotations: + if isinstance(annotation, PropertyInfo) and annotation.format is not None: + return await _async_format_data(data, annotation.format, annotation.format_template) + + return data + + +async def _async_format_data(data: object, format_: PropertyFormat, format_template: str | None) -> object: + if isinstance(data, (date, datetime)): + if format_ == "iso8601": + return data.isoformat() + + if format_ == "custom" and format_template is not None: + return data.strftime(format_template) + + if format_ == "base64" and is_base64_file_input(data): + binary: str | bytes | None = None + + if isinstance(data, pathlib.Path): + binary = await anyio.Path(data).read_bytes() + elif isinstance(data, io.IOBase): + binary = data.read() + + if isinstance(binary, str): # type: ignore[unreachable] + binary = binary.encode() + + if not isinstance(binary, bytes): + raise RuntimeError(f"Could not read bytes from {data}; Received {type(binary)}") + + return base64.b64encode(binary).decode("ascii") + + return data + + +async def _async_transform_typeddict( + data: Mapping[str, object], + expected_type: type, +) -> Mapping[str, object]: + result: dict[str, object] = {} + annotations = get_type_hints(expected_type, include_extras=True) + for key, value in data.items(): + if not is_given(value): + # we don't need to include `NotGiven` values here as they'll + # be stripped out before the request is sent anyway + continue + + type_ = annotations.get(key) + if type_ is None: + # we do not have a type annotation for this field, leave it as is + result[key] = value + else: + result[_maybe_transform_key(key, type_)] = await _async_transform_recursive(value, annotation=type_) + return result + + +@lru_cache(maxsize=8096) +def get_type_hints( + obj: Any, + globalns: dict[str, Any] | None = None, + localns: Mapping[str, Any] | None = None, + include_extras: bool = False, +) -> dict[str, Any]: + return _get_type_hints(obj, globalns=globalns, localns=localns, include_extras=include_extras) diff --git a/lib/python3.12/site-packages/openai/_utils/_typing.py b/lib/python3.12/site-packages/openai/_utils/_typing.py new file mode 100644 index 0000000000000000000000000000000000000000..1bac9542e20f6d3fc44352c86bf6fa151d34cb35 --- /dev/null +++ b/lib/python3.12/site-packages/openai/_utils/_typing.py @@ -0,0 +1,151 @@ +from __future__ import annotations + +import sys +import typing +import typing_extensions +from typing import Any, TypeVar, Iterable, cast +from collections import abc as _c_abc +from typing_extensions import ( + TypeIs, + Required, + Annotated, + get_args, + get_origin, +) + +from ._utils import lru_cache +from .._types import InheritsGeneric +from .._compat import is_union as _is_union + + +def is_annotated_type(typ: type) -> bool: + return get_origin(typ) == Annotated + + +def is_list_type(typ: type) -> bool: + return (get_origin(typ) or typ) == list + + +def is_iterable_type(typ: type) -> bool: + """If the given type is `typing.Iterable[T]`""" + origin = get_origin(typ) or typ + return origin == Iterable or origin == _c_abc.Iterable + + +def is_union_type(typ: type) -> bool: + return _is_union(get_origin(typ)) + + +def is_required_type(typ: type) -> bool: + return get_origin(typ) == Required + + +def is_typevar(typ: type) -> bool: + # type ignore is required because type checkers + # think this expression will always return False + return type(typ) == TypeVar # type: ignore + + +_TYPE_ALIAS_TYPES: tuple[type[typing_extensions.TypeAliasType], ...] = (typing_extensions.TypeAliasType,) +if sys.version_info >= (3, 12): + _TYPE_ALIAS_TYPES = (*_TYPE_ALIAS_TYPES, typing.TypeAliasType) + + +def is_type_alias_type(tp: Any, /) -> TypeIs[typing_extensions.TypeAliasType]: + """Return whether the provided argument is an instance of `TypeAliasType`. + + ```python + type Int = int + is_type_alias_type(Int) + # > True + Str = TypeAliasType("Str", str) + is_type_alias_type(Str) + # > True + ``` + """ + return isinstance(tp, _TYPE_ALIAS_TYPES) + + +# Extracts T from Annotated[T, ...] or from Required[Annotated[T, ...]] +@lru_cache(maxsize=8096) +def strip_annotated_type(typ: type) -> type: + if is_required_type(typ) or is_annotated_type(typ): + return strip_annotated_type(cast(type, get_args(typ)[0])) + + return typ + + +def extract_type_arg(typ: type, index: int) -> type: + args = get_args(typ) + try: + return cast(type, args[index]) + except IndexError as err: + raise RuntimeError(f"Expected type {typ} to have a type argument at index {index} but it did not") from err + + +def extract_type_var_from_base( + typ: type, + *, + generic_bases: tuple[type, ...], + index: int, + failure_message: str | None = None, +) -> type: + """Given a type like `Foo[T]`, returns the generic type variable `T`. + + This also handles the case where a concrete subclass is given, e.g. + ```py + class MyResponse(Foo[bytes]): + ... + + extract_type_var(MyResponse, bases=(Foo,), index=0) -> bytes + ``` + + And where a generic subclass is given: + ```py + _T = TypeVar('_T') + class MyResponse(Foo[_T]): + ... + + extract_type_var(MyResponse[bytes], bases=(Foo,), index=0) -> bytes + ``` + """ + cls = cast(object, get_origin(typ) or typ) + if cls in generic_bases: # pyright: ignore[reportUnnecessaryContains] + # we're given the class directly + return extract_type_arg(typ, index) + + # if a subclass is given + # --- + # this is needed as __orig_bases__ is not present in the typeshed stubs + # because it is intended to be for internal use only, however there does + # not seem to be a way to resolve generic TypeVars for inherited subclasses + # without using it. + if isinstance(cls, InheritsGeneric): + target_base_class: Any | None = None + for base in cls.__orig_bases__: + if base.__origin__ in generic_bases: + target_base_class = base + break + + if target_base_class is None: + raise RuntimeError( + "Could not find the generic base class;\n" + "This should never happen;\n" + f"Does {cls} inherit from one of {generic_bases} ?" + ) + + extracted = extract_type_arg(target_base_class, index) + if is_typevar(extracted): + # If the extracted type argument is itself a type variable + # then that means the subclass itself is generic, so we have + # to resolve the type argument from the class itself, not + # the base class. + # + # Note: if there is more than 1 type argument, the subclass could + # change the ordering of the type arguments, this is not currently + # supported. + return extract_type_arg(typ, index) + + return extracted + + raise RuntimeError(failure_message or f"Could not resolve inner type variable at index {index} for {typ}") diff --git a/lib/python3.12/site-packages/openai/_utils/_utils.py b/lib/python3.12/site-packages/openai/_utils/_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..d6734e6b8faa53d20446140acb6f7cc0f81a0e23 --- /dev/null +++ b/lib/python3.12/site-packages/openai/_utils/_utils.py @@ -0,0 +1,430 @@ +from __future__ import annotations + +import os +import re +import inspect +import functools +from typing import ( + TYPE_CHECKING, + Any, + Tuple, + Mapping, + TypeVar, + Callable, + Iterable, + Sequence, + cast, + overload, +) +from pathlib import Path +from datetime import date, datetime +from typing_extensions import TypeGuard + +import sniffio + +from .._types import NotGiven, FileTypes, NotGivenOr, HeadersLike +from .._compat import parse_date as parse_date, parse_datetime as parse_datetime + +_T = TypeVar("_T") +_TupleT = TypeVar("_TupleT", bound=Tuple[object, ...]) +_MappingT = TypeVar("_MappingT", bound=Mapping[str, object]) +_SequenceT = TypeVar("_SequenceT", bound=Sequence[object]) +CallableT = TypeVar("CallableT", bound=Callable[..., Any]) + +if TYPE_CHECKING: + from ..lib.azure import AzureOpenAI, AsyncAzureOpenAI + + +def flatten(t: Iterable[Iterable[_T]]) -> list[_T]: + return [item for sublist in t for item in sublist] + + +def extract_files( + # TODO: this needs to take Dict but variance issues..... + # create protocol type ? + query: Mapping[str, object], + *, + paths: Sequence[Sequence[str]], +) -> list[tuple[str, FileTypes]]: + """Recursively extract files from the given dictionary based on specified paths. + + A path may look like this ['foo', 'files', '', 'data']. + + Note: this mutates the given dictionary. + """ + files: list[tuple[str, FileTypes]] = [] + for path in paths: + files.extend(_extract_items(query, path, index=0, flattened_key=None)) + return files + + +def _extract_items( + obj: object, + path: Sequence[str], + *, + index: int, + flattened_key: str | None, +) -> list[tuple[str, FileTypes]]: + try: + key = path[index] + except IndexError: + if isinstance(obj, NotGiven): + # no value was provided - we can safely ignore + return [] + + # cyclical import + from .._files import assert_is_file_content + + # We have exhausted the path, return the entry we found. + assert_is_file_content(obj, key=flattened_key) + assert flattened_key is not None + return [(flattened_key, cast(FileTypes, obj))] + + index += 1 + if is_dict(obj): + try: + # We are at the last entry in the path so we must remove the field + if (len(path)) == index: + item = obj.pop(key) + else: + item = obj[key] + except KeyError: + # Key was not present in the dictionary, this is not indicative of an error + # as the given path may not point to a required field. We also do not want + # to enforce required fields as the API may differ from the spec in some cases. + return [] + if flattened_key is None: + flattened_key = key + else: + flattened_key += f"[{key}]" + return _extract_items( + item, + path, + index=index, + flattened_key=flattened_key, + ) + elif is_list(obj): + if key != "": + return [] + + return flatten( + [ + _extract_items( + item, + path, + index=index, + flattened_key=flattened_key + "[]" if flattened_key is not None else "[]", + ) + for item in obj + ] + ) + + # Something unexpected was passed, just ignore it. + return [] + + +def is_given(obj: NotGivenOr[_T]) -> TypeGuard[_T]: + return not isinstance(obj, NotGiven) + + +# Type safe methods for narrowing types with TypeVars. +# The default narrowing for isinstance(obj, dict) is dict[unknown, unknown], +# however this cause Pyright to rightfully report errors. As we know we don't +# care about the contained types we can safely use `object` in it's place. +# +# There are two separate functions defined, `is_*` and `is_*_t` for different use cases. +# `is_*` is for when you're dealing with an unknown input +# `is_*_t` is for when you're narrowing a known union type to a specific subset + + +def is_tuple(obj: object) -> TypeGuard[tuple[object, ...]]: + return isinstance(obj, tuple) + + +def is_tuple_t(obj: _TupleT | object) -> TypeGuard[_TupleT]: + return isinstance(obj, tuple) + + +def is_sequence(obj: object) -> TypeGuard[Sequence[object]]: + return isinstance(obj, Sequence) + + +def is_sequence_t(obj: _SequenceT | object) -> TypeGuard[_SequenceT]: + return isinstance(obj, Sequence) + + +def is_mapping(obj: object) -> TypeGuard[Mapping[str, object]]: + return isinstance(obj, Mapping) + + +def is_mapping_t(obj: _MappingT | object) -> TypeGuard[_MappingT]: + return isinstance(obj, Mapping) + + +def is_dict(obj: object) -> TypeGuard[dict[object, object]]: + return isinstance(obj, dict) + + +def is_list(obj: object) -> TypeGuard[list[object]]: + return isinstance(obj, list) + + +def is_iterable(obj: object) -> TypeGuard[Iterable[object]]: + return isinstance(obj, Iterable) + + +def deepcopy_minimal(item: _T) -> _T: + """Minimal reimplementation of copy.deepcopy() that will only copy certain object types: + + - mappings, e.g. `dict` + - list + + This is done for performance reasons. + """ + if is_mapping(item): + return cast(_T, {k: deepcopy_minimal(v) for k, v in item.items()}) + if is_list(item): + return cast(_T, [deepcopy_minimal(entry) for entry in item]) + return item + + +# copied from https://github.com/Rapptz/RoboDanny +def human_join(seq: Sequence[str], *, delim: str = ", ", final: str = "or") -> str: + size = len(seq) + if size == 0: + return "" + + if size == 1: + return seq[0] + + if size == 2: + return f"{seq[0]} {final} {seq[1]}" + + return delim.join(seq[:-1]) + f" {final} {seq[-1]}" + + +def quote(string: str) -> str: + """Add single quotation marks around the given string. Does *not* do any escaping.""" + return f"'{string}'" + + +def required_args(*variants: Sequence[str]) -> Callable[[CallableT], CallableT]: + """Decorator to enforce a given set of arguments or variants of arguments are passed to the decorated function. + + Useful for enforcing runtime validation of overloaded functions. + + Example usage: + ```py + @overload + def foo(*, a: str) -> str: ... + + + @overload + def foo(*, b: bool) -> str: ... + + + # This enforces the same constraints that a static type checker would + # i.e. that either a or b must be passed to the function + @required_args(["a"], ["b"]) + def foo(*, a: str | None = None, b: bool | None = None) -> str: ... + ``` + """ + + def inner(func: CallableT) -> CallableT: + params = inspect.signature(func).parameters + positional = [ + name + for name, param in params.items() + if param.kind + in { + param.POSITIONAL_ONLY, + param.POSITIONAL_OR_KEYWORD, + } + ] + + @functools.wraps(func) + def wrapper(*args: object, **kwargs: object) -> object: + given_params: set[str] = set() + for i, _ in enumerate(args): + try: + given_params.add(positional[i]) + except IndexError: + raise TypeError( + f"{func.__name__}() takes {len(positional)} argument(s) but {len(args)} were given" + ) from None + + for key in kwargs.keys(): + given_params.add(key) + + for variant in variants: + matches = all((param in given_params for param in variant)) + if matches: + break + else: # no break + if len(variants) > 1: + variations = human_join( + ["(" + human_join([quote(arg) for arg in variant], final="and") + ")" for variant in variants] + ) + msg = f"Missing required arguments; Expected either {variations} arguments to be given" + else: + assert len(variants) > 0 + + # TODO: this error message is not deterministic + missing = list(set(variants[0]) - given_params) + if len(missing) > 1: + msg = f"Missing required arguments: {human_join([quote(arg) for arg in missing])}" + else: + msg = f"Missing required argument: {quote(missing[0])}" + raise TypeError(msg) + return func(*args, **kwargs) + + return wrapper # type: ignore + + return inner + + +_K = TypeVar("_K") +_V = TypeVar("_V") + + +@overload +def strip_not_given(obj: None) -> None: ... + + +@overload +def strip_not_given(obj: Mapping[_K, _V | NotGiven]) -> dict[_K, _V]: ... + + +@overload +def strip_not_given(obj: object) -> object: ... + + +def strip_not_given(obj: object | None) -> object: + """Remove all top-level keys where their values are instances of `NotGiven`""" + if obj is None: + return None + + if not is_mapping(obj): + return obj + + return {key: value for key, value in obj.items() if not isinstance(value, NotGiven)} + + +def coerce_integer(val: str) -> int: + return int(val, base=10) + + +def coerce_float(val: str) -> float: + return float(val) + + +def coerce_boolean(val: str) -> bool: + return val == "true" or val == "1" or val == "on" + + +def maybe_coerce_integer(val: str | None) -> int | None: + if val is None: + return None + return coerce_integer(val) + + +def maybe_coerce_float(val: str | None) -> float | None: + if val is None: + return None + return coerce_float(val) + + +def maybe_coerce_boolean(val: str | None) -> bool | None: + if val is None: + return None + return coerce_boolean(val) + + +def removeprefix(string: str, prefix: str) -> str: + """Remove a prefix from a string. + + Backport of `str.removeprefix` for Python < 3.9 + """ + if string.startswith(prefix): + return string[len(prefix) :] + return string + + +def removesuffix(string: str, suffix: str) -> str: + """Remove a suffix from a string. + + Backport of `str.removesuffix` for Python < 3.9 + """ + if string.endswith(suffix): + return string[: -len(suffix)] + return string + + +def file_from_path(path: str) -> FileTypes: + contents = Path(path).read_bytes() + file_name = os.path.basename(path) + return (file_name, contents) + + +def get_required_header(headers: HeadersLike, header: str) -> str: + lower_header = header.lower() + if is_mapping_t(headers): + # mypy doesn't understand the type narrowing here + for k, v in headers.items(): # type: ignore + if k.lower() == lower_header and isinstance(v, str): + return v + + # to deal with the case where the header looks like Stainless-Event-Id + intercaps_header = re.sub(r"([^\w])(\w)", lambda pat: pat.group(1) + pat.group(2).upper(), header.capitalize()) + + for normalized_header in [header, lower_header, header.upper(), intercaps_header]: + value = headers.get(normalized_header) + if value: + return value + + raise ValueError(f"Could not find {header} header") + + +def get_async_library() -> str: + try: + return sniffio.current_async_library() + except Exception: + return "false" + + +def lru_cache(*, maxsize: int | None = 128) -> Callable[[CallableT], CallableT]: + """A version of functools.lru_cache that retains the type signature + for the wrapped function arguments. + """ + wrapper = functools.lru_cache( # noqa: TID251 + maxsize=maxsize, + ) + return cast(Any, wrapper) # type: ignore[no-any-return] + + +def json_safe(data: object) -> object: + """Translates a mapping / sequence recursively in the same fashion + as `pydantic` v2's `model_dump(mode="json")`. + """ + if is_mapping(data): + return {json_safe(key): json_safe(value) for key, value in data.items()} + + if is_iterable(data) and not isinstance(data, (str, bytes, bytearray)): + return [json_safe(item) for item in data] + + if isinstance(data, (datetime, date)): + return data.isoformat() + + return data + + +def is_azure_client(client: object) -> TypeGuard[AzureOpenAI]: + from ..lib.azure import AzureOpenAI + + return isinstance(client, AzureOpenAI) + + +def is_async_azure_client(client: object) -> TypeGuard[AsyncAzureOpenAI]: + from ..lib.azure import AsyncAzureOpenAI + + return isinstance(client, AsyncAzureOpenAI) diff --git a/lib/python3.12/site-packages/openai/cli/__init__.py b/lib/python3.12/site-packages/openai/cli/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d453d5e17948a20250a79a2b4d81b6df74719418 --- /dev/null +++ b/lib/python3.12/site-packages/openai/cli/__init__.py @@ -0,0 +1 @@ +from ._cli import main as main diff --git a/lib/python3.12/site-packages/openai/cli/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/openai/cli/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..17d6ae8beba32c62d75bde536bf7cdbe92487c24 Binary files /dev/null and b/lib/python3.12/site-packages/openai/cli/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/cli/__pycache__/_cli.cpython-312.pyc b/lib/python3.12/site-packages/openai/cli/__pycache__/_cli.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f286ce8c12e9d8a49745989761e3004b76bcd2f7 Binary files /dev/null and b/lib/python3.12/site-packages/openai/cli/__pycache__/_cli.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/cli/__pycache__/_errors.cpython-312.pyc b/lib/python3.12/site-packages/openai/cli/__pycache__/_errors.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..33301a80c568f04d5eef2eea6cf8603cc4c1adbd Binary files /dev/null and b/lib/python3.12/site-packages/openai/cli/__pycache__/_errors.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/cli/__pycache__/_models.cpython-312.pyc b/lib/python3.12/site-packages/openai/cli/__pycache__/_models.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..af20d6ca7ef18be4233a282b2d32d0aa67e75e92 Binary files /dev/null and b/lib/python3.12/site-packages/openai/cli/__pycache__/_models.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/cli/__pycache__/_progress.cpython-312.pyc b/lib/python3.12/site-packages/openai/cli/__pycache__/_progress.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..348a877360136fa38c3ce67ad10e58bd77b8f05b Binary files /dev/null and b/lib/python3.12/site-packages/openai/cli/__pycache__/_progress.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/cli/__pycache__/_utils.cpython-312.pyc b/lib/python3.12/site-packages/openai/cli/__pycache__/_utils.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c91b294527632deee01c90935db6c310b7ef0108 Binary files /dev/null and b/lib/python3.12/site-packages/openai/cli/__pycache__/_utils.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/cli/_api/__init__.py b/lib/python3.12/site-packages/openai/cli/_api/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..56a0260a6dc333995ff975158452eca83fc66012 --- /dev/null +++ b/lib/python3.12/site-packages/openai/cli/_api/__init__.py @@ -0,0 +1 @@ +from ._main import register_commands as register_commands diff --git a/lib/python3.12/site-packages/openai/cli/_api/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/openai/cli/_api/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..712b0f4c5995322e4a0c024e594de0164b99554d Binary files /dev/null and b/lib/python3.12/site-packages/openai/cli/_api/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/cli/_api/__pycache__/_main.cpython-312.pyc b/lib/python3.12/site-packages/openai/cli/_api/__pycache__/_main.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..da47d43018825cf02134743655ed2b1ca61ab57e Binary files /dev/null and b/lib/python3.12/site-packages/openai/cli/_api/__pycache__/_main.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/cli/_api/__pycache__/audio.cpython-312.pyc b/lib/python3.12/site-packages/openai/cli/_api/__pycache__/audio.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..014800d91b62758271b8e067102edfbdab308193 Binary files /dev/null and b/lib/python3.12/site-packages/openai/cli/_api/__pycache__/audio.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/cli/_api/__pycache__/completions.cpython-312.pyc b/lib/python3.12/site-packages/openai/cli/_api/__pycache__/completions.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..de94e7e042aa4430cf8da59f4da7c3638ea9a121 Binary files /dev/null and b/lib/python3.12/site-packages/openai/cli/_api/__pycache__/completions.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/cli/_api/__pycache__/files.cpython-312.pyc b/lib/python3.12/site-packages/openai/cli/_api/__pycache__/files.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..207c1373877cb83cca2d976a42abe1620d34751b Binary files /dev/null and b/lib/python3.12/site-packages/openai/cli/_api/__pycache__/files.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/cli/_api/__pycache__/image.cpython-312.pyc b/lib/python3.12/site-packages/openai/cli/_api/__pycache__/image.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a64c791b0896ce5f69137bc8b2eb791694845221 Binary files /dev/null and b/lib/python3.12/site-packages/openai/cli/_api/__pycache__/image.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/cli/_api/__pycache__/models.cpython-312.pyc b/lib/python3.12/site-packages/openai/cli/_api/__pycache__/models.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a726ae2c37a7688e2f2a16fc64d7bda465c0484f Binary files /dev/null and b/lib/python3.12/site-packages/openai/cli/_api/__pycache__/models.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/cli/_api/_main.py b/lib/python3.12/site-packages/openai/cli/_api/_main.py new file mode 100644 index 0000000000000000000000000000000000000000..fe5a5e6fc0b033a55f01cb07557136b59cec3d09 --- /dev/null +++ b/lib/python3.12/site-packages/openai/cli/_api/_main.py @@ -0,0 +1,16 @@ +from __future__ import annotations + +from argparse import ArgumentParser + +from . import chat, audio, files, image, models, completions + + +def register_commands(parser: ArgumentParser) -> None: + subparsers = parser.add_subparsers(help="All API subcommands") + + chat.register(subparsers) + image.register(subparsers) + audio.register(subparsers) + files.register(subparsers) + models.register(subparsers) + completions.register(subparsers) diff --git a/lib/python3.12/site-packages/openai/cli/_api/audio.py b/lib/python3.12/site-packages/openai/cli/_api/audio.py new file mode 100644 index 0000000000000000000000000000000000000000..269c67df28d477273d408a33be90ca59b3f91427 --- /dev/null +++ b/lib/python3.12/site-packages/openai/cli/_api/audio.py @@ -0,0 +1,108 @@ +from __future__ import annotations + +import sys +from typing import TYPE_CHECKING, Any, Optional, cast +from argparse import ArgumentParser + +from .._utils import get_client, print_model +from ..._types import NOT_GIVEN +from .._models import BaseModel +from .._progress import BufferReader +from ...types.audio import Transcription + +if TYPE_CHECKING: + from argparse import _SubParsersAction + + +def register(subparser: _SubParsersAction[ArgumentParser]) -> None: + # transcriptions + sub = subparser.add_parser("audio.transcriptions.create") + + # Required + sub.add_argument("-m", "--model", type=str, default="whisper-1") + sub.add_argument("-f", "--file", type=str, required=True) + # Optional + sub.add_argument("--response-format", type=str) + sub.add_argument("--language", type=str) + sub.add_argument("-t", "--temperature", type=float) + sub.add_argument("--prompt", type=str) + sub.set_defaults(func=CLIAudio.transcribe, args_model=CLITranscribeArgs) + + # translations + sub = subparser.add_parser("audio.translations.create") + + # Required + sub.add_argument("-f", "--file", type=str, required=True) + # Optional + sub.add_argument("-m", "--model", type=str, default="whisper-1") + sub.add_argument("--response-format", type=str) + # TODO: doesn't seem to be supported by the API + # sub.add_argument("--language", type=str) + sub.add_argument("-t", "--temperature", type=float) + sub.add_argument("--prompt", type=str) + sub.set_defaults(func=CLIAudio.translate, args_model=CLITranslationArgs) + + +class CLITranscribeArgs(BaseModel): + model: str + file: str + response_format: Optional[str] = None + language: Optional[str] = None + temperature: Optional[float] = None + prompt: Optional[str] = None + + +class CLITranslationArgs(BaseModel): + model: str + file: str + response_format: Optional[str] = None + language: Optional[str] = None + temperature: Optional[float] = None + prompt: Optional[str] = None + + +class CLIAudio: + @staticmethod + def transcribe(args: CLITranscribeArgs) -> None: + with open(args.file, "rb") as file_reader: + buffer_reader = BufferReader(file_reader.read(), desc="Upload progress") + + model = cast( + "Transcription | str", + get_client().audio.transcriptions.create( + file=(args.file, buffer_reader), + model=args.model, + language=args.language or NOT_GIVEN, + temperature=args.temperature or NOT_GIVEN, + prompt=args.prompt or NOT_GIVEN, + # casts required because the API is typed for enums + # but we don't want to validate that here for forwards-compat + response_format=cast(Any, args.response_format), + ), + ) + if isinstance(model, str): + sys.stdout.write(model + "\n") + else: + print_model(model) + + @staticmethod + def translate(args: CLITranslationArgs) -> None: + with open(args.file, "rb") as file_reader: + buffer_reader = BufferReader(file_reader.read(), desc="Upload progress") + + model = cast( + "Transcription | str", + get_client().audio.translations.create( + file=(args.file, buffer_reader), + model=args.model, + temperature=args.temperature or NOT_GIVEN, + prompt=args.prompt or NOT_GIVEN, + # casts required because the API is typed for enums + # but we don't want to validate that here for forwards-compat + response_format=cast(Any, args.response_format), + ), + ) + if isinstance(model, str): + sys.stdout.write(model + "\n") + else: + print_model(model) diff --git a/lib/python3.12/site-packages/openai/cli/_api/chat/__init__.py b/lib/python3.12/site-packages/openai/cli/_api/chat/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..87d971630aaef250bf91add818cb0db0a70e1410 --- /dev/null +++ b/lib/python3.12/site-packages/openai/cli/_api/chat/__init__.py @@ -0,0 +1,13 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING +from argparse import ArgumentParser + +from . import completions + +if TYPE_CHECKING: + from argparse import _SubParsersAction + + +def register(subparser: _SubParsersAction[ArgumentParser]) -> None: + completions.register(subparser) diff --git a/lib/python3.12/site-packages/openai/cli/_api/chat/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/openai/cli/_api/chat/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4b7d4e4b1b023aaf7991490d0fcd56347c01c357 Binary files /dev/null and b/lib/python3.12/site-packages/openai/cli/_api/chat/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/cli/_api/chat/__pycache__/completions.cpython-312.pyc b/lib/python3.12/site-packages/openai/cli/_api/chat/__pycache__/completions.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2ca3d75ddae8a2dec3f47768688d45659e3b2ada Binary files /dev/null and b/lib/python3.12/site-packages/openai/cli/_api/chat/__pycache__/completions.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/cli/_api/chat/completions.py b/lib/python3.12/site-packages/openai/cli/_api/chat/completions.py new file mode 100644 index 0000000000000000000000000000000000000000..344eeff37cc2675da69174016ffb5879126fce7b --- /dev/null +++ b/lib/python3.12/site-packages/openai/cli/_api/chat/completions.py @@ -0,0 +1,160 @@ +from __future__ import annotations + +import sys +from typing import TYPE_CHECKING, List, Optional, cast +from argparse import ArgumentParser +from typing_extensions import Literal, NamedTuple + +from ..._utils import get_client +from ..._models import BaseModel +from ...._streaming import Stream +from ....types.chat import ( + ChatCompletionRole, + ChatCompletionChunk, + CompletionCreateParams, +) +from ....types.chat.completion_create_params import ( + CompletionCreateParamsStreaming, + CompletionCreateParamsNonStreaming, +) + +if TYPE_CHECKING: + from argparse import _SubParsersAction + + +def register(subparser: _SubParsersAction[ArgumentParser]) -> None: + sub = subparser.add_parser("chat.completions.create") + + sub._action_groups.pop() + req = sub.add_argument_group("required arguments") + opt = sub.add_argument_group("optional arguments") + + req.add_argument( + "-g", + "--message", + action="append", + nargs=2, + metavar=("ROLE", "CONTENT"), + help="A message in `{role} {content}` format. Use this argument multiple times to add multiple messages.", + required=True, + ) + req.add_argument( + "-m", + "--model", + help="The model to use.", + required=True, + ) + + opt.add_argument( + "-n", + "--n", + help="How many completions to generate for the conversation.", + type=int, + ) + opt.add_argument("-M", "--max-tokens", help="The maximum number of tokens to generate.", type=int) + opt.add_argument( + "-t", + "--temperature", + help="""What sampling temperature to use. Higher values means the model will take more risks. Try 0.9 for more creative applications, and 0 (argmax sampling) for ones with a well-defined answer. + +Mutually exclusive with `top_p`.""", + type=float, + ) + opt.add_argument( + "-P", + "--top_p", + help="""An alternative to sampling with temperature, called nucleus sampling, where the considers the results of the tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10%% probability mass are considered. + + Mutually exclusive with `temperature`.""", + type=float, + ) + opt.add_argument( + "--stop", + help="A stop sequence at which to stop generating tokens for the message.", + ) + opt.add_argument("--stream", help="Stream messages as they're ready.", action="store_true") + sub.set_defaults(func=CLIChatCompletion.create, args_model=CLIChatCompletionCreateArgs) + + +class CLIMessage(NamedTuple): + role: ChatCompletionRole + content: str + + +class CLIChatCompletionCreateArgs(BaseModel): + message: List[CLIMessage] + model: str + n: Optional[int] = None + max_tokens: Optional[int] = None + temperature: Optional[float] = None + top_p: Optional[float] = None + stop: Optional[str] = None + stream: bool = False + + +class CLIChatCompletion: + @staticmethod + def create(args: CLIChatCompletionCreateArgs) -> None: + params: CompletionCreateParams = { + "model": args.model, + "messages": [ + {"role": cast(Literal["user"], message.role), "content": message.content} for message in args.message + ], + # type checkers are not good at inferring union types so we have to set stream afterwards + "stream": False, + } + if args.temperature is not None: + params["temperature"] = args.temperature + if args.stop is not None: + params["stop"] = args.stop + if args.top_p is not None: + params["top_p"] = args.top_p + if args.n is not None: + params["n"] = args.n + if args.stream: + params["stream"] = args.stream # type: ignore + if args.max_tokens is not None: + params["max_tokens"] = args.max_tokens + + if args.stream: + return CLIChatCompletion._stream_create(cast(CompletionCreateParamsStreaming, params)) + + return CLIChatCompletion._create(cast(CompletionCreateParamsNonStreaming, params)) + + @staticmethod + def _create(params: CompletionCreateParamsNonStreaming) -> None: + completion = get_client().chat.completions.create(**params) + should_print_header = len(completion.choices) > 1 + for choice in completion.choices: + if should_print_header: + sys.stdout.write("===== Chat Completion {} =====\n".format(choice.index)) + + content = choice.message.content if choice.message.content is not None else "None" + sys.stdout.write(content) + + if should_print_header or not content.endswith("\n"): + sys.stdout.write("\n") + + sys.stdout.flush() + + @staticmethod + def _stream_create(params: CompletionCreateParamsStreaming) -> None: + # cast is required for mypy + stream = cast( # pyright: ignore[reportUnnecessaryCast] + Stream[ChatCompletionChunk], get_client().chat.completions.create(**params) + ) + for chunk in stream: + should_print_header = len(chunk.choices) > 1 + for choice in chunk.choices: + if should_print_header: + sys.stdout.write("===== Chat Completion {} =====\n".format(choice.index)) + + content = choice.delta.content or "" + sys.stdout.write(content) + + if should_print_header: + sys.stdout.write("\n") + + sys.stdout.flush() + + sys.stdout.write("\n") diff --git a/lib/python3.12/site-packages/openai/cli/_api/completions.py b/lib/python3.12/site-packages/openai/cli/_api/completions.py new file mode 100644 index 0000000000000000000000000000000000000000..cbdb35bf3a9f1f8db56abf78420a66cdc5140b83 --- /dev/null +++ b/lib/python3.12/site-packages/openai/cli/_api/completions.py @@ -0,0 +1,173 @@ +from __future__ import annotations + +import sys +from typing import TYPE_CHECKING, Optional, cast +from argparse import ArgumentParser +from functools import partial + +from openai.types.completion import Completion + +from .._utils import get_client +from ..._types import NOT_GIVEN, NotGivenOr +from ..._utils import is_given +from .._errors import CLIError +from .._models import BaseModel +from ..._streaming import Stream + +if TYPE_CHECKING: + from argparse import _SubParsersAction + + +def register(subparser: _SubParsersAction[ArgumentParser]) -> None: + sub = subparser.add_parser("completions.create") + + # Required + sub.add_argument( + "-m", + "--model", + help="The model to use", + required=True, + ) + + # Optional + sub.add_argument("-p", "--prompt", help="An optional prompt to complete from") + sub.add_argument("--stream", help="Stream tokens as they're ready.", action="store_true") + sub.add_argument("-M", "--max-tokens", help="The maximum number of tokens to generate", type=int) + sub.add_argument( + "-t", + "--temperature", + help="""What sampling temperature to use. Higher values means the model will take more risks. Try 0.9 for more creative applications, and 0 (argmax sampling) for ones with a well-defined answer. + +Mutually exclusive with `top_p`.""", + type=float, + ) + sub.add_argument( + "-P", + "--top_p", + help="""An alternative to sampling with temperature, called nucleus sampling, where the considers the results of the tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10%% probability mass are considered. + + Mutually exclusive with `temperature`.""", + type=float, + ) + sub.add_argument( + "-n", + "--n", + help="How many sub-completions to generate for each prompt.", + type=int, + ) + sub.add_argument( + "--logprobs", + help="Include the log probabilities on the `logprobs` most likely tokens, as well the chosen tokens. So for example, if `logprobs` is 10, the API will return a list of the 10 most likely tokens. If `logprobs` is 0, only the chosen tokens will have logprobs returned.", + type=int, + ) + sub.add_argument( + "--best_of", + help="Generates `best_of` completions server-side and returns the 'best' (the one with the highest log probability per token). Results cannot be streamed.", + type=int, + ) + sub.add_argument( + "--echo", + help="Echo back the prompt in addition to the completion", + action="store_true", + ) + sub.add_argument( + "--frequency_penalty", + help="Positive values penalize new tokens based on their existing frequency in the text so far, decreasing the model's likelihood to repeat the same line verbatim.", + type=float, + ) + sub.add_argument( + "--presence_penalty", + help="Positive values penalize new tokens based on whether they appear in the text so far, increasing the model's likelihood to talk about new topics.", + type=float, + ) + sub.add_argument("--suffix", help="The suffix that comes after a completion of inserted text.") + sub.add_argument("--stop", help="A stop sequence at which to stop generating tokens.") + sub.add_argument( + "--user", + help="A unique identifier representing your end-user, which can help OpenAI to monitor and detect abuse.", + ) + # TODO: add support for logit_bias + sub.set_defaults(func=CLICompletions.create, args_model=CLICompletionCreateArgs) + + +class CLICompletionCreateArgs(BaseModel): + model: str + stream: bool = False + + prompt: Optional[str] = None + n: NotGivenOr[int] = NOT_GIVEN + stop: NotGivenOr[str] = NOT_GIVEN + user: NotGivenOr[str] = NOT_GIVEN + echo: NotGivenOr[bool] = NOT_GIVEN + suffix: NotGivenOr[str] = NOT_GIVEN + best_of: NotGivenOr[int] = NOT_GIVEN + top_p: NotGivenOr[float] = NOT_GIVEN + logprobs: NotGivenOr[int] = NOT_GIVEN + max_tokens: NotGivenOr[int] = NOT_GIVEN + temperature: NotGivenOr[float] = NOT_GIVEN + presence_penalty: NotGivenOr[float] = NOT_GIVEN + frequency_penalty: NotGivenOr[float] = NOT_GIVEN + + +class CLICompletions: + @staticmethod + def create(args: CLICompletionCreateArgs) -> None: + if is_given(args.n) and args.n > 1 and args.stream: + raise CLIError("Can't stream completions with n>1 with the current CLI") + + make_request = partial( + get_client().completions.create, + n=args.n, + echo=args.echo, + stop=args.stop, + user=args.user, + model=args.model, + top_p=args.top_p, + prompt=args.prompt, + suffix=args.suffix, + best_of=args.best_of, + logprobs=args.logprobs, + max_tokens=args.max_tokens, + temperature=args.temperature, + presence_penalty=args.presence_penalty, + frequency_penalty=args.frequency_penalty, + ) + + if args.stream: + return CLICompletions._stream_create( + # mypy doesn't understand the `partial` function but pyright does + cast(Stream[Completion], make_request(stream=True)) # pyright: ignore[reportUnnecessaryCast] + ) + + return CLICompletions._create(make_request()) + + @staticmethod + def _create(completion: Completion) -> None: + should_print_header = len(completion.choices) > 1 + for choice in completion.choices: + if should_print_header: + sys.stdout.write("===== Completion {} =====\n".format(choice.index)) + + sys.stdout.write(choice.text) + + if should_print_header or not choice.text.endswith("\n"): + sys.stdout.write("\n") + + sys.stdout.flush() + + @staticmethod + def _stream_create(stream: Stream[Completion]) -> None: + for completion in stream: + should_print_header = len(completion.choices) > 1 + for choice in sorted(completion.choices, key=lambda c: c.index): + if should_print_header: + sys.stdout.write("===== Chat Completion {} =====\n".format(choice.index)) + + sys.stdout.write(choice.text) + + if should_print_header: + sys.stdout.write("\n") + + sys.stdout.flush() + + sys.stdout.write("\n") diff --git a/lib/python3.12/site-packages/openai/cli/_api/files.py b/lib/python3.12/site-packages/openai/cli/_api/files.py new file mode 100644 index 0000000000000000000000000000000000000000..5f3631b2840aa35511f9c26d550f1429f21c7155 --- /dev/null +++ b/lib/python3.12/site-packages/openai/cli/_api/files.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, cast +from argparse import ArgumentParser + +from .._utils import get_client, print_model +from .._models import BaseModel +from .._progress import BufferReader + +if TYPE_CHECKING: + from argparse import _SubParsersAction + + +def register(subparser: _SubParsersAction[ArgumentParser]) -> None: + sub = subparser.add_parser("files.create") + + sub.add_argument( + "-f", + "--file", + required=True, + help="File to upload", + ) + sub.add_argument( + "-p", + "--purpose", + help="Why are you uploading this file? (see https://platform.openai.com/docs/api-reference/ for purposes)", + required=True, + ) + sub.set_defaults(func=CLIFile.create, args_model=CLIFileCreateArgs) + + sub = subparser.add_parser("files.retrieve") + sub.add_argument("-i", "--id", required=True, help="The files ID") + sub.set_defaults(func=CLIFile.get, args_model=CLIFileCreateArgs) + + sub = subparser.add_parser("files.delete") + sub.add_argument("-i", "--id", required=True, help="The files ID") + sub.set_defaults(func=CLIFile.delete, args_model=CLIFileCreateArgs) + + sub = subparser.add_parser("files.list") + sub.set_defaults(func=CLIFile.list) + + +class CLIFileIDArgs(BaseModel): + id: str + + +class CLIFileCreateArgs(BaseModel): + file: str + purpose: str + + +class CLIFile: + @staticmethod + def create(args: CLIFileCreateArgs) -> None: + with open(args.file, "rb") as file_reader: + buffer_reader = BufferReader(file_reader.read(), desc="Upload progress") + + file = get_client().files.create( + file=(args.file, buffer_reader), + # casts required because the API is typed for enums + # but we don't want to validate that here for forwards-compat + purpose=cast(Any, args.purpose), + ) + print_model(file) + + @staticmethod + def get(args: CLIFileIDArgs) -> None: + file = get_client().files.retrieve(file_id=args.id) + print_model(file) + + @staticmethod + def delete(args: CLIFileIDArgs) -> None: + file = get_client().files.delete(file_id=args.id) + print_model(file) + + @staticmethod + def list() -> None: + files = get_client().files.list() + for file in files: + print_model(file) diff --git a/lib/python3.12/site-packages/openai/cli/_api/image.py b/lib/python3.12/site-packages/openai/cli/_api/image.py new file mode 100644 index 0000000000000000000000000000000000000000..3e2a0a90f1b41a1e0a39e401932e0f5ccfc3e3b0 --- /dev/null +++ b/lib/python3.12/site-packages/openai/cli/_api/image.py @@ -0,0 +1,139 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, cast +from argparse import ArgumentParser + +from .._utils import get_client, print_model +from ..._types import NOT_GIVEN, NotGiven, NotGivenOr +from .._models import BaseModel +from .._progress import BufferReader + +if TYPE_CHECKING: + from argparse import _SubParsersAction + + +def register(subparser: _SubParsersAction[ArgumentParser]) -> None: + sub = subparser.add_parser("images.generate") + sub.add_argument("-m", "--model", type=str) + sub.add_argument("-p", "--prompt", type=str, required=True) + sub.add_argument("-n", "--num-images", type=int, default=1) + sub.add_argument("-s", "--size", type=str, default="1024x1024", help="Size of the output image") + sub.add_argument("--response-format", type=str, default="url") + sub.set_defaults(func=CLIImage.create, args_model=CLIImageCreateArgs) + + sub = subparser.add_parser("images.edit") + sub.add_argument("-m", "--model", type=str) + sub.add_argument("-p", "--prompt", type=str, required=True) + sub.add_argument("-n", "--num-images", type=int, default=1) + sub.add_argument( + "-I", + "--image", + type=str, + required=True, + help="Image to modify. Should be a local path and a PNG encoded image.", + ) + sub.add_argument("-s", "--size", type=str, default="1024x1024", help="Size of the output image") + sub.add_argument("--response-format", type=str, default="url") + sub.add_argument( + "-M", + "--mask", + type=str, + required=False, + help="Path to a mask image. It should be the same size as the image you're editing and a RGBA PNG image. The Alpha channel acts as the mask.", + ) + sub.set_defaults(func=CLIImage.edit, args_model=CLIImageEditArgs) + + sub = subparser.add_parser("images.create_variation") + sub.add_argument("-m", "--model", type=str) + sub.add_argument("-n", "--num-images", type=int, default=1) + sub.add_argument( + "-I", + "--image", + type=str, + required=True, + help="Image to modify. Should be a local path and a PNG encoded image.", + ) + sub.add_argument("-s", "--size", type=str, default="1024x1024", help="Size of the output image") + sub.add_argument("--response-format", type=str, default="url") + sub.set_defaults(func=CLIImage.create_variation, args_model=CLIImageCreateVariationArgs) + + +class CLIImageCreateArgs(BaseModel): + prompt: str + num_images: int + size: str + response_format: str + model: NotGivenOr[str] = NOT_GIVEN + + +class CLIImageCreateVariationArgs(BaseModel): + image: str + num_images: int + size: str + response_format: str + model: NotGivenOr[str] = NOT_GIVEN + + +class CLIImageEditArgs(BaseModel): + image: str + num_images: int + size: str + response_format: str + prompt: str + mask: NotGivenOr[str] = NOT_GIVEN + model: NotGivenOr[str] = NOT_GIVEN + + +class CLIImage: + @staticmethod + def create(args: CLIImageCreateArgs) -> None: + image = get_client().images.generate( + model=args.model, + prompt=args.prompt, + n=args.num_images, + # casts required because the API is typed for enums + # but we don't want to validate that here for forwards-compat + size=cast(Any, args.size), + response_format=cast(Any, args.response_format), + ) + print_model(image) + + @staticmethod + def create_variation(args: CLIImageCreateVariationArgs) -> None: + with open(args.image, "rb") as file_reader: + buffer_reader = BufferReader(file_reader.read(), desc="Upload progress") + + image = get_client().images.create_variation( + model=args.model, + image=("image", buffer_reader), + n=args.num_images, + # casts required because the API is typed for enums + # but we don't want to validate that here for forwards-compat + size=cast(Any, args.size), + response_format=cast(Any, args.response_format), + ) + print_model(image) + + @staticmethod + def edit(args: CLIImageEditArgs) -> None: + with open(args.image, "rb") as file_reader: + buffer_reader = BufferReader(file_reader.read(), desc="Image upload progress") + + if isinstance(args.mask, NotGiven): + mask: NotGivenOr[BufferReader] = NOT_GIVEN + else: + with open(args.mask, "rb") as file_reader: + mask = BufferReader(file_reader.read(), desc="Mask progress") + + image = get_client().images.edit( + model=args.model, + prompt=args.prompt, + image=("image", buffer_reader), + n=args.num_images, + mask=("mask", mask) if not isinstance(mask, NotGiven) else mask, + # casts required because the API is typed for enums + # but we don't want to validate that here for forwards-compat + size=cast(Any, args.size), + response_format=cast(Any, args.response_format), + ) + print_model(image) diff --git a/lib/python3.12/site-packages/openai/cli/_api/models.py b/lib/python3.12/site-packages/openai/cli/_api/models.py new file mode 100644 index 0000000000000000000000000000000000000000..017218fa6e3ace8d42ad745d4649bfd341535460 --- /dev/null +++ b/lib/python3.12/site-packages/openai/cli/_api/models.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING +from argparse import ArgumentParser + +from .._utils import get_client, print_model +from .._models import BaseModel + +if TYPE_CHECKING: + from argparse import _SubParsersAction + + +def register(subparser: _SubParsersAction[ArgumentParser]) -> None: + sub = subparser.add_parser("models.list") + sub.set_defaults(func=CLIModels.list) + + sub = subparser.add_parser("models.retrieve") + sub.add_argument("-i", "--id", required=True, help="The model ID") + sub.set_defaults(func=CLIModels.get, args_model=CLIModelIDArgs) + + sub = subparser.add_parser("models.delete") + sub.add_argument("-i", "--id", required=True, help="The model ID") + sub.set_defaults(func=CLIModels.delete, args_model=CLIModelIDArgs) + + +class CLIModelIDArgs(BaseModel): + id: str + + +class CLIModels: + @staticmethod + def get(args: CLIModelIDArgs) -> None: + model = get_client().models.retrieve(model=args.id) + print_model(model) + + @staticmethod + def delete(args: CLIModelIDArgs) -> None: + model = get_client().models.delete(model=args.id) + print_model(model) + + @staticmethod + def list() -> None: + models = get_client().models.list() + for model in models: + print_model(model) diff --git a/lib/python3.12/site-packages/openai/cli/_cli.py b/lib/python3.12/site-packages/openai/cli/_cli.py new file mode 100644 index 0000000000000000000000000000000000000000..fd165f48abf98611b30dd319a1e5052a96140bd8 --- /dev/null +++ b/lib/python3.12/site-packages/openai/cli/_cli.py @@ -0,0 +1,233 @@ +from __future__ import annotations + +import sys +import logging +import argparse +from typing import Any, List, Type, Optional +from typing_extensions import ClassVar + +import httpx +import pydantic + +import openai + +from . import _tools +from .. import _ApiType, __version__ +from ._api import register_commands +from ._utils import can_use_http2 +from ._errors import CLIError, display_error +from .._compat import PYDANTIC_V2, ConfigDict, model_parse +from .._models import BaseModel +from .._exceptions import APIError + +logger = logging.getLogger() +formatter = logging.Formatter("[%(asctime)s] %(message)s") +handler = logging.StreamHandler(sys.stderr) +handler.setFormatter(formatter) +logger.addHandler(handler) + + +class Arguments(BaseModel): + if PYDANTIC_V2: + model_config: ClassVar[ConfigDict] = ConfigDict( + extra="ignore", + ) + else: + + class Config(pydantic.BaseConfig): # type: ignore + extra: Any = pydantic.Extra.ignore # type: ignore + + verbosity: int + version: Optional[str] = None + + api_key: Optional[str] + api_base: Optional[str] + organization: Optional[str] + proxy: Optional[List[str]] + api_type: Optional[_ApiType] = None + api_version: Optional[str] = None + + # azure + azure_endpoint: Optional[str] = None + azure_ad_token: Optional[str] = None + + # internal, set by subparsers to parse their specific args + args_model: Optional[Type[BaseModel]] = None + + # internal, used so that subparsers can forward unknown arguments + unknown_args: List[str] = [] + allow_unknown_args: bool = False + + +def _build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=None, prog="openai") + parser.add_argument( + "-v", + "--verbose", + action="count", + dest="verbosity", + default=0, + help="Set verbosity.", + ) + parser.add_argument("-b", "--api-base", help="What API base url to use.") + parser.add_argument("-k", "--api-key", help="What API key to use.") + parser.add_argument("-p", "--proxy", nargs="+", help="What proxy to use.") + parser.add_argument( + "-o", + "--organization", + help="Which organization to run as (will use your default organization if not specified)", + ) + parser.add_argument( + "-t", + "--api-type", + type=str, + choices=("openai", "azure"), + help="The backend API to call, must be `openai` or `azure`", + ) + parser.add_argument( + "--api-version", + help="The Azure API version, e.g. 'https://learn.microsoft.com/en-us/azure/ai-services/openai/reference#rest-api-versioning'", + ) + + # azure + parser.add_argument( + "--azure-endpoint", + help="The Azure endpoint, e.g. 'https://endpoint.openai.azure.com'", + ) + parser.add_argument( + "--azure-ad-token", + help="A token from Azure Active Directory, https://www.microsoft.com/en-us/security/business/identity-access/microsoft-entra-id", + ) + + # prints the package version + parser.add_argument( + "-V", + "--version", + action="version", + version="%(prog)s " + __version__, + ) + + def help() -> None: + parser.print_help() + + parser.set_defaults(func=help) + + subparsers = parser.add_subparsers() + sub_api = subparsers.add_parser("api", help="Direct API calls") + + register_commands(sub_api) + + sub_tools = subparsers.add_parser("tools", help="Client side tools for convenience") + _tools.register_commands(sub_tools, subparsers) + + return parser + + +def main() -> int: + try: + _main() + except (APIError, CLIError, pydantic.ValidationError) as err: + display_error(err) + return 1 + except KeyboardInterrupt: + sys.stderr.write("\n") + return 1 + return 0 + + +def _parse_args(parser: argparse.ArgumentParser) -> tuple[argparse.Namespace, Arguments, list[str]]: + # argparse by default will strip out the `--` but we want to keep it for unknown arguments + if "--" in sys.argv: + idx = sys.argv.index("--") + known_args = sys.argv[1:idx] + unknown_args = sys.argv[idx:] + else: + known_args = sys.argv[1:] + unknown_args = [] + + parsed, remaining_unknown = parser.parse_known_args(known_args) + + # append any remaining unknown arguments from the initial parsing + remaining_unknown.extend(unknown_args) + + args = model_parse(Arguments, vars(parsed)) + if not args.allow_unknown_args: + # we have to parse twice to ensure any unknown arguments + # result in an error if that behaviour is desired + parser.parse_args() + + return parsed, args, remaining_unknown + + +def _main() -> None: + parser = _build_parser() + parsed, args, unknown = _parse_args(parser) + + if args.verbosity != 0: + sys.stderr.write("Warning: --verbosity isn't supported yet\n") + + proxies: dict[str, httpx.BaseTransport] = {} + if args.proxy is not None: + for proxy in args.proxy: + key = "https://" if proxy.startswith("https") else "http://" + if key in proxies: + raise CLIError(f"Multiple {key} proxies given - only the last one would be used") + + proxies[key] = httpx.HTTPTransport(proxy=httpx.Proxy(httpx.URL(proxy))) + + http_client = httpx.Client( + mounts=proxies or None, + http2=can_use_http2(), + ) + openai.http_client = http_client + + if args.organization: + openai.organization = args.organization + + if args.api_key: + openai.api_key = args.api_key + + if args.api_base: + openai.base_url = args.api_base + + # azure + if args.api_type is not None: + openai.api_type = args.api_type + + if args.azure_endpoint is not None: + openai.azure_endpoint = args.azure_endpoint + + if args.api_version is not None: + openai.api_version = args.api_version + + if args.azure_ad_token is not None: + openai.azure_ad_token = args.azure_ad_token + + try: + if args.args_model: + parsed.func( + model_parse( + args.args_model, + { + **{ + # we omit None values so that they can be defaulted to `NotGiven` + # and we'll strip it from the API request + key: value + for key, value in vars(parsed).items() + if value is not None + }, + "unknown_args": unknown, + }, + ) + ) + else: + parsed.func() + finally: + try: + http_client.close() + except Exception: + pass + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/lib/python3.12/site-packages/openai/cli/_errors.py b/lib/python3.12/site-packages/openai/cli/_errors.py new file mode 100644 index 0000000000000000000000000000000000000000..7d0292dab267893a533eb3a5d8a634aa23c53b5f --- /dev/null +++ b/lib/python3.12/site-packages/openai/cli/_errors.py @@ -0,0 +1,21 @@ +from __future__ import annotations + +import sys + +import pydantic + +from ._utils import Colors, organization_info +from .._exceptions import APIError, OpenAIError + + +class CLIError(OpenAIError): ... + + +class SilentCLIError(CLIError): ... + + +def display_error(err: CLIError | APIError | pydantic.ValidationError) -> None: + if isinstance(err, SilentCLIError): + return + + sys.stderr.write("{}{}Error:{} {}\n".format(organization_info(), Colors.FAIL, Colors.ENDC, err)) diff --git a/lib/python3.12/site-packages/openai/cli/_models.py b/lib/python3.12/site-packages/openai/cli/_models.py new file mode 100644 index 0000000000000000000000000000000000000000..5583db26091d6514eab01b83ad98e8ab4c55286e --- /dev/null +++ b/lib/python3.12/site-packages/openai/cli/_models.py @@ -0,0 +1,17 @@ +from typing import Any +from typing_extensions import ClassVar + +import pydantic + +from .. import _models +from .._compat import PYDANTIC_V2, ConfigDict + + +class BaseModel(_models.BaseModel): + if PYDANTIC_V2: + model_config: ClassVar[ConfigDict] = ConfigDict(extra="ignore", arbitrary_types_allowed=True) + else: + + class Config(pydantic.BaseConfig): # type: ignore + extra: Any = pydantic.Extra.ignore # type: ignore + arbitrary_types_allowed: bool = True diff --git a/lib/python3.12/site-packages/openai/cli/_progress.py b/lib/python3.12/site-packages/openai/cli/_progress.py new file mode 100644 index 0000000000000000000000000000000000000000..8a7f2525deb80aa9f24b22537cac725cefad4f4d --- /dev/null +++ b/lib/python3.12/site-packages/openai/cli/_progress.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +import io +from typing import Callable +from typing_extensions import override + + +class CancelledError(Exception): + def __init__(self, msg: str) -> None: + self.msg = msg + super().__init__(msg) + + @override + def __str__(self) -> str: + return self.msg + + __repr__ = __str__ + + +class BufferReader(io.BytesIO): + def __init__(self, buf: bytes = b"", desc: str | None = None) -> None: + super().__init__(buf) + self._len = len(buf) + self._progress = 0 + self._callback = progress(len(buf), desc=desc) + + def __len__(self) -> int: + return self._len + + @override + def read(self, n: int | None = -1) -> bytes: + chunk = io.BytesIO.read(self, n) + self._progress += len(chunk) + + try: + self._callback(self._progress) + except Exception as e: # catches exception from the callback + raise CancelledError("The upload was cancelled: {}".format(e)) from e + + return chunk + + +def progress(total: float, desc: str | None) -> Callable[[float], None]: + import tqdm + + meter = tqdm.tqdm(total=total, unit_scale=True, desc=desc) + + def incr(progress: float) -> None: + meter.n = progress + if progress == total: + meter.close() + else: + meter.refresh() + + return incr + + +def MB(i: int) -> int: + return int(i // 1024**2) diff --git a/lib/python3.12/site-packages/openai/cli/_tools/__init__.py b/lib/python3.12/site-packages/openai/cli/_tools/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..56a0260a6dc333995ff975158452eca83fc66012 --- /dev/null +++ b/lib/python3.12/site-packages/openai/cli/_tools/__init__.py @@ -0,0 +1 @@ +from ._main import register_commands as register_commands diff --git a/lib/python3.12/site-packages/openai/cli/_tools/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/openai/cli/_tools/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..81f44315fc3827389725ac24f8a5e9c725bfe92d Binary files /dev/null and b/lib/python3.12/site-packages/openai/cli/_tools/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/cli/_tools/__pycache__/_main.cpython-312.pyc b/lib/python3.12/site-packages/openai/cli/_tools/__pycache__/_main.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d22098b3c0ac276d2488d416b3833b95e5ddf875 Binary files /dev/null and b/lib/python3.12/site-packages/openai/cli/_tools/__pycache__/_main.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/cli/_tools/__pycache__/fine_tunes.cpython-312.pyc b/lib/python3.12/site-packages/openai/cli/_tools/__pycache__/fine_tunes.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cf684b597eec59658da0eb0f7683aa3c8c1df941 Binary files /dev/null and b/lib/python3.12/site-packages/openai/cli/_tools/__pycache__/fine_tunes.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/cli/_tools/__pycache__/migrate.cpython-312.pyc b/lib/python3.12/site-packages/openai/cli/_tools/__pycache__/migrate.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0b7145b81e1d40451ba81f4ecb86ee654ea6ea5d Binary files /dev/null and b/lib/python3.12/site-packages/openai/cli/_tools/__pycache__/migrate.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/cli/_tools/_main.py b/lib/python3.12/site-packages/openai/cli/_tools/_main.py new file mode 100644 index 0000000000000000000000000000000000000000..bd6cda408fdb92f0355476756551b29295ab2ade --- /dev/null +++ b/lib/python3.12/site-packages/openai/cli/_tools/_main.py @@ -0,0 +1,17 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING +from argparse import ArgumentParser + +from . import migrate, fine_tunes + +if TYPE_CHECKING: + from argparse import _SubParsersAction + + +def register_commands(parser: ArgumentParser, subparser: _SubParsersAction[ArgumentParser]) -> None: + migrate.register(subparser) + + namespaced = parser.add_subparsers(title="Tools", help="Convenience client side tools") + + fine_tunes.register(namespaced) diff --git a/lib/python3.12/site-packages/openai/cli/_tools/fine_tunes.py b/lib/python3.12/site-packages/openai/cli/_tools/fine_tunes.py new file mode 100644 index 0000000000000000000000000000000000000000..2128b8895224b1b16190bbe7b5a4bd98662e7f82 --- /dev/null +++ b/lib/python3.12/site-packages/openai/cli/_tools/fine_tunes.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +import sys +from typing import TYPE_CHECKING +from argparse import ArgumentParser + +from .._models import BaseModel +from ...lib._validators import ( + get_validators, + write_out_file, + read_any_format, + apply_validators, + apply_necessary_remediation, +) + +if TYPE_CHECKING: + from argparse import _SubParsersAction + + +def register(subparser: _SubParsersAction[ArgumentParser]) -> None: + sub = subparser.add_parser("fine_tunes.prepare_data") + sub.add_argument( + "-f", + "--file", + required=True, + help="JSONL, JSON, CSV, TSV, TXT or XLSX file containing prompt-completion examples to be analyzed." + "This should be the local file path.", + ) + sub.add_argument( + "-q", + "--quiet", + required=False, + action="store_true", + help="Auto accepts all suggestions, without asking for user input. To be used within scripts.", + ) + sub.set_defaults(func=prepare_data, args_model=PrepareDataArgs) + + +class PrepareDataArgs(BaseModel): + file: str + + quiet: bool + + +def prepare_data(args: PrepareDataArgs) -> None: + sys.stdout.write("Analyzing...\n") + fname = args.file + auto_accept = args.quiet + df, remediation = read_any_format(fname) + apply_necessary_remediation(None, remediation) + + validators = get_validators() + + assert df is not None + + apply_validators( + df, + fname, + remediation, + validators, + auto_accept, + write_out_file_func=write_out_file, + ) diff --git a/lib/python3.12/site-packages/openai/cli/_tools/migrate.py b/lib/python3.12/site-packages/openai/cli/_tools/migrate.py new file mode 100644 index 0000000000000000000000000000000000000000..841b777528069fa17181b1009e6a62e9d5b939f9 --- /dev/null +++ b/lib/python3.12/site-packages/openai/cli/_tools/migrate.py @@ -0,0 +1,164 @@ +from __future__ import annotations + +import os +import sys +import shutil +import tarfile +import platform +import subprocess +from typing import TYPE_CHECKING, List +from pathlib import Path +from argparse import ArgumentParser + +import httpx + +from .._errors import CLIError, SilentCLIError +from .._models import BaseModel + +if TYPE_CHECKING: + from argparse import _SubParsersAction + + +def register(subparser: _SubParsersAction[ArgumentParser]) -> None: + sub = subparser.add_parser("migrate") + sub.set_defaults(func=migrate, args_model=MigrateArgs, allow_unknown_args=True) + + sub = subparser.add_parser("grit") + sub.set_defaults(func=grit, args_model=GritArgs, allow_unknown_args=True) + + +class GritArgs(BaseModel): + # internal + unknown_args: List[str] = [] + + +def grit(args: GritArgs) -> None: + grit_path = install() + + try: + subprocess.check_call([grit_path, *args.unknown_args]) + except subprocess.CalledProcessError: + # stdout and stderr are forwarded by subprocess so an error will already + # have been displayed + raise SilentCLIError() from None + + +class MigrateArgs(BaseModel): + # internal + unknown_args: List[str] = [] + + +def migrate(args: MigrateArgs) -> None: + grit_path = install() + + try: + subprocess.check_call([grit_path, "apply", "openai", *args.unknown_args]) + except subprocess.CalledProcessError: + # stdout and stderr are forwarded by subprocess so an error will already + # have been displayed + raise SilentCLIError() from None + + +# handles downloading the Grit CLI until they provide their own PyPi package + +KEYGEN_ACCOUNT = "custodian-dev" + + +def _cache_dir() -> Path: + xdg = os.environ.get("XDG_CACHE_HOME") + if xdg is not None: + return Path(xdg) + + return Path.home() / ".cache" + + +def _debug(message: str) -> None: + if not os.environ.get("DEBUG"): + return + + sys.stdout.write(f"[DEBUG]: {message}\n") + + +def install() -> Path: + """Installs the Grit CLI and returns the location of the binary""" + if sys.platform == "win32": + raise CLIError("Windows is not supported yet in the migration CLI") + + _debug("Using Grit installer from GitHub") + + platform = "apple-darwin" if sys.platform == "darwin" else "unknown-linux-gnu" + + dir_name = _cache_dir() / "openai-python" + install_dir = dir_name / ".install" + target_dir = install_dir / "bin" + + target_path = target_dir / "grit" + temp_file = target_dir / "grit.tmp" + + if target_path.exists(): + _debug(f"{target_path} already exists") + sys.stdout.flush() + return target_path + + _debug(f"Using Grit CLI path: {target_path}") + + target_dir.mkdir(parents=True, exist_ok=True) + + if temp_file.exists(): + temp_file.unlink() + + arch = _get_arch() + _debug(f"Using architecture {arch}") + + file_name = f"grit-{arch}-{platform}" + download_url = f"https://github.com/getgrit/gritql/releases/latest/download/{file_name}.tar.gz" + + sys.stdout.write(f"Downloading Grit CLI from {download_url}\n") + with httpx.Client() as client: + download_response = client.get(download_url, follow_redirects=True) + if download_response.status_code != 200: + raise CLIError(f"Failed to download Grit CLI from {download_url}") + with open(temp_file, "wb") as file: + for chunk in download_response.iter_bytes(): + file.write(chunk) + + unpacked_dir = target_dir / "cli-bin" + unpacked_dir.mkdir(parents=True, exist_ok=True) + + with tarfile.open(temp_file, "r:gz") as archive: + if sys.version_info >= (3, 12): + archive.extractall(unpacked_dir, filter="data") + else: + archive.extractall(unpacked_dir) + + _move_files_recursively(unpacked_dir, target_dir) + + shutil.rmtree(unpacked_dir) + os.remove(temp_file) + os.chmod(target_path, 0o755) + + sys.stdout.flush() + + return target_path + + +def _move_files_recursively(source_dir: Path, target_dir: Path) -> None: + for item in source_dir.iterdir(): + if item.is_file(): + item.rename(target_dir / item.name) + elif item.is_dir(): + _move_files_recursively(item, target_dir) + + +def _get_arch() -> str: + architecture = platform.machine().lower() + + # Map the architecture names to Grit equivalents + arch_map = { + "x86_64": "x86_64", + "amd64": "x86_64", + "armv7l": "aarch64", + "arm64": "aarch64", + } + + return arch_map.get(architecture, architecture) diff --git a/lib/python3.12/site-packages/openai/cli/_utils.py b/lib/python3.12/site-packages/openai/cli/_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..673eed613cc7d77a65674aef7221fea62c8ccc6b --- /dev/null +++ b/lib/python3.12/site-packages/openai/cli/_utils.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +import sys + +import openai + +from .. import OpenAI, _load_client +from .._compat import model_json +from .._models import BaseModel + + +class Colors: + HEADER = "\033[95m" + OKBLUE = "\033[94m" + OKGREEN = "\033[92m" + WARNING = "\033[93m" + FAIL = "\033[91m" + ENDC = "\033[0m" + BOLD = "\033[1m" + UNDERLINE = "\033[4m" + + +def get_client() -> OpenAI: + return _load_client() + + +def organization_info() -> str: + organization = openai.organization + if organization is not None: + return "[organization={}] ".format(organization) + + return "" + + +def print_model(model: BaseModel) -> None: + sys.stdout.write(model_json(model, indent=2) + "\n") + + +def can_use_http2() -> bool: + try: + import h2 # type: ignore # noqa + except ImportError: + return False + + return True diff --git a/lib/python3.12/site-packages/openai/helpers/__init__.py b/lib/python3.12/site-packages/openai/helpers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ab3044da59c54bce5f0742cdde22fb85dbbbcbf5 --- /dev/null +++ b/lib/python3.12/site-packages/openai/helpers/__init__.py @@ -0,0 +1,4 @@ +from .microphone import Microphone +from .local_audio_player import LocalAudioPlayer + +__all__ = ["Microphone", "LocalAudioPlayer"] diff --git a/lib/python3.12/site-packages/openai/helpers/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/openai/helpers/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2b1cb9d99a833dfc4c6095857b95d7eca17a7d20 Binary files /dev/null and b/lib/python3.12/site-packages/openai/helpers/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/helpers/__pycache__/local_audio_player.cpython-312.pyc b/lib/python3.12/site-packages/openai/helpers/__pycache__/local_audio_player.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4fbfb476697e295684035d7bb53ebaf0074a3a47 Binary files /dev/null and b/lib/python3.12/site-packages/openai/helpers/__pycache__/local_audio_player.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/helpers/__pycache__/microphone.cpython-312.pyc b/lib/python3.12/site-packages/openai/helpers/__pycache__/microphone.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e64ba9c1d83bf5cf4aefcbcd4093c0716b2f367f Binary files /dev/null and b/lib/python3.12/site-packages/openai/helpers/__pycache__/microphone.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/helpers/local_audio_player.py b/lib/python3.12/site-packages/openai/helpers/local_audio_player.py new file mode 100644 index 0000000000000000000000000000000000000000..eed68aa21d4b93c3e30207447d9d19a232e7a7e7 --- /dev/null +++ b/lib/python3.12/site-packages/openai/helpers/local_audio_player.py @@ -0,0 +1,165 @@ +# mypy: ignore-errors +from __future__ import annotations + +import queue +import asyncio +from typing import Any, Union, Callable, AsyncGenerator, cast +from typing_extensions import TYPE_CHECKING + +from .. import _legacy_response +from .._extras import numpy as np, sounddevice as sd +from .._response import StreamedBinaryAPIResponse, AsyncStreamedBinaryAPIResponse + +if TYPE_CHECKING: + import numpy.typing as npt + +SAMPLE_RATE = 24000 + + +class LocalAudioPlayer: + def __init__( + self, + should_stop: Union[Callable[[], bool], None] = None, + ): + self.channels = 1 + self.dtype = np.float32 + self.should_stop = should_stop + + async def _tts_response_to_buffer( + self, + response: Union[ + _legacy_response.HttpxBinaryResponseContent, + AsyncStreamedBinaryAPIResponse, + StreamedBinaryAPIResponse, + ], + ) -> npt.NDArray[np.float32]: + chunks: list[bytes] = [] + if isinstance(response, _legacy_response.HttpxBinaryResponseContent) or isinstance( + response, StreamedBinaryAPIResponse + ): + for chunk in response.iter_bytes(chunk_size=1024): + if chunk: + chunks.append(chunk) + else: + async for chunk in response.iter_bytes(chunk_size=1024): + if chunk: + chunks.append(chunk) + + audio_bytes = b"".join(chunks) + audio_np = np.frombuffer(audio_bytes, dtype=np.int16).astype(np.float32) / 32767.0 + audio_np = audio_np.reshape(-1, 1) + return audio_np + + async def play( + self, + input: Union[ + npt.NDArray[np.int16], + npt.NDArray[np.float32], + _legacy_response.HttpxBinaryResponseContent, + AsyncStreamedBinaryAPIResponse, + StreamedBinaryAPIResponse, + ], + ) -> None: + audio_content: npt.NDArray[np.float32] + if isinstance(input, np.ndarray): + if input.dtype == np.int16 and self.dtype == np.float32: + audio_content = (input.astype(np.float32) / 32767.0).reshape(-1, self.channels) + elif input.dtype == np.float32: + audio_content = cast('npt.NDArray[np.float32]', input) + else: + raise ValueError(f"Unsupported dtype: {input.dtype}") + else: + audio_content = await self._tts_response_to_buffer(input) + + loop = asyncio.get_event_loop() + event = asyncio.Event() + idx = 0 + + def callback( + outdata: npt.NDArray[np.float32], + frame_count: int, + _time_info: Any, + _status: Any, + ): + nonlocal idx + + remainder = len(audio_content) - idx + if remainder == 0 or (callable(self.should_stop) and self.should_stop()): + loop.call_soon_threadsafe(event.set) + raise sd.CallbackStop + valid_frames = frame_count if remainder >= frame_count else remainder + outdata[:valid_frames] = audio_content[idx : idx + valid_frames] + outdata[valid_frames:] = 0 + idx += valid_frames + + stream = sd.OutputStream( + samplerate=SAMPLE_RATE, + callback=callback, + dtype=audio_content.dtype, + channels=audio_content.shape[1], + ) + with stream: + await event.wait() + + async def play_stream( + self, + buffer_stream: AsyncGenerator[Union[npt.NDArray[np.float32], npt.NDArray[np.int16], None], None], + ) -> None: + loop = asyncio.get_event_loop() + event = asyncio.Event() + buffer_queue: queue.Queue[Union[npt.NDArray[np.float32], npt.NDArray[np.int16], None]] = queue.Queue(maxsize=50) + + async def buffer_producer(): + async for buffer in buffer_stream: + if buffer is None: + break + await loop.run_in_executor(None, buffer_queue.put, buffer) + await loop.run_in_executor(None, buffer_queue.put, None) # Signal completion + + def callback( + outdata: npt.NDArray[np.float32], + frame_count: int, + _time_info: Any, + _status: Any, + ): + nonlocal current_buffer, buffer_pos + + frames_written = 0 + while frames_written < frame_count: + if current_buffer is None or buffer_pos >= len(current_buffer): + try: + current_buffer = buffer_queue.get(timeout=0.1) + if current_buffer is None: + loop.call_soon_threadsafe(event.set) + raise sd.CallbackStop + buffer_pos = 0 + + if current_buffer.dtype == np.int16 and self.dtype == np.float32: + current_buffer = (current_buffer.astype(np.float32) / 32767.0).reshape(-1, self.channels) + + except queue.Empty: + outdata[frames_written:] = 0 + return + + remaining_frames = len(current_buffer) - buffer_pos + frames_to_write = min(frame_count - frames_written, remaining_frames) + outdata[frames_written : frames_written + frames_to_write] = current_buffer[ + buffer_pos : buffer_pos + frames_to_write + ] + buffer_pos += frames_to_write + frames_written += frames_to_write + + current_buffer = None + buffer_pos = 0 + + producer_task = asyncio.create_task(buffer_producer()) + + with sd.OutputStream( + samplerate=SAMPLE_RATE, + channels=self.channels, + dtype=self.dtype, + callback=callback, + ): + await event.wait() + + await producer_task diff --git a/lib/python3.12/site-packages/openai/helpers/microphone.py b/lib/python3.12/site-packages/openai/helpers/microphone.py new file mode 100644 index 0000000000000000000000000000000000000000..62a6d8d8a911ddb5bcc7af02a43450cabc76ff4d --- /dev/null +++ b/lib/python3.12/site-packages/openai/helpers/microphone.py @@ -0,0 +1,100 @@ +# mypy: ignore-errors +from __future__ import annotations + +import io +import time +import wave +import asyncio +from typing import Any, Type, Union, Generic, TypeVar, Callable, overload +from typing_extensions import TYPE_CHECKING, Literal + +from .._types import FileTypes, FileContent +from .._extras import numpy as np, sounddevice as sd + +if TYPE_CHECKING: + import numpy.typing as npt + +SAMPLE_RATE = 24000 + +DType = TypeVar("DType", bound=np.generic) + + +class Microphone(Generic[DType]): + def __init__( + self, + channels: int = 1, + dtype: Type[DType] = np.int16, + should_record: Union[Callable[[], bool], None] = None, + timeout: Union[float, None] = None, + ): + self.channels = channels + self.dtype = dtype + self.should_record = should_record + self.buffer_chunks = [] + self.timeout = timeout + self.has_record_function = callable(should_record) + + def _ndarray_to_wav(self, audio_data: npt.NDArray[DType]) -> FileTypes: + buffer: FileContent = io.BytesIO() + with wave.open(buffer, "w") as wav_file: + wav_file.setnchannels(self.channels) + wav_file.setsampwidth(np.dtype(self.dtype).itemsize) + wav_file.setframerate(SAMPLE_RATE) + wav_file.writeframes(audio_data.tobytes()) + buffer.seek(0) + return ("audio.wav", buffer, "audio/wav") + + @overload + async def record(self, return_ndarray: Literal[True]) -> npt.NDArray[DType]: ... + + @overload + async def record(self, return_ndarray: Literal[False]) -> FileTypes: ... + + @overload + async def record(self, return_ndarray: None = ...) -> FileTypes: ... + + async def record(self, return_ndarray: Union[bool, None] = False) -> Union[npt.NDArray[DType], FileTypes]: + loop = asyncio.get_event_loop() + event = asyncio.Event() + self.buffer_chunks: list[npt.NDArray[DType]] = [] + start_time = time.perf_counter() + + def callback( + indata: npt.NDArray[DType], + _frame_count: int, + _time_info: Any, + _status: Any, + ): + execution_time = time.perf_counter() - start_time + reached_recording_timeout = execution_time > self.timeout if self.timeout is not None else False + if reached_recording_timeout: + loop.call_soon_threadsafe(event.set) + raise sd.CallbackStop + + should_be_recording = self.should_record() if callable(self.should_record) else True + if not should_be_recording: + loop.call_soon_threadsafe(event.set) + raise sd.CallbackStop + + self.buffer_chunks.append(indata.copy()) + + stream = sd.InputStream( + callback=callback, + dtype=self.dtype, + samplerate=SAMPLE_RATE, + channels=self.channels, + ) + with stream: + await event.wait() + + # Concatenate all chunks into a single buffer, handle empty case + concatenated_chunks: npt.NDArray[DType] = ( + np.concatenate(self.buffer_chunks, axis=0) + if len(self.buffer_chunks) > 0 + else np.array([], dtype=self.dtype) + ) + + if return_ndarray: + return concatenated_chunks + else: + return self._ndarray_to_wav(concatenated_chunks) diff --git a/lib/python3.12/site-packages/openai/resources/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/openai/resources/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f4fa707d8c7c411b8bd2bd99fc98c38c5c85873d Binary files /dev/null and b/lib/python3.12/site-packages/openai/resources/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/resources/__pycache__/batches.cpython-312.pyc b/lib/python3.12/site-packages/openai/resources/__pycache__/batches.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5370207071eec5bb39c19fdb001bba5f6d9fd2a8 Binary files /dev/null and b/lib/python3.12/site-packages/openai/resources/__pycache__/batches.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/resources/__pycache__/completions.cpython-312.pyc b/lib/python3.12/site-packages/openai/resources/__pycache__/completions.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..83f1567053e08d1bdc3607165c622d03aba4eefd Binary files /dev/null and b/lib/python3.12/site-packages/openai/resources/__pycache__/completions.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/resources/__pycache__/embeddings.cpython-312.pyc b/lib/python3.12/site-packages/openai/resources/__pycache__/embeddings.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fdf13cba47b789f326ae7767161df232a9ad2e20 Binary files /dev/null and b/lib/python3.12/site-packages/openai/resources/__pycache__/embeddings.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/resources/__pycache__/files.cpython-312.pyc b/lib/python3.12/site-packages/openai/resources/__pycache__/files.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..919c46db12443b1a72ce8c3ad6a3934e18c66bfa Binary files /dev/null and b/lib/python3.12/site-packages/openai/resources/__pycache__/files.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/resources/__pycache__/images.cpython-312.pyc b/lib/python3.12/site-packages/openai/resources/__pycache__/images.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7ed54de650202fee88aaa8ddcaec77e491d19e59 Binary files /dev/null and b/lib/python3.12/site-packages/openai/resources/__pycache__/images.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/resources/__pycache__/models.cpython-312.pyc b/lib/python3.12/site-packages/openai/resources/__pycache__/models.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4aa54c9d9fed4c2ee2859f0206ce4af3e9bc4d9c Binary files /dev/null and b/lib/python3.12/site-packages/openai/resources/__pycache__/models.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/resources/__pycache__/moderations.cpython-312.pyc b/lib/python3.12/site-packages/openai/resources/__pycache__/moderations.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1676dc0bc97befa6e88074560607b17a8dde2e34 Binary files /dev/null and b/lib/python3.12/site-packages/openai/resources/__pycache__/moderations.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/resources/chat/__init__.py b/lib/python3.12/site-packages/openai/resources/chat/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..52dfdceacc4d3e7b88e8d7cf016c0670cd2fe977 --- /dev/null +++ b/lib/python3.12/site-packages/openai/resources/chat/__init__.py @@ -0,0 +1,33 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from .chat import ( + Chat, + AsyncChat, + ChatWithRawResponse, + AsyncChatWithRawResponse, + ChatWithStreamingResponse, + AsyncChatWithStreamingResponse, +) +from .completions import ( + Completions, + AsyncCompletions, + CompletionsWithRawResponse, + AsyncCompletionsWithRawResponse, + CompletionsWithStreamingResponse, + AsyncCompletionsWithStreamingResponse, +) + +__all__ = [ + "Completions", + "AsyncCompletions", + "CompletionsWithRawResponse", + "AsyncCompletionsWithRawResponse", + "CompletionsWithStreamingResponse", + "AsyncCompletionsWithStreamingResponse", + "Chat", + "AsyncChat", + "ChatWithRawResponse", + "AsyncChatWithRawResponse", + "ChatWithStreamingResponse", + "AsyncChatWithStreamingResponse", +] diff --git a/lib/python3.12/site-packages/openai/resources/chat/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/openai/resources/chat/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..05b7aa546641525a21d35ba7adcd115206e6e942 Binary files /dev/null and b/lib/python3.12/site-packages/openai/resources/chat/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/resources/chat/__pycache__/chat.cpython-312.pyc b/lib/python3.12/site-packages/openai/resources/chat/__pycache__/chat.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..449efcfead60f2b04d7d31f66963c50aefa758c6 Binary files /dev/null and b/lib/python3.12/site-packages/openai/resources/chat/__pycache__/chat.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/resources/chat/chat.py b/lib/python3.12/site-packages/openai/resources/chat/chat.py new file mode 100644 index 0000000000000000000000000000000000000000..14f9224b416002d1de21fff09a1a5aab36143935 --- /dev/null +++ b/lib/python3.12/site-packages/openai/resources/chat/chat.py @@ -0,0 +1,102 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from ..._compat import cached_property +from ..._resource import SyncAPIResource, AsyncAPIResource +from .completions.completions import ( + Completions, + AsyncCompletions, + CompletionsWithRawResponse, + AsyncCompletionsWithRawResponse, + CompletionsWithStreamingResponse, + AsyncCompletionsWithStreamingResponse, +) + +__all__ = ["Chat", "AsyncChat"] + + +class Chat(SyncAPIResource): + @cached_property + def completions(self) -> Completions: + return Completions(self._client) + + @cached_property + def with_raw_response(self) -> ChatWithRawResponse: + """ + This property can be used as a prefix for any HTTP method call to return + the raw response object instead of the parsed content. + + For more information, see https://www.github.com/openai/openai-python#accessing-raw-response-data-eg-headers + """ + return ChatWithRawResponse(self) + + @cached_property + def with_streaming_response(self) -> ChatWithStreamingResponse: + """ + An alternative to `.with_raw_response` that doesn't eagerly read the response body. + + For more information, see https://www.github.com/openai/openai-python#with_streaming_response + """ + return ChatWithStreamingResponse(self) + + +class AsyncChat(AsyncAPIResource): + @cached_property + def completions(self) -> AsyncCompletions: + return AsyncCompletions(self._client) + + @cached_property + def with_raw_response(self) -> AsyncChatWithRawResponse: + """ + This property can be used as a prefix for any HTTP method call to return + the raw response object instead of the parsed content. + + For more information, see https://www.github.com/openai/openai-python#accessing-raw-response-data-eg-headers + """ + return AsyncChatWithRawResponse(self) + + @cached_property + def with_streaming_response(self) -> AsyncChatWithStreamingResponse: + """ + An alternative to `.with_raw_response` that doesn't eagerly read the response body. + + For more information, see https://www.github.com/openai/openai-python#with_streaming_response + """ + return AsyncChatWithStreamingResponse(self) + + +class ChatWithRawResponse: + def __init__(self, chat: Chat) -> None: + self._chat = chat + + @cached_property + def completions(self) -> CompletionsWithRawResponse: + return CompletionsWithRawResponse(self._chat.completions) + + +class AsyncChatWithRawResponse: + def __init__(self, chat: AsyncChat) -> None: + self._chat = chat + + @cached_property + def completions(self) -> AsyncCompletionsWithRawResponse: + return AsyncCompletionsWithRawResponse(self._chat.completions) + + +class ChatWithStreamingResponse: + def __init__(self, chat: Chat) -> None: + self._chat = chat + + @cached_property + def completions(self) -> CompletionsWithStreamingResponse: + return CompletionsWithStreamingResponse(self._chat.completions) + + +class AsyncChatWithStreamingResponse: + def __init__(self, chat: AsyncChat) -> None: + self._chat = chat + + @cached_property + def completions(self) -> AsyncCompletionsWithStreamingResponse: + return AsyncCompletionsWithStreamingResponse(self._chat.completions) diff --git a/lib/python3.12/site-packages/openai/resources/chat/completions/__init__.py b/lib/python3.12/site-packages/openai/resources/chat/completions/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..12d3b3aa2887d27c5babd492bfe1eb04a07d2379 --- /dev/null +++ b/lib/python3.12/site-packages/openai/resources/chat/completions/__init__.py @@ -0,0 +1,33 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from .messages import ( + Messages, + AsyncMessages, + MessagesWithRawResponse, + AsyncMessagesWithRawResponse, + MessagesWithStreamingResponse, + AsyncMessagesWithStreamingResponse, +) +from .completions import ( + Completions, + AsyncCompletions, + CompletionsWithRawResponse, + AsyncCompletionsWithRawResponse, + CompletionsWithStreamingResponse, + AsyncCompletionsWithStreamingResponse, +) + +__all__ = [ + "Messages", + "AsyncMessages", + "MessagesWithRawResponse", + "AsyncMessagesWithRawResponse", + "MessagesWithStreamingResponse", + "AsyncMessagesWithStreamingResponse", + "Completions", + "AsyncCompletions", + "CompletionsWithRawResponse", + "AsyncCompletionsWithRawResponse", + "CompletionsWithStreamingResponse", + "AsyncCompletionsWithStreamingResponse", +] diff --git a/lib/python3.12/site-packages/openai/resources/chat/completions/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/openai/resources/chat/completions/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..da5f34b3f2e1226ec77c08f59e244549bc6dc89a Binary files /dev/null and b/lib/python3.12/site-packages/openai/resources/chat/completions/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/resources/chat/completions/__pycache__/completions.cpython-312.pyc b/lib/python3.12/site-packages/openai/resources/chat/completions/__pycache__/completions.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7000e58a1fd859160bd4f0d773f0d173aef10600 Binary files /dev/null and b/lib/python3.12/site-packages/openai/resources/chat/completions/__pycache__/completions.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/resources/chat/completions/__pycache__/messages.cpython-312.pyc b/lib/python3.12/site-packages/openai/resources/chat/completions/__pycache__/messages.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ab0b178b2a082df90a0f078209097ccfd4bff42f Binary files /dev/null and b/lib/python3.12/site-packages/openai/resources/chat/completions/__pycache__/messages.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/resources/chat/completions/completions.py b/lib/python3.12/site-packages/openai/resources/chat/completions/completions.py new file mode 100644 index 0000000000000000000000000000000000000000..d6214225d8ff9248631d8471db0d6dc066ff9742 --- /dev/null +++ b/lib/python3.12/site-packages/openai/resources/chat/completions/completions.py @@ -0,0 +1,2365 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +import inspect +from typing import Dict, List, Union, Iterable, Optional +from typing_extensions import Literal, overload + +import httpx +import pydantic + +from .... import _legacy_response +from .messages import ( + Messages, + AsyncMessages, + MessagesWithRawResponse, + AsyncMessagesWithRawResponse, + MessagesWithStreamingResponse, + AsyncMessagesWithStreamingResponse, +) +from ...._types import NOT_GIVEN, Body, Query, Headers, NotGiven +from ...._utils import ( + required_args, + maybe_transform, + async_maybe_transform, +) +from ...._compat import cached_property +from ...._resource import SyncAPIResource, AsyncAPIResource +from ...._response import to_streamed_response_wrapper, async_to_streamed_response_wrapper +from ...._streaming import Stream, AsyncStream +from ....pagination import SyncCursorPage, AsyncCursorPage +from ....types.chat import ( + ChatCompletionAudioParam, + completion_list_params, + completion_create_params, + completion_update_params, +) +from ...._base_client import AsyncPaginator, make_request_options +from ....types.shared.chat_model import ChatModel +from ....types.chat.chat_completion import ChatCompletion +from ....types.shared_params.metadata import Metadata +from ....types.shared.reasoning_effort import ReasoningEffort +from ....types.chat.chat_completion_chunk import ChatCompletionChunk +from ....types.chat.chat_completion_deleted import ChatCompletionDeleted +from ....types.chat.chat_completion_tool_param import ChatCompletionToolParam +from ....types.chat.chat_completion_audio_param import ChatCompletionAudioParam +from ....types.chat.chat_completion_message_param import ChatCompletionMessageParam +from ....types.chat.chat_completion_stream_options_param import ChatCompletionStreamOptionsParam +from ....types.chat.chat_completion_prediction_content_param import ChatCompletionPredictionContentParam +from ....types.chat.chat_completion_tool_choice_option_param import ChatCompletionToolChoiceOptionParam + +__all__ = ["Completions", "AsyncCompletions"] + + +class Completions(SyncAPIResource): + @cached_property + def messages(self) -> Messages: + return Messages(self._client) + + @cached_property + def with_raw_response(self) -> CompletionsWithRawResponse: + """ + This property can be used as a prefix for any HTTP method call to return + the raw response object instead of the parsed content. + + For more information, see https://www.github.com/openai/openai-python#accessing-raw-response-data-eg-headers + """ + return CompletionsWithRawResponse(self) + + @cached_property + def with_streaming_response(self) -> CompletionsWithStreamingResponse: + """ + An alternative to `.with_raw_response` that doesn't eagerly read the response body. + + For more information, see https://www.github.com/openai/openai-python#with_streaming_response + """ + return CompletionsWithStreamingResponse(self) + + @overload + def create( + self, + *, + messages: Iterable[ChatCompletionMessageParam], + model: Union[str, ChatModel], + audio: Optional[ChatCompletionAudioParam] | NotGiven = NOT_GIVEN, + frequency_penalty: Optional[float] | NotGiven = NOT_GIVEN, + function_call: completion_create_params.FunctionCall | NotGiven = NOT_GIVEN, + functions: Iterable[completion_create_params.Function] | NotGiven = NOT_GIVEN, + logit_bias: Optional[Dict[str, int]] | NotGiven = NOT_GIVEN, + logprobs: Optional[bool] | NotGiven = NOT_GIVEN, + max_completion_tokens: Optional[int] | NotGiven = NOT_GIVEN, + max_tokens: Optional[int] | NotGiven = NOT_GIVEN, + metadata: Optional[Metadata] | NotGiven = NOT_GIVEN, + modalities: Optional[List[Literal["text", "audio"]]] | NotGiven = NOT_GIVEN, + n: Optional[int] | NotGiven = NOT_GIVEN, + parallel_tool_calls: bool | NotGiven = NOT_GIVEN, + prediction: Optional[ChatCompletionPredictionContentParam] | NotGiven = NOT_GIVEN, + presence_penalty: Optional[float] | NotGiven = NOT_GIVEN, + reasoning_effort: Optional[ReasoningEffort] | NotGiven = NOT_GIVEN, + response_format: completion_create_params.ResponseFormat | NotGiven = NOT_GIVEN, + seed: Optional[int] | NotGiven = NOT_GIVEN, + service_tier: Optional[Literal["auto", "default", "flex"]] | NotGiven = NOT_GIVEN, + stop: Union[Optional[str], List[str], None] | NotGiven = NOT_GIVEN, + store: Optional[bool] | NotGiven = NOT_GIVEN, + stream: Optional[Literal[False]] | NotGiven = NOT_GIVEN, + stream_options: Optional[ChatCompletionStreamOptionsParam] | NotGiven = NOT_GIVEN, + temperature: Optional[float] | NotGiven = NOT_GIVEN, + tool_choice: ChatCompletionToolChoiceOptionParam | NotGiven = NOT_GIVEN, + tools: Iterable[ChatCompletionToolParam] | NotGiven = NOT_GIVEN, + top_logprobs: Optional[int] | NotGiven = NOT_GIVEN, + top_p: Optional[float] | NotGiven = NOT_GIVEN, + user: str | NotGiven = NOT_GIVEN, + web_search_options: completion_create_params.WebSearchOptions | NotGiven = NOT_GIVEN, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> ChatCompletion: + """ + **Starting a new project?** We recommend trying + [Responses](https://platform.openai.com/docs/api-reference/responses) to take + advantage of the latest OpenAI platform features. Compare + [Chat Completions with Responses](https://platform.openai.com/docs/guides/responses-vs-chat-completions?api-mode=responses). + + --- + + Creates a model response for the given chat conversation. Learn more in the + [text generation](https://platform.openai.com/docs/guides/text-generation), + [vision](https://platform.openai.com/docs/guides/vision), and + [audio](https://platform.openai.com/docs/guides/audio) guides. + + Parameter support can differ depending on the model used to generate the + response, particularly for newer reasoning models. Parameters that are only + supported for reasoning models are noted below. For the current state of + unsupported parameters in reasoning models, + [refer to the reasoning guide](https://platform.openai.com/docs/guides/reasoning). + + Args: + messages: A list of messages comprising the conversation so far. Depending on the + [model](https://platform.openai.com/docs/models) you use, different message + types (modalities) are supported, like + [text](https://platform.openai.com/docs/guides/text-generation), + [images](https://platform.openai.com/docs/guides/vision), and + [audio](https://platform.openai.com/docs/guides/audio). + + model: Model ID used to generate the response, like `gpt-4o` or `o3`. OpenAI offers a + wide range of models with different capabilities, performance characteristics, + and price points. Refer to the + [model guide](https://platform.openai.com/docs/models) to browse and compare + available models. + + audio: Parameters for audio output. Required when audio output is requested with + `modalities: ["audio"]`. + [Learn more](https://platform.openai.com/docs/guides/audio). + + frequency_penalty: Number between -2.0 and 2.0. Positive values penalize new tokens based on their + existing frequency in the text so far, decreasing the model's likelihood to + repeat the same line verbatim. + + function_call: Deprecated in favor of `tool_choice`. + + Controls which (if any) function is called by the model. + + `none` means the model will not call a function and instead generates a message. + + `auto` means the model can pick between generating a message or calling a + function. + + Specifying a particular function via `{"name": "my_function"}` forces the model + to call that function. + + `none` is the default when no functions are present. `auto` is the default if + functions are present. + + functions: Deprecated in favor of `tools`. + + A list of functions the model may generate JSON inputs for. + + logit_bias: Modify the likelihood of specified tokens appearing in the completion. + + Accepts a JSON object that maps tokens (specified by their token ID in the + tokenizer) to an associated bias value from -100 to 100. Mathematically, the + bias is added to the logits generated by the model prior to sampling. The exact + effect will vary per model, but values between -1 and 1 should decrease or + increase likelihood of selection; values like -100 or 100 should result in a ban + or exclusive selection of the relevant token. + + logprobs: Whether to return log probabilities of the output tokens or not. If true, + returns the log probabilities of each output token returned in the `content` of + `message`. + + max_completion_tokens: An upper bound for the number of tokens that can be generated for a completion, + including visible output tokens and + [reasoning tokens](https://platform.openai.com/docs/guides/reasoning). + + max_tokens: The maximum number of [tokens](/tokenizer) that can be generated in the chat + completion. This value can be used to control + [costs](https://openai.com/api/pricing/) for text generated via API. + + This value is now deprecated in favor of `max_completion_tokens`, and is not + compatible with + [o-series models](https://platform.openai.com/docs/guides/reasoning). + + metadata: Set of 16 key-value pairs that can be attached to an object. This can be useful + for storing additional information about the object in a structured format, and + querying for objects via API or the dashboard. + + Keys are strings with a maximum length of 64 characters. Values are strings with + a maximum length of 512 characters. + + modalities: Output types that you would like the model to generate. Most models are capable + of generating text, which is the default: + + `["text"]` + + The `gpt-4o-audio-preview` model can also be used to + [generate audio](https://platform.openai.com/docs/guides/audio). To request that + this model generate both text and audio responses, you can use: + + `["text", "audio"]` + + n: How many chat completion choices to generate for each input message. Note that + you will be charged based on the number of generated tokens across all of the + choices. Keep `n` as `1` to minimize costs. + + parallel_tool_calls: Whether to enable + [parallel function calling](https://platform.openai.com/docs/guides/function-calling#configuring-parallel-function-calling) + during tool use. + + prediction: Static predicted output content, such as the content of a text file that is + being regenerated. + + presence_penalty: Number between -2.0 and 2.0. Positive values penalize new tokens based on + whether they appear in the text so far, increasing the model's likelihood to + talk about new topics. + + reasoning_effort: **o-series models only** + + Constrains effort on reasoning for + [reasoning models](https://platform.openai.com/docs/guides/reasoning). Currently + supported values are `low`, `medium`, and `high`. Reducing reasoning effort can + result in faster responses and fewer tokens used on reasoning in a response. + + response_format: An object specifying the format that the model must output. + + Setting to `{ "type": "json_schema", "json_schema": {...} }` enables Structured + Outputs which ensures the model will match your supplied JSON schema. Learn more + in the + [Structured Outputs guide](https://platform.openai.com/docs/guides/structured-outputs). + + Setting to `{ "type": "json_object" }` enables the older JSON mode, which + ensures the message the model generates is valid JSON. Using `json_schema` is + preferred for models that support it. + + seed: This feature is in Beta. If specified, our system will make a best effort to + sample deterministically, such that repeated requests with the same `seed` and + parameters should return the same result. Determinism is not guaranteed, and you + should refer to the `system_fingerprint` response parameter to monitor changes + in the backend. + + service_tier: Specifies the latency tier to use for processing the request. This parameter is + relevant for customers subscribed to the scale tier service: + + - If set to 'auto', and the Project is Scale tier enabled, the system will + utilize scale tier credits until they are exhausted. + - If set to 'auto', and the Project is not Scale tier enabled, the request will + be processed using the default service tier with a lower uptime SLA and no + latency guarentee. + - If set to 'default', the request will be processed using the default service + tier with a lower uptime SLA and no latency guarentee. + - If set to 'flex', the request will be processed with the Flex Processing + service tier. + [Learn more](https://platform.openai.com/docs/guides/flex-processing). + - When not set, the default behavior is 'auto'. + + When this parameter is set, the response body will include the `service_tier` + utilized. + + stop: Not supported with latest reasoning models `o3` and `o4-mini`. + + Up to 4 sequences where the API will stop generating further tokens. The + returned text will not contain the stop sequence. + + store: Whether or not to store the output of this chat completion request for use in + our [model distillation](https://platform.openai.com/docs/guides/distillation) + or [evals](https://platform.openai.com/docs/guides/evals) products. + + stream: If set to true, the model response data will be streamed to the client as it is + generated using + [server-sent events](https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events#Event_stream_format). + See the + [Streaming section below](https://platform.openai.com/docs/api-reference/chat/streaming) + for more information, along with the + [streaming responses](https://platform.openai.com/docs/guides/streaming-responses) + guide for more information on how to handle the streaming events. + + stream_options: Options for streaming response. Only set this when you set `stream: true`. + + temperature: What sampling temperature to use, between 0 and 2. Higher values like 0.8 will + make the output more random, while lower values like 0.2 will make it more + focused and deterministic. We generally recommend altering this or `top_p` but + not both. + + tool_choice: Controls which (if any) tool is called by the model. `none` means the model will + not call any tool and instead generates a message. `auto` means the model can + pick between generating a message or calling one or more tools. `required` means + the model must call one or more tools. Specifying a particular tool via + `{"type": "function", "function": {"name": "my_function"}}` forces the model to + call that tool. + + `none` is the default when no tools are present. `auto` is the default if tools + are present. + + tools: A list of tools the model may call. Currently, only functions are supported as a + tool. Use this to provide a list of functions the model may generate JSON inputs + for. A max of 128 functions are supported. + + top_logprobs: An integer between 0 and 20 specifying the number of most likely tokens to + return at each token position, each with an associated log probability. + `logprobs` must be set to `true` if this parameter is used. + + top_p: An alternative to sampling with temperature, called nucleus sampling, where the + model considers the results of the tokens with top_p probability mass. So 0.1 + means only the tokens comprising the top 10% probability mass are considered. + + We generally recommend altering this or `temperature` but not both. + + user: A unique identifier representing your end-user, which can help OpenAI to monitor + and detect abuse. + [Learn more](https://platform.openai.com/docs/guides/safety-best-practices#end-user-ids). + + web_search_options: This tool searches the web for relevant results to use in a response. Learn more + about the + [web search tool](https://platform.openai.com/docs/guides/tools-web-search?api-mode=chat). + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + ... + + @overload + def create( + self, + *, + messages: Iterable[ChatCompletionMessageParam], + model: Union[str, ChatModel], + stream: Literal[True], + audio: Optional[ChatCompletionAudioParam] | NotGiven = NOT_GIVEN, + frequency_penalty: Optional[float] | NotGiven = NOT_GIVEN, + function_call: completion_create_params.FunctionCall | NotGiven = NOT_GIVEN, + functions: Iterable[completion_create_params.Function] | NotGiven = NOT_GIVEN, + logit_bias: Optional[Dict[str, int]] | NotGiven = NOT_GIVEN, + logprobs: Optional[bool] | NotGiven = NOT_GIVEN, + max_completion_tokens: Optional[int] | NotGiven = NOT_GIVEN, + max_tokens: Optional[int] | NotGiven = NOT_GIVEN, + metadata: Optional[Metadata] | NotGiven = NOT_GIVEN, + modalities: Optional[List[Literal["text", "audio"]]] | NotGiven = NOT_GIVEN, + n: Optional[int] | NotGiven = NOT_GIVEN, + parallel_tool_calls: bool | NotGiven = NOT_GIVEN, + prediction: Optional[ChatCompletionPredictionContentParam] | NotGiven = NOT_GIVEN, + presence_penalty: Optional[float] | NotGiven = NOT_GIVEN, + reasoning_effort: Optional[ReasoningEffort] | NotGiven = NOT_GIVEN, + response_format: completion_create_params.ResponseFormat | NotGiven = NOT_GIVEN, + seed: Optional[int] | NotGiven = NOT_GIVEN, + service_tier: Optional[Literal["auto", "default", "flex"]] | NotGiven = NOT_GIVEN, + stop: Union[Optional[str], List[str], None] | NotGiven = NOT_GIVEN, + store: Optional[bool] | NotGiven = NOT_GIVEN, + stream_options: Optional[ChatCompletionStreamOptionsParam] | NotGiven = NOT_GIVEN, + temperature: Optional[float] | NotGiven = NOT_GIVEN, + tool_choice: ChatCompletionToolChoiceOptionParam | NotGiven = NOT_GIVEN, + tools: Iterable[ChatCompletionToolParam] | NotGiven = NOT_GIVEN, + top_logprobs: Optional[int] | NotGiven = NOT_GIVEN, + top_p: Optional[float] | NotGiven = NOT_GIVEN, + user: str | NotGiven = NOT_GIVEN, + web_search_options: completion_create_params.WebSearchOptions | NotGiven = NOT_GIVEN, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> Stream[ChatCompletionChunk]: + """ + **Starting a new project?** We recommend trying + [Responses](https://platform.openai.com/docs/api-reference/responses) to take + advantage of the latest OpenAI platform features. Compare + [Chat Completions with Responses](https://platform.openai.com/docs/guides/responses-vs-chat-completions?api-mode=responses). + + --- + + Creates a model response for the given chat conversation. Learn more in the + [text generation](https://platform.openai.com/docs/guides/text-generation), + [vision](https://platform.openai.com/docs/guides/vision), and + [audio](https://platform.openai.com/docs/guides/audio) guides. + + Parameter support can differ depending on the model used to generate the + response, particularly for newer reasoning models. Parameters that are only + supported for reasoning models are noted below. For the current state of + unsupported parameters in reasoning models, + [refer to the reasoning guide](https://platform.openai.com/docs/guides/reasoning). + + Args: + messages: A list of messages comprising the conversation so far. Depending on the + [model](https://platform.openai.com/docs/models) you use, different message + types (modalities) are supported, like + [text](https://platform.openai.com/docs/guides/text-generation), + [images](https://platform.openai.com/docs/guides/vision), and + [audio](https://platform.openai.com/docs/guides/audio). + + model: Model ID used to generate the response, like `gpt-4o` or `o3`. OpenAI offers a + wide range of models with different capabilities, performance characteristics, + and price points. Refer to the + [model guide](https://platform.openai.com/docs/models) to browse and compare + available models. + + stream: If set to true, the model response data will be streamed to the client as it is + generated using + [server-sent events](https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events#Event_stream_format). + See the + [Streaming section below](https://platform.openai.com/docs/api-reference/chat/streaming) + for more information, along with the + [streaming responses](https://platform.openai.com/docs/guides/streaming-responses) + guide for more information on how to handle the streaming events. + + audio: Parameters for audio output. Required when audio output is requested with + `modalities: ["audio"]`. + [Learn more](https://platform.openai.com/docs/guides/audio). + + frequency_penalty: Number between -2.0 and 2.0. Positive values penalize new tokens based on their + existing frequency in the text so far, decreasing the model's likelihood to + repeat the same line verbatim. + + function_call: Deprecated in favor of `tool_choice`. + + Controls which (if any) function is called by the model. + + `none` means the model will not call a function and instead generates a message. + + `auto` means the model can pick between generating a message or calling a + function. + + Specifying a particular function via `{"name": "my_function"}` forces the model + to call that function. + + `none` is the default when no functions are present. `auto` is the default if + functions are present. + + functions: Deprecated in favor of `tools`. + + A list of functions the model may generate JSON inputs for. + + logit_bias: Modify the likelihood of specified tokens appearing in the completion. + + Accepts a JSON object that maps tokens (specified by their token ID in the + tokenizer) to an associated bias value from -100 to 100. Mathematically, the + bias is added to the logits generated by the model prior to sampling. The exact + effect will vary per model, but values between -1 and 1 should decrease or + increase likelihood of selection; values like -100 or 100 should result in a ban + or exclusive selection of the relevant token. + + logprobs: Whether to return log probabilities of the output tokens or not. If true, + returns the log probabilities of each output token returned in the `content` of + `message`. + + max_completion_tokens: An upper bound for the number of tokens that can be generated for a completion, + including visible output tokens and + [reasoning tokens](https://platform.openai.com/docs/guides/reasoning). + + max_tokens: The maximum number of [tokens](/tokenizer) that can be generated in the chat + completion. This value can be used to control + [costs](https://openai.com/api/pricing/) for text generated via API. + + This value is now deprecated in favor of `max_completion_tokens`, and is not + compatible with + [o-series models](https://platform.openai.com/docs/guides/reasoning). + + metadata: Set of 16 key-value pairs that can be attached to an object. This can be useful + for storing additional information about the object in a structured format, and + querying for objects via API or the dashboard. + + Keys are strings with a maximum length of 64 characters. Values are strings with + a maximum length of 512 characters. + + modalities: Output types that you would like the model to generate. Most models are capable + of generating text, which is the default: + + `["text"]` + + The `gpt-4o-audio-preview` model can also be used to + [generate audio](https://platform.openai.com/docs/guides/audio). To request that + this model generate both text and audio responses, you can use: + + `["text", "audio"]` + + n: How many chat completion choices to generate for each input message. Note that + you will be charged based on the number of generated tokens across all of the + choices. Keep `n` as `1` to minimize costs. + + parallel_tool_calls: Whether to enable + [parallel function calling](https://platform.openai.com/docs/guides/function-calling#configuring-parallel-function-calling) + during tool use. + + prediction: Static predicted output content, such as the content of a text file that is + being regenerated. + + presence_penalty: Number between -2.0 and 2.0. Positive values penalize new tokens based on + whether they appear in the text so far, increasing the model's likelihood to + talk about new topics. + + reasoning_effort: **o-series models only** + + Constrains effort on reasoning for + [reasoning models](https://platform.openai.com/docs/guides/reasoning). Currently + supported values are `low`, `medium`, and `high`. Reducing reasoning effort can + result in faster responses and fewer tokens used on reasoning in a response. + + response_format: An object specifying the format that the model must output. + + Setting to `{ "type": "json_schema", "json_schema": {...} }` enables Structured + Outputs which ensures the model will match your supplied JSON schema. Learn more + in the + [Structured Outputs guide](https://platform.openai.com/docs/guides/structured-outputs). + + Setting to `{ "type": "json_object" }` enables the older JSON mode, which + ensures the message the model generates is valid JSON. Using `json_schema` is + preferred for models that support it. + + seed: This feature is in Beta. If specified, our system will make a best effort to + sample deterministically, such that repeated requests with the same `seed` and + parameters should return the same result. Determinism is not guaranteed, and you + should refer to the `system_fingerprint` response parameter to monitor changes + in the backend. + + service_tier: Specifies the latency tier to use for processing the request. This parameter is + relevant for customers subscribed to the scale tier service: + + - If set to 'auto', and the Project is Scale tier enabled, the system will + utilize scale tier credits until they are exhausted. + - If set to 'auto', and the Project is not Scale tier enabled, the request will + be processed using the default service tier with a lower uptime SLA and no + latency guarentee. + - If set to 'default', the request will be processed using the default service + tier with a lower uptime SLA and no latency guarentee. + - If set to 'flex', the request will be processed with the Flex Processing + service tier. + [Learn more](https://platform.openai.com/docs/guides/flex-processing). + - When not set, the default behavior is 'auto'. + + When this parameter is set, the response body will include the `service_tier` + utilized. + + stop: Not supported with latest reasoning models `o3` and `o4-mini`. + + Up to 4 sequences where the API will stop generating further tokens. The + returned text will not contain the stop sequence. + + store: Whether or not to store the output of this chat completion request for use in + our [model distillation](https://platform.openai.com/docs/guides/distillation) + or [evals](https://platform.openai.com/docs/guides/evals) products. + + stream_options: Options for streaming response. Only set this when you set `stream: true`. + + temperature: What sampling temperature to use, between 0 and 2. Higher values like 0.8 will + make the output more random, while lower values like 0.2 will make it more + focused and deterministic. We generally recommend altering this or `top_p` but + not both. + + tool_choice: Controls which (if any) tool is called by the model. `none` means the model will + not call any tool and instead generates a message. `auto` means the model can + pick between generating a message or calling one or more tools. `required` means + the model must call one or more tools. Specifying a particular tool via + `{"type": "function", "function": {"name": "my_function"}}` forces the model to + call that tool. + + `none` is the default when no tools are present. `auto` is the default if tools + are present. + + tools: A list of tools the model may call. Currently, only functions are supported as a + tool. Use this to provide a list of functions the model may generate JSON inputs + for. A max of 128 functions are supported. + + top_logprobs: An integer between 0 and 20 specifying the number of most likely tokens to + return at each token position, each with an associated log probability. + `logprobs` must be set to `true` if this parameter is used. + + top_p: An alternative to sampling with temperature, called nucleus sampling, where the + model considers the results of the tokens with top_p probability mass. So 0.1 + means only the tokens comprising the top 10% probability mass are considered. + + We generally recommend altering this or `temperature` but not both. + + user: A unique identifier representing your end-user, which can help OpenAI to monitor + and detect abuse. + [Learn more](https://platform.openai.com/docs/guides/safety-best-practices#end-user-ids). + + web_search_options: This tool searches the web for relevant results to use in a response. Learn more + about the + [web search tool](https://platform.openai.com/docs/guides/tools-web-search?api-mode=chat). + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + ... + + @overload + def create( + self, + *, + messages: Iterable[ChatCompletionMessageParam], + model: Union[str, ChatModel], + stream: bool, + audio: Optional[ChatCompletionAudioParam] | NotGiven = NOT_GIVEN, + frequency_penalty: Optional[float] | NotGiven = NOT_GIVEN, + function_call: completion_create_params.FunctionCall | NotGiven = NOT_GIVEN, + functions: Iterable[completion_create_params.Function] | NotGiven = NOT_GIVEN, + logit_bias: Optional[Dict[str, int]] | NotGiven = NOT_GIVEN, + logprobs: Optional[bool] | NotGiven = NOT_GIVEN, + max_completion_tokens: Optional[int] | NotGiven = NOT_GIVEN, + max_tokens: Optional[int] | NotGiven = NOT_GIVEN, + metadata: Optional[Metadata] | NotGiven = NOT_GIVEN, + modalities: Optional[List[Literal["text", "audio"]]] | NotGiven = NOT_GIVEN, + n: Optional[int] | NotGiven = NOT_GIVEN, + parallel_tool_calls: bool | NotGiven = NOT_GIVEN, + prediction: Optional[ChatCompletionPredictionContentParam] | NotGiven = NOT_GIVEN, + presence_penalty: Optional[float] | NotGiven = NOT_GIVEN, + reasoning_effort: Optional[ReasoningEffort] | NotGiven = NOT_GIVEN, + response_format: completion_create_params.ResponseFormat | NotGiven = NOT_GIVEN, + seed: Optional[int] | NotGiven = NOT_GIVEN, + service_tier: Optional[Literal["auto", "default", "flex"]] | NotGiven = NOT_GIVEN, + stop: Union[Optional[str], List[str], None] | NotGiven = NOT_GIVEN, + store: Optional[bool] | NotGiven = NOT_GIVEN, + stream_options: Optional[ChatCompletionStreamOptionsParam] | NotGiven = NOT_GIVEN, + temperature: Optional[float] | NotGiven = NOT_GIVEN, + tool_choice: ChatCompletionToolChoiceOptionParam | NotGiven = NOT_GIVEN, + tools: Iterable[ChatCompletionToolParam] | NotGiven = NOT_GIVEN, + top_logprobs: Optional[int] | NotGiven = NOT_GIVEN, + top_p: Optional[float] | NotGiven = NOT_GIVEN, + user: str | NotGiven = NOT_GIVEN, + web_search_options: completion_create_params.WebSearchOptions | NotGiven = NOT_GIVEN, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> ChatCompletion | Stream[ChatCompletionChunk]: + """ + **Starting a new project?** We recommend trying + [Responses](https://platform.openai.com/docs/api-reference/responses) to take + advantage of the latest OpenAI platform features. Compare + [Chat Completions with Responses](https://platform.openai.com/docs/guides/responses-vs-chat-completions?api-mode=responses). + + --- + + Creates a model response for the given chat conversation. Learn more in the + [text generation](https://platform.openai.com/docs/guides/text-generation), + [vision](https://platform.openai.com/docs/guides/vision), and + [audio](https://platform.openai.com/docs/guides/audio) guides. + + Parameter support can differ depending on the model used to generate the + response, particularly for newer reasoning models. Parameters that are only + supported for reasoning models are noted below. For the current state of + unsupported parameters in reasoning models, + [refer to the reasoning guide](https://platform.openai.com/docs/guides/reasoning). + + Args: + messages: A list of messages comprising the conversation so far. Depending on the + [model](https://platform.openai.com/docs/models) you use, different message + types (modalities) are supported, like + [text](https://platform.openai.com/docs/guides/text-generation), + [images](https://platform.openai.com/docs/guides/vision), and + [audio](https://platform.openai.com/docs/guides/audio). + + model: Model ID used to generate the response, like `gpt-4o` or `o3`. OpenAI offers a + wide range of models with different capabilities, performance characteristics, + and price points. Refer to the + [model guide](https://platform.openai.com/docs/models) to browse and compare + available models. + + stream: If set to true, the model response data will be streamed to the client as it is + generated using + [server-sent events](https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events#Event_stream_format). + See the + [Streaming section below](https://platform.openai.com/docs/api-reference/chat/streaming) + for more information, along with the + [streaming responses](https://platform.openai.com/docs/guides/streaming-responses) + guide for more information on how to handle the streaming events. + + audio: Parameters for audio output. Required when audio output is requested with + `modalities: ["audio"]`. + [Learn more](https://platform.openai.com/docs/guides/audio). + + frequency_penalty: Number between -2.0 and 2.0. Positive values penalize new tokens based on their + existing frequency in the text so far, decreasing the model's likelihood to + repeat the same line verbatim. + + function_call: Deprecated in favor of `tool_choice`. + + Controls which (if any) function is called by the model. + + `none` means the model will not call a function and instead generates a message. + + `auto` means the model can pick between generating a message or calling a + function. + + Specifying a particular function via `{"name": "my_function"}` forces the model + to call that function. + + `none` is the default when no functions are present. `auto` is the default if + functions are present. + + functions: Deprecated in favor of `tools`. + + A list of functions the model may generate JSON inputs for. + + logit_bias: Modify the likelihood of specified tokens appearing in the completion. + + Accepts a JSON object that maps tokens (specified by their token ID in the + tokenizer) to an associated bias value from -100 to 100. Mathematically, the + bias is added to the logits generated by the model prior to sampling. The exact + effect will vary per model, but values between -1 and 1 should decrease or + increase likelihood of selection; values like -100 or 100 should result in a ban + or exclusive selection of the relevant token. + + logprobs: Whether to return log probabilities of the output tokens or not. If true, + returns the log probabilities of each output token returned in the `content` of + `message`. + + max_completion_tokens: An upper bound for the number of tokens that can be generated for a completion, + including visible output tokens and + [reasoning tokens](https://platform.openai.com/docs/guides/reasoning). + + max_tokens: The maximum number of [tokens](/tokenizer) that can be generated in the chat + completion. This value can be used to control + [costs](https://openai.com/api/pricing/) for text generated via API. + + This value is now deprecated in favor of `max_completion_tokens`, and is not + compatible with + [o-series models](https://platform.openai.com/docs/guides/reasoning). + + metadata: Set of 16 key-value pairs that can be attached to an object. This can be useful + for storing additional information about the object in a structured format, and + querying for objects via API or the dashboard. + + Keys are strings with a maximum length of 64 characters. Values are strings with + a maximum length of 512 characters. + + modalities: Output types that you would like the model to generate. Most models are capable + of generating text, which is the default: + + `["text"]` + + The `gpt-4o-audio-preview` model can also be used to + [generate audio](https://platform.openai.com/docs/guides/audio). To request that + this model generate both text and audio responses, you can use: + + `["text", "audio"]` + + n: How many chat completion choices to generate for each input message. Note that + you will be charged based on the number of generated tokens across all of the + choices. Keep `n` as `1` to minimize costs. + + parallel_tool_calls: Whether to enable + [parallel function calling](https://platform.openai.com/docs/guides/function-calling#configuring-parallel-function-calling) + during tool use. + + prediction: Static predicted output content, such as the content of a text file that is + being regenerated. + + presence_penalty: Number between -2.0 and 2.0. Positive values penalize new tokens based on + whether they appear in the text so far, increasing the model's likelihood to + talk about new topics. + + reasoning_effort: **o-series models only** + + Constrains effort on reasoning for + [reasoning models](https://platform.openai.com/docs/guides/reasoning). Currently + supported values are `low`, `medium`, and `high`. Reducing reasoning effort can + result in faster responses and fewer tokens used on reasoning in a response. + + response_format: An object specifying the format that the model must output. + + Setting to `{ "type": "json_schema", "json_schema": {...} }` enables Structured + Outputs which ensures the model will match your supplied JSON schema. Learn more + in the + [Structured Outputs guide](https://platform.openai.com/docs/guides/structured-outputs). + + Setting to `{ "type": "json_object" }` enables the older JSON mode, which + ensures the message the model generates is valid JSON. Using `json_schema` is + preferred for models that support it. + + seed: This feature is in Beta. If specified, our system will make a best effort to + sample deterministically, such that repeated requests with the same `seed` and + parameters should return the same result. Determinism is not guaranteed, and you + should refer to the `system_fingerprint` response parameter to monitor changes + in the backend. + + service_tier: Specifies the latency tier to use for processing the request. This parameter is + relevant for customers subscribed to the scale tier service: + + - If set to 'auto', and the Project is Scale tier enabled, the system will + utilize scale tier credits until they are exhausted. + - If set to 'auto', and the Project is not Scale tier enabled, the request will + be processed using the default service tier with a lower uptime SLA and no + latency guarentee. + - If set to 'default', the request will be processed using the default service + tier with a lower uptime SLA and no latency guarentee. + - If set to 'flex', the request will be processed with the Flex Processing + service tier. + [Learn more](https://platform.openai.com/docs/guides/flex-processing). + - When not set, the default behavior is 'auto'. + + When this parameter is set, the response body will include the `service_tier` + utilized. + + stop: Not supported with latest reasoning models `o3` and `o4-mini`. + + Up to 4 sequences where the API will stop generating further tokens. The + returned text will not contain the stop sequence. + + store: Whether or not to store the output of this chat completion request for use in + our [model distillation](https://platform.openai.com/docs/guides/distillation) + or [evals](https://platform.openai.com/docs/guides/evals) products. + + stream_options: Options for streaming response. Only set this when you set `stream: true`. + + temperature: What sampling temperature to use, between 0 and 2. Higher values like 0.8 will + make the output more random, while lower values like 0.2 will make it more + focused and deterministic. We generally recommend altering this or `top_p` but + not both. + + tool_choice: Controls which (if any) tool is called by the model. `none` means the model will + not call any tool and instead generates a message. `auto` means the model can + pick between generating a message or calling one or more tools. `required` means + the model must call one or more tools. Specifying a particular tool via + `{"type": "function", "function": {"name": "my_function"}}` forces the model to + call that tool. + + `none` is the default when no tools are present. `auto` is the default if tools + are present. + + tools: A list of tools the model may call. Currently, only functions are supported as a + tool. Use this to provide a list of functions the model may generate JSON inputs + for. A max of 128 functions are supported. + + top_logprobs: An integer between 0 and 20 specifying the number of most likely tokens to + return at each token position, each with an associated log probability. + `logprobs` must be set to `true` if this parameter is used. + + top_p: An alternative to sampling with temperature, called nucleus sampling, where the + model considers the results of the tokens with top_p probability mass. So 0.1 + means only the tokens comprising the top 10% probability mass are considered. + + We generally recommend altering this or `temperature` but not both. + + user: A unique identifier representing your end-user, which can help OpenAI to monitor + and detect abuse. + [Learn more](https://platform.openai.com/docs/guides/safety-best-practices#end-user-ids). + + web_search_options: This tool searches the web for relevant results to use in a response. Learn more + about the + [web search tool](https://platform.openai.com/docs/guides/tools-web-search?api-mode=chat). + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + ... + + @required_args(["messages", "model"], ["messages", "model", "stream"]) + def create( + self, + *, + messages: Iterable[ChatCompletionMessageParam], + model: Union[str, ChatModel], + audio: Optional[ChatCompletionAudioParam] | NotGiven = NOT_GIVEN, + frequency_penalty: Optional[float] | NotGiven = NOT_GIVEN, + function_call: completion_create_params.FunctionCall | NotGiven = NOT_GIVEN, + functions: Iterable[completion_create_params.Function] | NotGiven = NOT_GIVEN, + logit_bias: Optional[Dict[str, int]] | NotGiven = NOT_GIVEN, + logprobs: Optional[bool] | NotGiven = NOT_GIVEN, + max_completion_tokens: Optional[int] | NotGiven = NOT_GIVEN, + max_tokens: Optional[int] | NotGiven = NOT_GIVEN, + metadata: Optional[Metadata] | NotGiven = NOT_GIVEN, + modalities: Optional[List[Literal["text", "audio"]]] | NotGiven = NOT_GIVEN, + n: Optional[int] | NotGiven = NOT_GIVEN, + parallel_tool_calls: bool | NotGiven = NOT_GIVEN, + prediction: Optional[ChatCompletionPredictionContentParam] | NotGiven = NOT_GIVEN, + presence_penalty: Optional[float] | NotGiven = NOT_GIVEN, + reasoning_effort: Optional[ReasoningEffort] | NotGiven = NOT_GIVEN, + response_format: completion_create_params.ResponseFormat | NotGiven = NOT_GIVEN, + seed: Optional[int] | NotGiven = NOT_GIVEN, + service_tier: Optional[Literal["auto", "default", "flex"]] | NotGiven = NOT_GIVEN, + stop: Union[Optional[str], List[str], None] | NotGiven = NOT_GIVEN, + store: Optional[bool] | NotGiven = NOT_GIVEN, + stream: Optional[Literal[False]] | Literal[True] | NotGiven = NOT_GIVEN, + stream_options: Optional[ChatCompletionStreamOptionsParam] | NotGiven = NOT_GIVEN, + temperature: Optional[float] | NotGiven = NOT_GIVEN, + tool_choice: ChatCompletionToolChoiceOptionParam | NotGiven = NOT_GIVEN, + tools: Iterable[ChatCompletionToolParam] | NotGiven = NOT_GIVEN, + top_logprobs: Optional[int] | NotGiven = NOT_GIVEN, + top_p: Optional[float] | NotGiven = NOT_GIVEN, + user: str | NotGiven = NOT_GIVEN, + web_search_options: completion_create_params.WebSearchOptions | NotGiven = NOT_GIVEN, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> ChatCompletion | Stream[ChatCompletionChunk]: + validate_response_format(response_format) + return self._post( + "/chat/completions", + body=maybe_transform( + { + "messages": messages, + "model": model, + "audio": audio, + "frequency_penalty": frequency_penalty, + "function_call": function_call, + "functions": functions, + "logit_bias": logit_bias, + "logprobs": logprobs, + "max_completion_tokens": max_completion_tokens, + "max_tokens": max_tokens, + "metadata": metadata, + "modalities": modalities, + "n": n, + "parallel_tool_calls": parallel_tool_calls, + "prediction": prediction, + "presence_penalty": presence_penalty, + "reasoning_effort": reasoning_effort, + "response_format": response_format, + "seed": seed, + "service_tier": service_tier, + "stop": stop, + "store": store, + "stream": stream, + "stream_options": stream_options, + "temperature": temperature, + "tool_choice": tool_choice, + "tools": tools, + "top_logprobs": top_logprobs, + "top_p": top_p, + "user": user, + "web_search_options": web_search_options, + }, + completion_create_params.CompletionCreateParamsStreaming + if stream + else completion_create_params.CompletionCreateParamsNonStreaming, + ), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=ChatCompletion, + stream=stream or False, + stream_cls=Stream[ChatCompletionChunk], + ) + + def retrieve( + self, + completion_id: str, + *, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> ChatCompletion: + """Get a stored chat completion. + + Only Chat Completions that have been created with + the `store` parameter set to `true` will be returned. + + Args: + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if not completion_id: + raise ValueError(f"Expected a non-empty value for `completion_id` but received {completion_id!r}") + return self._get( + f"/chat/completions/{completion_id}", + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=ChatCompletion, + ) + + def update( + self, + completion_id: str, + *, + metadata: Optional[Metadata], + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> ChatCompletion: + """Modify a stored chat completion. + + Only Chat Completions that have been created + with the `store` parameter set to `true` can be modified. Currently, the only + supported modification is to update the `metadata` field. + + Args: + metadata: Set of 16 key-value pairs that can be attached to an object. This can be useful + for storing additional information about the object in a structured format, and + querying for objects via API or the dashboard. + + Keys are strings with a maximum length of 64 characters. Values are strings with + a maximum length of 512 characters. + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if not completion_id: + raise ValueError(f"Expected a non-empty value for `completion_id` but received {completion_id!r}") + return self._post( + f"/chat/completions/{completion_id}", + body=maybe_transform({"metadata": metadata}, completion_update_params.CompletionUpdateParams), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=ChatCompletion, + ) + + def list( + self, + *, + after: str | NotGiven = NOT_GIVEN, + limit: int | NotGiven = NOT_GIVEN, + metadata: Optional[Metadata] | NotGiven = NOT_GIVEN, + model: str | NotGiven = NOT_GIVEN, + order: Literal["asc", "desc"] | NotGiven = NOT_GIVEN, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> SyncCursorPage[ChatCompletion]: + """List stored Chat Completions. + + Only Chat Completions that have been stored with + the `store` parameter set to `true` will be returned. + + Args: + after: Identifier for the last chat completion from the previous pagination request. + + limit: Number of Chat Completions to retrieve. + + metadata: + A list of metadata keys to filter the Chat Completions by. Example: + + `metadata[key1]=value1&metadata[key2]=value2` + + model: The model used to generate the Chat Completions. + + order: Sort order for Chat Completions by timestamp. Use `asc` for ascending order or + `desc` for descending order. Defaults to `asc`. + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + return self._get_api_list( + "/chat/completions", + page=SyncCursorPage[ChatCompletion], + options=make_request_options( + extra_headers=extra_headers, + extra_query=extra_query, + extra_body=extra_body, + timeout=timeout, + query=maybe_transform( + { + "after": after, + "limit": limit, + "metadata": metadata, + "model": model, + "order": order, + }, + completion_list_params.CompletionListParams, + ), + ), + model=ChatCompletion, + ) + + def delete( + self, + completion_id: str, + *, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> ChatCompletionDeleted: + """Delete a stored chat completion. + + Only Chat Completions that have been created + with the `store` parameter set to `true` can be deleted. + + Args: + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if not completion_id: + raise ValueError(f"Expected a non-empty value for `completion_id` but received {completion_id!r}") + return self._delete( + f"/chat/completions/{completion_id}", + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=ChatCompletionDeleted, + ) + + +class AsyncCompletions(AsyncAPIResource): + @cached_property + def messages(self) -> AsyncMessages: + return AsyncMessages(self._client) + + @cached_property + def with_raw_response(self) -> AsyncCompletionsWithRawResponse: + """ + This property can be used as a prefix for any HTTP method call to return + the raw response object instead of the parsed content. + + For more information, see https://www.github.com/openai/openai-python#accessing-raw-response-data-eg-headers + """ + return AsyncCompletionsWithRawResponse(self) + + @cached_property + def with_streaming_response(self) -> AsyncCompletionsWithStreamingResponse: + """ + An alternative to `.with_raw_response` that doesn't eagerly read the response body. + + For more information, see https://www.github.com/openai/openai-python#with_streaming_response + """ + return AsyncCompletionsWithStreamingResponse(self) + + @overload + async def create( + self, + *, + messages: Iterable[ChatCompletionMessageParam], + model: Union[str, ChatModel], + audio: Optional[ChatCompletionAudioParam] | NotGiven = NOT_GIVEN, + frequency_penalty: Optional[float] | NotGiven = NOT_GIVEN, + function_call: completion_create_params.FunctionCall | NotGiven = NOT_GIVEN, + functions: Iterable[completion_create_params.Function] | NotGiven = NOT_GIVEN, + logit_bias: Optional[Dict[str, int]] | NotGiven = NOT_GIVEN, + logprobs: Optional[bool] | NotGiven = NOT_GIVEN, + max_completion_tokens: Optional[int] | NotGiven = NOT_GIVEN, + max_tokens: Optional[int] | NotGiven = NOT_GIVEN, + metadata: Optional[Metadata] | NotGiven = NOT_GIVEN, + modalities: Optional[List[Literal["text", "audio"]]] | NotGiven = NOT_GIVEN, + n: Optional[int] | NotGiven = NOT_GIVEN, + parallel_tool_calls: bool | NotGiven = NOT_GIVEN, + prediction: Optional[ChatCompletionPredictionContentParam] | NotGiven = NOT_GIVEN, + presence_penalty: Optional[float] | NotGiven = NOT_GIVEN, + reasoning_effort: Optional[ReasoningEffort] | NotGiven = NOT_GIVEN, + response_format: completion_create_params.ResponseFormat | NotGiven = NOT_GIVEN, + seed: Optional[int] | NotGiven = NOT_GIVEN, + service_tier: Optional[Literal["auto", "default", "flex"]] | NotGiven = NOT_GIVEN, + stop: Union[Optional[str], List[str], None] | NotGiven = NOT_GIVEN, + store: Optional[bool] | NotGiven = NOT_GIVEN, + stream: Optional[Literal[False]] | NotGiven = NOT_GIVEN, + stream_options: Optional[ChatCompletionStreamOptionsParam] | NotGiven = NOT_GIVEN, + temperature: Optional[float] | NotGiven = NOT_GIVEN, + tool_choice: ChatCompletionToolChoiceOptionParam | NotGiven = NOT_GIVEN, + tools: Iterable[ChatCompletionToolParam] | NotGiven = NOT_GIVEN, + top_logprobs: Optional[int] | NotGiven = NOT_GIVEN, + top_p: Optional[float] | NotGiven = NOT_GIVEN, + user: str | NotGiven = NOT_GIVEN, + web_search_options: completion_create_params.WebSearchOptions | NotGiven = NOT_GIVEN, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> ChatCompletion: + """ + **Starting a new project?** We recommend trying + [Responses](https://platform.openai.com/docs/api-reference/responses) to take + advantage of the latest OpenAI platform features. Compare + [Chat Completions with Responses](https://platform.openai.com/docs/guides/responses-vs-chat-completions?api-mode=responses). + + --- + + Creates a model response for the given chat conversation. Learn more in the + [text generation](https://platform.openai.com/docs/guides/text-generation), + [vision](https://platform.openai.com/docs/guides/vision), and + [audio](https://platform.openai.com/docs/guides/audio) guides. + + Parameter support can differ depending on the model used to generate the + response, particularly for newer reasoning models. Parameters that are only + supported for reasoning models are noted below. For the current state of + unsupported parameters in reasoning models, + [refer to the reasoning guide](https://platform.openai.com/docs/guides/reasoning). + + Args: + messages: A list of messages comprising the conversation so far. Depending on the + [model](https://platform.openai.com/docs/models) you use, different message + types (modalities) are supported, like + [text](https://platform.openai.com/docs/guides/text-generation), + [images](https://platform.openai.com/docs/guides/vision), and + [audio](https://platform.openai.com/docs/guides/audio). + + model: Model ID used to generate the response, like `gpt-4o` or `o3`. OpenAI offers a + wide range of models with different capabilities, performance characteristics, + and price points. Refer to the + [model guide](https://platform.openai.com/docs/models) to browse and compare + available models. + + audio: Parameters for audio output. Required when audio output is requested with + `modalities: ["audio"]`. + [Learn more](https://platform.openai.com/docs/guides/audio). + + frequency_penalty: Number between -2.0 and 2.0. Positive values penalize new tokens based on their + existing frequency in the text so far, decreasing the model's likelihood to + repeat the same line verbatim. + + function_call: Deprecated in favor of `tool_choice`. + + Controls which (if any) function is called by the model. + + `none` means the model will not call a function and instead generates a message. + + `auto` means the model can pick between generating a message or calling a + function. + + Specifying a particular function via `{"name": "my_function"}` forces the model + to call that function. + + `none` is the default when no functions are present. `auto` is the default if + functions are present. + + functions: Deprecated in favor of `tools`. + + A list of functions the model may generate JSON inputs for. + + logit_bias: Modify the likelihood of specified tokens appearing in the completion. + + Accepts a JSON object that maps tokens (specified by their token ID in the + tokenizer) to an associated bias value from -100 to 100. Mathematically, the + bias is added to the logits generated by the model prior to sampling. The exact + effect will vary per model, but values between -1 and 1 should decrease or + increase likelihood of selection; values like -100 or 100 should result in a ban + or exclusive selection of the relevant token. + + logprobs: Whether to return log probabilities of the output tokens or not. If true, + returns the log probabilities of each output token returned in the `content` of + `message`. + + max_completion_tokens: An upper bound for the number of tokens that can be generated for a completion, + including visible output tokens and + [reasoning tokens](https://platform.openai.com/docs/guides/reasoning). + + max_tokens: The maximum number of [tokens](/tokenizer) that can be generated in the chat + completion. This value can be used to control + [costs](https://openai.com/api/pricing/) for text generated via API. + + This value is now deprecated in favor of `max_completion_tokens`, and is not + compatible with + [o-series models](https://platform.openai.com/docs/guides/reasoning). + + metadata: Set of 16 key-value pairs that can be attached to an object. This can be useful + for storing additional information about the object in a structured format, and + querying for objects via API or the dashboard. + + Keys are strings with a maximum length of 64 characters. Values are strings with + a maximum length of 512 characters. + + modalities: Output types that you would like the model to generate. Most models are capable + of generating text, which is the default: + + `["text"]` + + The `gpt-4o-audio-preview` model can also be used to + [generate audio](https://platform.openai.com/docs/guides/audio). To request that + this model generate both text and audio responses, you can use: + + `["text", "audio"]` + + n: How many chat completion choices to generate for each input message. Note that + you will be charged based on the number of generated tokens across all of the + choices. Keep `n` as `1` to minimize costs. + + parallel_tool_calls: Whether to enable + [parallel function calling](https://platform.openai.com/docs/guides/function-calling#configuring-parallel-function-calling) + during tool use. + + prediction: Static predicted output content, such as the content of a text file that is + being regenerated. + + presence_penalty: Number between -2.0 and 2.0. Positive values penalize new tokens based on + whether they appear in the text so far, increasing the model's likelihood to + talk about new topics. + + reasoning_effort: **o-series models only** + + Constrains effort on reasoning for + [reasoning models](https://platform.openai.com/docs/guides/reasoning). Currently + supported values are `low`, `medium`, and `high`. Reducing reasoning effort can + result in faster responses and fewer tokens used on reasoning in a response. + + response_format: An object specifying the format that the model must output. + + Setting to `{ "type": "json_schema", "json_schema": {...} }` enables Structured + Outputs which ensures the model will match your supplied JSON schema. Learn more + in the + [Structured Outputs guide](https://platform.openai.com/docs/guides/structured-outputs). + + Setting to `{ "type": "json_object" }` enables the older JSON mode, which + ensures the message the model generates is valid JSON. Using `json_schema` is + preferred for models that support it. + + seed: This feature is in Beta. If specified, our system will make a best effort to + sample deterministically, such that repeated requests with the same `seed` and + parameters should return the same result. Determinism is not guaranteed, and you + should refer to the `system_fingerprint` response parameter to monitor changes + in the backend. + + service_tier: Specifies the latency tier to use for processing the request. This parameter is + relevant for customers subscribed to the scale tier service: + + - If set to 'auto', and the Project is Scale tier enabled, the system will + utilize scale tier credits until they are exhausted. + - If set to 'auto', and the Project is not Scale tier enabled, the request will + be processed using the default service tier with a lower uptime SLA and no + latency guarentee. + - If set to 'default', the request will be processed using the default service + tier with a lower uptime SLA and no latency guarentee. + - If set to 'flex', the request will be processed with the Flex Processing + service tier. + [Learn more](https://platform.openai.com/docs/guides/flex-processing). + - When not set, the default behavior is 'auto'. + + When this parameter is set, the response body will include the `service_tier` + utilized. + + stop: Not supported with latest reasoning models `o3` and `o4-mini`. + + Up to 4 sequences where the API will stop generating further tokens. The + returned text will not contain the stop sequence. + + store: Whether or not to store the output of this chat completion request for use in + our [model distillation](https://platform.openai.com/docs/guides/distillation) + or [evals](https://platform.openai.com/docs/guides/evals) products. + + stream: If set to true, the model response data will be streamed to the client as it is + generated using + [server-sent events](https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events#Event_stream_format). + See the + [Streaming section below](https://platform.openai.com/docs/api-reference/chat/streaming) + for more information, along with the + [streaming responses](https://platform.openai.com/docs/guides/streaming-responses) + guide for more information on how to handle the streaming events. + + stream_options: Options for streaming response. Only set this when you set `stream: true`. + + temperature: What sampling temperature to use, between 0 and 2. Higher values like 0.8 will + make the output more random, while lower values like 0.2 will make it more + focused and deterministic. We generally recommend altering this or `top_p` but + not both. + + tool_choice: Controls which (if any) tool is called by the model. `none` means the model will + not call any tool and instead generates a message. `auto` means the model can + pick between generating a message or calling one or more tools. `required` means + the model must call one or more tools. Specifying a particular tool via + `{"type": "function", "function": {"name": "my_function"}}` forces the model to + call that tool. + + `none` is the default when no tools are present. `auto` is the default if tools + are present. + + tools: A list of tools the model may call. Currently, only functions are supported as a + tool. Use this to provide a list of functions the model may generate JSON inputs + for. A max of 128 functions are supported. + + top_logprobs: An integer between 0 and 20 specifying the number of most likely tokens to + return at each token position, each with an associated log probability. + `logprobs` must be set to `true` if this parameter is used. + + top_p: An alternative to sampling with temperature, called nucleus sampling, where the + model considers the results of the tokens with top_p probability mass. So 0.1 + means only the tokens comprising the top 10% probability mass are considered. + + We generally recommend altering this or `temperature` but not both. + + user: A unique identifier representing your end-user, which can help OpenAI to monitor + and detect abuse. + [Learn more](https://platform.openai.com/docs/guides/safety-best-practices#end-user-ids). + + web_search_options: This tool searches the web for relevant results to use in a response. Learn more + about the + [web search tool](https://platform.openai.com/docs/guides/tools-web-search?api-mode=chat). + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + ... + + @overload + async def create( + self, + *, + messages: Iterable[ChatCompletionMessageParam], + model: Union[str, ChatModel], + stream: Literal[True], + audio: Optional[ChatCompletionAudioParam] | NotGiven = NOT_GIVEN, + frequency_penalty: Optional[float] | NotGiven = NOT_GIVEN, + function_call: completion_create_params.FunctionCall | NotGiven = NOT_GIVEN, + functions: Iterable[completion_create_params.Function] | NotGiven = NOT_GIVEN, + logit_bias: Optional[Dict[str, int]] | NotGiven = NOT_GIVEN, + logprobs: Optional[bool] | NotGiven = NOT_GIVEN, + max_completion_tokens: Optional[int] | NotGiven = NOT_GIVEN, + max_tokens: Optional[int] | NotGiven = NOT_GIVEN, + metadata: Optional[Metadata] | NotGiven = NOT_GIVEN, + modalities: Optional[List[Literal["text", "audio"]]] | NotGiven = NOT_GIVEN, + n: Optional[int] | NotGiven = NOT_GIVEN, + parallel_tool_calls: bool | NotGiven = NOT_GIVEN, + prediction: Optional[ChatCompletionPredictionContentParam] | NotGiven = NOT_GIVEN, + presence_penalty: Optional[float] | NotGiven = NOT_GIVEN, + reasoning_effort: Optional[ReasoningEffort] | NotGiven = NOT_GIVEN, + response_format: completion_create_params.ResponseFormat | NotGiven = NOT_GIVEN, + seed: Optional[int] | NotGiven = NOT_GIVEN, + service_tier: Optional[Literal["auto", "default", "flex"]] | NotGiven = NOT_GIVEN, + stop: Union[Optional[str], List[str], None] | NotGiven = NOT_GIVEN, + store: Optional[bool] | NotGiven = NOT_GIVEN, + stream_options: Optional[ChatCompletionStreamOptionsParam] | NotGiven = NOT_GIVEN, + temperature: Optional[float] | NotGiven = NOT_GIVEN, + tool_choice: ChatCompletionToolChoiceOptionParam | NotGiven = NOT_GIVEN, + tools: Iterable[ChatCompletionToolParam] | NotGiven = NOT_GIVEN, + top_logprobs: Optional[int] | NotGiven = NOT_GIVEN, + top_p: Optional[float] | NotGiven = NOT_GIVEN, + user: str | NotGiven = NOT_GIVEN, + web_search_options: completion_create_params.WebSearchOptions | NotGiven = NOT_GIVEN, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> AsyncStream[ChatCompletionChunk]: + """ + **Starting a new project?** We recommend trying + [Responses](https://platform.openai.com/docs/api-reference/responses) to take + advantage of the latest OpenAI platform features. Compare + [Chat Completions with Responses](https://platform.openai.com/docs/guides/responses-vs-chat-completions?api-mode=responses). + + --- + + Creates a model response for the given chat conversation. Learn more in the + [text generation](https://platform.openai.com/docs/guides/text-generation), + [vision](https://platform.openai.com/docs/guides/vision), and + [audio](https://platform.openai.com/docs/guides/audio) guides. + + Parameter support can differ depending on the model used to generate the + response, particularly for newer reasoning models. Parameters that are only + supported for reasoning models are noted below. For the current state of + unsupported parameters in reasoning models, + [refer to the reasoning guide](https://platform.openai.com/docs/guides/reasoning). + + Args: + messages: A list of messages comprising the conversation so far. Depending on the + [model](https://platform.openai.com/docs/models) you use, different message + types (modalities) are supported, like + [text](https://platform.openai.com/docs/guides/text-generation), + [images](https://platform.openai.com/docs/guides/vision), and + [audio](https://platform.openai.com/docs/guides/audio). + + model: Model ID used to generate the response, like `gpt-4o` or `o3`. OpenAI offers a + wide range of models with different capabilities, performance characteristics, + and price points. Refer to the + [model guide](https://platform.openai.com/docs/models) to browse and compare + available models. + + stream: If set to true, the model response data will be streamed to the client as it is + generated using + [server-sent events](https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events#Event_stream_format). + See the + [Streaming section below](https://platform.openai.com/docs/api-reference/chat/streaming) + for more information, along with the + [streaming responses](https://platform.openai.com/docs/guides/streaming-responses) + guide for more information on how to handle the streaming events. + + audio: Parameters for audio output. Required when audio output is requested with + `modalities: ["audio"]`. + [Learn more](https://platform.openai.com/docs/guides/audio). + + frequency_penalty: Number between -2.0 and 2.0. Positive values penalize new tokens based on their + existing frequency in the text so far, decreasing the model's likelihood to + repeat the same line verbatim. + + function_call: Deprecated in favor of `tool_choice`. + + Controls which (if any) function is called by the model. + + `none` means the model will not call a function and instead generates a message. + + `auto` means the model can pick between generating a message or calling a + function. + + Specifying a particular function via `{"name": "my_function"}` forces the model + to call that function. + + `none` is the default when no functions are present. `auto` is the default if + functions are present. + + functions: Deprecated in favor of `tools`. + + A list of functions the model may generate JSON inputs for. + + logit_bias: Modify the likelihood of specified tokens appearing in the completion. + + Accepts a JSON object that maps tokens (specified by their token ID in the + tokenizer) to an associated bias value from -100 to 100. Mathematically, the + bias is added to the logits generated by the model prior to sampling. The exact + effect will vary per model, but values between -1 and 1 should decrease or + increase likelihood of selection; values like -100 or 100 should result in a ban + or exclusive selection of the relevant token. + + logprobs: Whether to return log probabilities of the output tokens or not. If true, + returns the log probabilities of each output token returned in the `content` of + `message`. + + max_completion_tokens: An upper bound for the number of tokens that can be generated for a completion, + including visible output tokens and + [reasoning tokens](https://platform.openai.com/docs/guides/reasoning). + + max_tokens: The maximum number of [tokens](/tokenizer) that can be generated in the chat + completion. This value can be used to control + [costs](https://openai.com/api/pricing/) for text generated via API. + + This value is now deprecated in favor of `max_completion_tokens`, and is not + compatible with + [o-series models](https://platform.openai.com/docs/guides/reasoning). + + metadata: Set of 16 key-value pairs that can be attached to an object. This can be useful + for storing additional information about the object in a structured format, and + querying for objects via API or the dashboard. + + Keys are strings with a maximum length of 64 characters. Values are strings with + a maximum length of 512 characters. + + modalities: Output types that you would like the model to generate. Most models are capable + of generating text, which is the default: + + `["text"]` + + The `gpt-4o-audio-preview` model can also be used to + [generate audio](https://platform.openai.com/docs/guides/audio). To request that + this model generate both text and audio responses, you can use: + + `["text", "audio"]` + + n: How many chat completion choices to generate for each input message. Note that + you will be charged based on the number of generated tokens across all of the + choices. Keep `n` as `1` to minimize costs. + + parallel_tool_calls: Whether to enable + [parallel function calling](https://platform.openai.com/docs/guides/function-calling#configuring-parallel-function-calling) + during tool use. + + prediction: Static predicted output content, such as the content of a text file that is + being regenerated. + + presence_penalty: Number between -2.0 and 2.0. Positive values penalize new tokens based on + whether they appear in the text so far, increasing the model's likelihood to + talk about new topics. + + reasoning_effort: **o-series models only** + + Constrains effort on reasoning for + [reasoning models](https://platform.openai.com/docs/guides/reasoning). Currently + supported values are `low`, `medium`, and `high`. Reducing reasoning effort can + result in faster responses and fewer tokens used on reasoning in a response. + + response_format: An object specifying the format that the model must output. + + Setting to `{ "type": "json_schema", "json_schema": {...} }` enables Structured + Outputs which ensures the model will match your supplied JSON schema. Learn more + in the + [Structured Outputs guide](https://platform.openai.com/docs/guides/structured-outputs). + + Setting to `{ "type": "json_object" }` enables the older JSON mode, which + ensures the message the model generates is valid JSON. Using `json_schema` is + preferred for models that support it. + + seed: This feature is in Beta. If specified, our system will make a best effort to + sample deterministically, such that repeated requests with the same `seed` and + parameters should return the same result. Determinism is not guaranteed, and you + should refer to the `system_fingerprint` response parameter to monitor changes + in the backend. + + service_tier: Specifies the latency tier to use for processing the request. This parameter is + relevant for customers subscribed to the scale tier service: + + - If set to 'auto', and the Project is Scale tier enabled, the system will + utilize scale tier credits until they are exhausted. + - If set to 'auto', and the Project is not Scale tier enabled, the request will + be processed using the default service tier with a lower uptime SLA and no + latency guarentee. + - If set to 'default', the request will be processed using the default service + tier with a lower uptime SLA and no latency guarentee. + - If set to 'flex', the request will be processed with the Flex Processing + service tier. + [Learn more](https://platform.openai.com/docs/guides/flex-processing). + - When not set, the default behavior is 'auto'. + + When this parameter is set, the response body will include the `service_tier` + utilized. + + stop: Not supported with latest reasoning models `o3` and `o4-mini`. + + Up to 4 sequences where the API will stop generating further tokens. The + returned text will not contain the stop sequence. + + store: Whether or not to store the output of this chat completion request for use in + our [model distillation](https://platform.openai.com/docs/guides/distillation) + or [evals](https://platform.openai.com/docs/guides/evals) products. + + stream_options: Options for streaming response. Only set this when you set `stream: true`. + + temperature: What sampling temperature to use, between 0 and 2. Higher values like 0.8 will + make the output more random, while lower values like 0.2 will make it more + focused and deterministic. We generally recommend altering this or `top_p` but + not both. + + tool_choice: Controls which (if any) tool is called by the model. `none` means the model will + not call any tool and instead generates a message. `auto` means the model can + pick between generating a message or calling one or more tools. `required` means + the model must call one or more tools. Specifying a particular tool via + `{"type": "function", "function": {"name": "my_function"}}` forces the model to + call that tool. + + `none` is the default when no tools are present. `auto` is the default if tools + are present. + + tools: A list of tools the model may call. Currently, only functions are supported as a + tool. Use this to provide a list of functions the model may generate JSON inputs + for. A max of 128 functions are supported. + + top_logprobs: An integer between 0 and 20 specifying the number of most likely tokens to + return at each token position, each with an associated log probability. + `logprobs` must be set to `true` if this parameter is used. + + top_p: An alternative to sampling with temperature, called nucleus sampling, where the + model considers the results of the tokens with top_p probability mass. So 0.1 + means only the tokens comprising the top 10% probability mass are considered. + + We generally recommend altering this or `temperature` but not both. + + user: A unique identifier representing your end-user, which can help OpenAI to monitor + and detect abuse. + [Learn more](https://platform.openai.com/docs/guides/safety-best-practices#end-user-ids). + + web_search_options: This tool searches the web for relevant results to use in a response. Learn more + about the + [web search tool](https://platform.openai.com/docs/guides/tools-web-search?api-mode=chat). + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + ... + + @overload + async def create( + self, + *, + messages: Iterable[ChatCompletionMessageParam], + model: Union[str, ChatModel], + stream: bool, + audio: Optional[ChatCompletionAudioParam] | NotGiven = NOT_GIVEN, + frequency_penalty: Optional[float] | NotGiven = NOT_GIVEN, + function_call: completion_create_params.FunctionCall | NotGiven = NOT_GIVEN, + functions: Iterable[completion_create_params.Function] | NotGiven = NOT_GIVEN, + logit_bias: Optional[Dict[str, int]] | NotGiven = NOT_GIVEN, + logprobs: Optional[bool] | NotGiven = NOT_GIVEN, + max_completion_tokens: Optional[int] | NotGiven = NOT_GIVEN, + max_tokens: Optional[int] | NotGiven = NOT_GIVEN, + metadata: Optional[Metadata] | NotGiven = NOT_GIVEN, + modalities: Optional[List[Literal["text", "audio"]]] | NotGiven = NOT_GIVEN, + n: Optional[int] | NotGiven = NOT_GIVEN, + parallel_tool_calls: bool | NotGiven = NOT_GIVEN, + prediction: Optional[ChatCompletionPredictionContentParam] | NotGiven = NOT_GIVEN, + presence_penalty: Optional[float] | NotGiven = NOT_GIVEN, + reasoning_effort: Optional[ReasoningEffort] | NotGiven = NOT_GIVEN, + response_format: completion_create_params.ResponseFormat | NotGiven = NOT_GIVEN, + seed: Optional[int] | NotGiven = NOT_GIVEN, + service_tier: Optional[Literal["auto", "default", "flex"]] | NotGiven = NOT_GIVEN, + stop: Union[Optional[str], List[str], None] | NotGiven = NOT_GIVEN, + store: Optional[bool] | NotGiven = NOT_GIVEN, + stream_options: Optional[ChatCompletionStreamOptionsParam] | NotGiven = NOT_GIVEN, + temperature: Optional[float] | NotGiven = NOT_GIVEN, + tool_choice: ChatCompletionToolChoiceOptionParam | NotGiven = NOT_GIVEN, + tools: Iterable[ChatCompletionToolParam] | NotGiven = NOT_GIVEN, + top_logprobs: Optional[int] | NotGiven = NOT_GIVEN, + top_p: Optional[float] | NotGiven = NOT_GIVEN, + user: str | NotGiven = NOT_GIVEN, + web_search_options: completion_create_params.WebSearchOptions | NotGiven = NOT_GIVEN, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> ChatCompletion | AsyncStream[ChatCompletionChunk]: + """ + **Starting a new project?** We recommend trying + [Responses](https://platform.openai.com/docs/api-reference/responses) to take + advantage of the latest OpenAI platform features. Compare + [Chat Completions with Responses](https://platform.openai.com/docs/guides/responses-vs-chat-completions?api-mode=responses). + + --- + + Creates a model response for the given chat conversation. Learn more in the + [text generation](https://platform.openai.com/docs/guides/text-generation), + [vision](https://platform.openai.com/docs/guides/vision), and + [audio](https://platform.openai.com/docs/guides/audio) guides. + + Parameter support can differ depending on the model used to generate the + response, particularly for newer reasoning models. Parameters that are only + supported for reasoning models are noted below. For the current state of + unsupported parameters in reasoning models, + [refer to the reasoning guide](https://platform.openai.com/docs/guides/reasoning). + + Args: + messages: A list of messages comprising the conversation so far. Depending on the + [model](https://platform.openai.com/docs/models) you use, different message + types (modalities) are supported, like + [text](https://platform.openai.com/docs/guides/text-generation), + [images](https://platform.openai.com/docs/guides/vision), and + [audio](https://platform.openai.com/docs/guides/audio). + + model: Model ID used to generate the response, like `gpt-4o` or `o3`. OpenAI offers a + wide range of models with different capabilities, performance characteristics, + and price points. Refer to the + [model guide](https://platform.openai.com/docs/models) to browse and compare + available models. + + stream: If set to true, the model response data will be streamed to the client as it is + generated using + [server-sent events](https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events#Event_stream_format). + See the + [Streaming section below](https://platform.openai.com/docs/api-reference/chat/streaming) + for more information, along with the + [streaming responses](https://platform.openai.com/docs/guides/streaming-responses) + guide for more information on how to handle the streaming events. + + audio: Parameters for audio output. Required when audio output is requested with + `modalities: ["audio"]`. + [Learn more](https://platform.openai.com/docs/guides/audio). + + frequency_penalty: Number between -2.0 and 2.0. Positive values penalize new tokens based on their + existing frequency in the text so far, decreasing the model's likelihood to + repeat the same line verbatim. + + function_call: Deprecated in favor of `tool_choice`. + + Controls which (if any) function is called by the model. + + `none` means the model will not call a function and instead generates a message. + + `auto` means the model can pick between generating a message or calling a + function. + + Specifying a particular function via `{"name": "my_function"}` forces the model + to call that function. + + `none` is the default when no functions are present. `auto` is the default if + functions are present. + + functions: Deprecated in favor of `tools`. + + A list of functions the model may generate JSON inputs for. + + logit_bias: Modify the likelihood of specified tokens appearing in the completion. + + Accepts a JSON object that maps tokens (specified by their token ID in the + tokenizer) to an associated bias value from -100 to 100. Mathematically, the + bias is added to the logits generated by the model prior to sampling. The exact + effect will vary per model, but values between -1 and 1 should decrease or + increase likelihood of selection; values like -100 or 100 should result in a ban + or exclusive selection of the relevant token. + + logprobs: Whether to return log probabilities of the output tokens or not. If true, + returns the log probabilities of each output token returned in the `content` of + `message`. + + max_completion_tokens: An upper bound for the number of tokens that can be generated for a completion, + including visible output tokens and + [reasoning tokens](https://platform.openai.com/docs/guides/reasoning). + + max_tokens: The maximum number of [tokens](/tokenizer) that can be generated in the chat + completion. This value can be used to control + [costs](https://openai.com/api/pricing/) for text generated via API. + + This value is now deprecated in favor of `max_completion_tokens`, and is not + compatible with + [o-series models](https://platform.openai.com/docs/guides/reasoning). + + metadata: Set of 16 key-value pairs that can be attached to an object. This can be useful + for storing additional information about the object in a structured format, and + querying for objects via API or the dashboard. + + Keys are strings with a maximum length of 64 characters. Values are strings with + a maximum length of 512 characters. + + modalities: Output types that you would like the model to generate. Most models are capable + of generating text, which is the default: + + `["text"]` + + The `gpt-4o-audio-preview` model can also be used to + [generate audio](https://platform.openai.com/docs/guides/audio). To request that + this model generate both text and audio responses, you can use: + + `["text", "audio"]` + + n: How many chat completion choices to generate for each input message. Note that + you will be charged based on the number of generated tokens across all of the + choices. Keep `n` as `1` to minimize costs. + + parallel_tool_calls: Whether to enable + [parallel function calling](https://platform.openai.com/docs/guides/function-calling#configuring-parallel-function-calling) + during tool use. + + prediction: Static predicted output content, such as the content of a text file that is + being regenerated. + + presence_penalty: Number between -2.0 and 2.0. Positive values penalize new tokens based on + whether they appear in the text so far, increasing the model's likelihood to + talk about new topics. + + reasoning_effort: **o-series models only** + + Constrains effort on reasoning for + [reasoning models](https://platform.openai.com/docs/guides/reasoning). Currently + supported values are `low`, `medium`, and `high`. Reducing reasoning effort can + result in faster responses and fewer tokens used on reasoning in a response. + + response_format: An object specifying the format that the model must output. + + Setting to `{ "type": "json_schema", "json_schema": {...} }` enables Structured + Outputs which ensures the model will match your supplied JSON schema. Learn more + in the + [Structured Outputs guide](https://platform.openai.com/docs/guides/structured-outputs). + + Setting to `{ "type": "json_object" }` enables the older JSON mode, which + ensures the message the model generates is valid JSON. Using `json_schema` is + preferred for models that support it. + + seed: This feature is in Beta. If specified, our system will make a best effort to + sample deterministically, such that repeated requests with the same `seed` and + parameters should return the same result. Determinism is not guaranteed, and you + should refer to the `system_fingerprint` response parameter to monitor changes + in the backend. + + service_tier: Specifies the latency tier to use for processing the request. This parameter is + relevant for customers subscribed to the scale tier service: + + - If set to 'auto', and the Project is Scale tier enabled, the system will + utilize scale tier credits until they are exhausted. + - If set to 'auto', and the Project is not Scale tier enabled, the request will + be processed using the default service tier with a lower uptime SLA and no + latency guarentee. + - If set to 'default', the request will be processed using the default service + tier with a lower uptime SLA and no latency guarentee. + - If set to 'flex', the request will be processed with the Flex Processing + service tier. + [Learn more](https://platform.openai.com/docs/guides/flex-processing). + - When not set, the default behavior is 'auto'. + + When this parameter is set, the response body will include the `service_tier` + utilized. + + stop: Not supported with latest reasoning models `o3` and `o4-mini`. + + Up to 4 sequences where the API will stop generating further tokens. The + returned text will not contain the stop sequence. + + store: Whether or not to store the output of this chat completion request for use in + our [model distillation](https://platform.openai.com/docs/guides/distillation) + or [evals](https://platform.openai.com/docs/guides/evals) products. + + stream_options: Options for streaming response. Only set this when you set `stream: true`. + + temperature: What sampling temperature to use, between 0 and 2. Higher values like 0.8 will + make the output more random, while lower values like 0.2 will make it more + focused and deterministic. We generally recommend altering this or `top_p` but + not both. + + tool_choice: Controls which (if any) tool is called by the model. `none` means the model will + not call any tool and instead generates a message. `auto` means the model can + pick between generating a message or calling one or more tools. `required` means + the model must call one or more tools. Specifying a particular tool via + `{"type": "function", "function": {"name": "my_function"}}` forces the model to + call that tool. + + `none` is the default when no tools are present. `auto` is the default if tools + are present. + + tools: A list of tools the model may call. Currently, only functions are supported as a + tool. Use this to provide a list of functions the model may generate JSON inputs + for. A max of 128 functions are supported. + + top_logprobs: An integer between 0 and 20 specifying the number of most likely tokens to + return at each token position, each with an associated log probability. + `logprobs` must be set to `true` if this parameter is used. + + top_p: An alternative to sampling with temperature, called nucleus sampling, where the + model considers the results of the tokens with top_p probability mass. So 0.1 + means only the tokens comprising the top 10% probability mass are considered. + + We generally recommend altering this or `temperature` but not both. + + user: A unique identifier representing your end-user, which can help OpenAI to monitor + and detect abuse. + [Learn more](https://platform.openai.com/docs/guides/safety-best-practices#end-user-ids). + + web_search_options: This tool searches the web for relevant results to use in a response. Learn more + about the + [web search tool](https://platform.openai.com/docs/guides/tools-web-search?api-mode=chat). + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + ... + + @required_args(["messages", "model"], ["messages", "model", "stream"]) + async def create( + self, + *, + messages: Iterable[ChatCompletionMessageParam], + model: Union[str, ChatModel], + audio: Optional[ChatCompletionAudioParam] | NotGiven = NOT_GIVEN, + frequency_penalty: Optional[float] | NotGiven = NOT_GIVEN, + function_call: completion_create_params.FunctionCall | NotGiven = NOT_GIVEN, + functions: Iterable[completion_create_params.Function] | NotGiven = NOT_GIVEN, + logit_bias: Optional[Dict[str, int]] | NotGiven = NOT_GIVEN, + logprobs: Optional[bool] | NotGiven = NOT_GIVEN, + max_completion_tokens: Optional[int] | NotGiven = NOT_GIVEN, + max_tokens: Optional[int] | NotGiven = NOT_GIVEN, + metadata: Optional[Metadata] | NotGiven = NOT_GIVEN, + modalities: Optional[List[Literal["text", "audio"]]] | NotGiven = NOT_GIVEN, + n: Optional[int] | NotGiven = NOT_GIVEN, + parallel_tool_calls: bool | NotGiven = NOT_GIVEN, + prediction: Optional[ChatCompletionPredictionContentParam] | NotGiven = NOT_GIVEN, + presence_penalty: Optional[float] | NotGiven = NOT_GIVEN, + reasoning_effort: Optional[ReasoningEffort] | NotGiven = NOT_GIVEN, + response_format: completion_create_params.ResponseFormat | NotGiven = NOT_GIVEN, + seed: Optional[int] | NotGiven = NOT_GIVEN, + service_tier: Optional[Literal["auto", "default", "flex"]] | NotGiven = NOT_GIVEN, + stop: Union[Optional[str], List[str], None] | NotGiven = NOT_GIVEN, + store: Optional[bool] | NotGiven = NOT_GIVEN, + stream: Optional[Literal[False]] | Literal[True] | NotGiven = NOT_GIVEN, + stream_options: Optional[ChatCompletionStreamOptionsParam] | NotGiven = NOT_GIVEN, + temperature: Optional[float] | NotGiven = NOT_GIVEN, + tool_choice: ChatCompletionToolChoiceOptionParam | NotGiven = NOT_GIVEN, + tools: Iterable[ChatCompletionToolParam] | NotGiven = NOT_GIVEN, + top_logprobs: Optional[int] | NotGiven = NOT_GIVEN, + top_p: Optional[float] | NotGiven = NOT_GIVEN, + user: str | NotGiven = NOT_GIVEN, + web_search_options: completion_create_params.WebSearchOptions | NotGiven = NOT_GIVEN, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> ChatCompletion | AsyncStream[ChatCompletionChunk]: + validate_response_format(response_format) + return await self._post( + "/chat/completions", + body=await async_maybe_transform( + { + "messages": messages, + "model": model, + "audio": audio, + "frequency_penalty": frequency_penalty, + "function_call": function_call, + "functions": functions, + "logit_bias": logit_bias, + "logprobs": logprobs, + "max_completion_tokens": max_completion_tokens, + "max_tokens": max_tokens, + "metadata": metadata, + "modalities": modalities, + "n": n, + "parallel_tool_calls": parallel_tool_calls, + "prediction": prediction, + "presence_penalty": presence_penalty, + "reasoning_effort": reasoning_effort, + "response_format": response_format, + "seed": seed, + "service_tier": service_tier, + "stop": stop, + "store": store, + "stream": stream, + "stream_options": stream_options, + "temperature": temperature, + "tool_choice": tool_choice, + "tools": tools, + "top_logprobs": top_logprobs, + "top_p": top_p, + "user": user, + "web_search_options": web_search_options, + }, + completion_create_params.CompletionCreateParamsStreaming + if stream + else completion_create_params.CompletionCreateParamsNonStreaming, + ), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=ChatCompletion, + stream=stream or False, + stream_cls=AsyncStream[ChatCompletionChunk], + ) + + async def retrieve( + self, + completion_id: str, + *, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> ChatCompletion: + """Get a stored chat completion. + + Only Chat Completions that have been created with + the `store` parameter set to `true` will be returned. + + Args: + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if not completion_id: + raise ValueError(f"Expected a non-empty value for `completion_id` but received {completion_id!r}") + return await self._get( + f"/chat/completions/{completion_id}", + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=ChatCompletion, + ) + + async def update( + self, + completion_id: str, + *, + metadata: Optional[Metadata], + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> ChatCompletion: + """Modify a stored chat completion. + + Only Chat Completions that have been created + with the `store` parameter set to `true` can be modified. Currently, the only + supported modification is to update the `metadata` field. + + Args: + metadata: Set of 16 key-value pairs that can be attached to an object. This can be useful + for storing additional information about the object in a structured format, and + querying for objects via API or the dashboard. + + Keys are strings with a maximum length of 64 characters. Values are strings with + a maximum length of 512 characters. + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if not completion_id: + raise ValueError(f"Expected a non-empty value for `completion_id` but received {completion_id!r}") + return await self._post( + f"/chat/completions/{completion_id}", + body=await async_maybe_transform({"metadata": metadata}, completion_update_params.CompletionUpdateParams), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=ChatCompletion, + ) + + def list( + self, + *, + after: str | NotGiven = NOT_GIVEN, + limit: int | NotGiven = NOT_GIVEN, + metadata: Optional[Metadata] | NotGiven = NOT_GIVEN, + model: str | NotGiven = NOT_GIVEN, + order: Literal["asc", "desc"] | NotGiven = NOT_GIVEN, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> AsyncPaginator[ChatCompletion, AsyncCursorPage[ChatCompletion]]: + """List stored Chat Completions. + + Only Chat Completions that have been stored with + the `store` parameter set to `true` will be returned. + + Args: + after: Identifier for the last chat completion from the previous pagination request. + + limit: Number of Chat Completions to retrieve. + + metadata: + A list of metadata keys to filter the Chat Completions by. Example: + + `metadata[key1]=value1&metadata[key2]=value2` + + model: The model used to generate the Chat Completions. + + order: Sort order for Chat Completions by timestamp. Use `asc` for ascending order or + `desc` for descending order. Defaults to `asc`. + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + return self._get_api_list( + "/chat/completions", + page=AsyncCursorPage[ChatCompletion], + options=make_request_options( + extra_headers=extra_headers, + extra_query=extra_query, + extra_body=extra_body, + timeout=timeout, + query=maybe_transform( + { + "after": after, + "limit": limit, + "metadata": metadata, + "model": model, + "order": order, + }, + completion_list_params.CompletionListParams, + ), + ), + model=ChatCompletion, + ) + + async def delete( + self, + completion_id: str, + *, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> ChatCompletionDeleted: + """Delete a stored chat completion. + + Only Chat Completions that have been created + with the `store` parameter set to `true` can be deleted. + + Args: + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if not completion_id: + raise ValueError(f"Expected a non-empty value for `completion_id` but received {completion_id!r}") + return await self._delete( + f"/chat/completions/{completion_id}", + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=ChatCompletionDeleted, + ) + + +class CompletionsWithRawResponse: + def __init__(self, completions: Completions) -> None: + self._completions = completions + + self.create = _legacy_response.to_raw_response_wrapper( + completions.create, + ) + self.retrieve = _legacy_response.to_raw_response_wrapper( + completions.retrieve, + ) + self.update = _legacy_response.to_raw_response_wrapper( + completions.update, + ) + self.list = _legacy_response.to_raw_response_wrapper( + completions.list, + ) + self.delete = _legacy_response.to_raw_response_wrapper( + completions.delete, + ) + + @cached_property + def messages(self) -> MessagesWithRawResponse: + return MessagesWithRawResponse(self._completions.messages) + + +class AsyncCompletionsWithRawResponse: + def __init__(self, completions: AsyncCompletions) -> None: + self._completions = completions + + self.create = _legacy_response.async_to_raw_response_wrapper( + completions.create, + ) + self.retrieve = _legacy_response.async_to_raw_response_wrapper( + completions.retrieve, + ) + self.update = _legacy_response.async_to_raw_response_wrapper( + completions.update, + ) + self.list = _legacy_response.async_to_raw_response_wrapper( + completions.list, + ) + self.delete = _legacy_response.async_to_raw_response_wrapper( + completions.delete, + ) + + @cached_property + def messages(self) -> AsyncMessagesWithRawResponse: + return AsyncMessagesWithRawResponse(self._completions.messages) + + +class CompletionsWithStreamingResponse: + def __init__(self, completions: Completions) -> None: + self._completions = completions + + self.create = to_streamed_response_wrapper( + completions.create, + ) + self.retrieve = to_streamed_response_wrapper( + completions.retrieve, + ) + self.update = to_streamed_response_wrapper( + completions.update, + ) + self.list = to_streamed_response_wrapper( + completions.list, + ) + self.delete = to_streamed_response_wrapper( + completions.delete, + ) + + @cached_property + def messages(self) -> MessagesWithStreamingResponse: + return MessagesWithStreamingResponse(self._completions.messages) + + +class AsyncCompletionsWithStreamingResponse: + def __init__(self, completions: AsyncCompletions) -> None: + self._completions = completions + + self.create = async_to_streamed_response_wrapper( + completions.create, + ) + self.retrieve = async_to_streamed_response_wrapper( + completions.retrieve, + ) + self.update = async_to_streamed_response_wrapper( + completions.update, + ) + self.list = async_to_streamed_response_wrapper( + completions.list, + ) + self.delete = async_to_streamed_response_wrapper( + completions.delete, + ) + + @cached_property + def messages(self) -> AsyncMessagesWithStreamingResponse: + return AsyncMessagesWithStreamingResponse(self._completions.messages) + + +def validate_response_format(response_format: object) -> None: + if inspect.isclass(response_format) and issubclass(response_format, pydantic.BaseModel): + raise TypeError( + "You tried to pass a `BaseModel` class to `chat.completions.create()`; You must use `beta.chat.completions.parse()` instead" + ) diff --git a/lib/python3.12/site-packages/openai/resources/chat/completions/messages.py b/lib/python3.12/site-packages/openai/resources/chat/completions/messages.py new file mode 100644 index 0000000000000000000000000000000000000000..fac15fba8b8b13d61f263c991ea3accfad07e8b2 --- /dev/null +++ b/lib/python3.12/site-packages/openai/resources/chat/completions/messages.py @@ -0,0 +1,212 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing_extensions import Literal + +import httpx + +from .... import _legacy_response +from ...._types import NOT_GIVEN, Body, Query, Headers, NotGiven +from ...._utils import maybe_transform +from ...._compat import cached_property +from ...._resource import SyncAPIResource, AsyncAPIResource +from ...._response import to_streamed_response_wrapper, async_to_streamed_response_wrapper +from ....pagination import SyncCursorPage, AsyncCursorPage +from ...._base_client import AsyncPaginator, make_request_options +from ....types.chat.completions import message_list_params +from ....types.chat.chat_completion_store_message import ChatCompletionStoreMessage + +__all__ = ["Messages", "AsyncMessages"] + + +class Messages(SyncAPIResource): + @cached_property + def with_raw_response(self) -> MessagesWithRawResponse: + """ + This property can be used as a prefix for any HTTP method call to return + the raw response object instead of the parsed content. + + For more information, see https://www.github.com/openai/openai-python#accessing-raw-response-data-eg-headers + """ + return MessagesWithRawResponse(self) + + @cached_property + def with_streaming_response(self) -> MessagesWithStreamingResponse: + """ + An alternative to `.with_raw_response` that doesn't eagerly read the response body. + + For more information, see https://www.github.com/openai/openai-python#with_streaming_response + """ + return MessagesWithStreamingResponse(self) + + def list( + self, + completion_id: str, + *, + after: str | NotGiven = NOT_GIVEN, + limit: int | NotGiven = NOT_GIVEN, + order: Literal["asc", "desc"] | NotGiven = NOT_GIVEN, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> SyncCursorPage[ChatCompletionStoreMessage]: + """Get the messages in a stored chat completion. + + Only Chat Completions that have + been created with the `store` parameter set to `true` will be returned. + + Args: + after: Identifier for the last message from the previous pagination request. + + limit: Number of messages to retrieve. + + order: Sort order for messages by timestamp. Use `asc` for ascending order or `desc` + for descending order. Defaults to `asc`. + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if not completion_id: + raise ValueError(f"Expected a non-empty value for `completion_id` but received {completion_id!r}") + return self._get_api_list( + f"/chat/completions/{completion_id}/messages", + page=SyncCursorPage[ChatCompletionStoreMessage], + options=make_request_options( + extra_headers=extra_headers, + extra_query=extra_query, + extra_body=extra_body, + timeout=timeout, + query=maybe_transform( + { + "after": after, + "limit": limit, + "order": order, + }, + message_list_params.MessageListParams, + ), + ), + model=ChatCompletionStoreMessage, + ) + + +class AsyncMessages(AsyncAPIResource): + @cached_property + def with_raw_response(self) -> AsyncMessagesWithRawResponse: + """ + This property can be used as a prefix for any HTTP method call to return + the raw response object instead of the parsed content. + + For more information, see https://www.github.com/openai/openai-python#accessing-raw-response-data-eg-headers + """ + return AsyncMessagesWithRawResponse(self) + + @cached_property + def with_streaming_response(self) -> AsyncMessagesWithStreamingResponse: + """ + An alternative to `.with_raw_response` that doesn't eagerly read the response body. + + For more information, see https://www.github.com/openai/openai-python#with_streaming_response + """ + return AsyncMessagesWithStreamingResponse(self) + + def list( + self, + completion_id: str, + *, + after: str | NotGiven = NOT_GIVEN, + limit: int | NotGiven = NOT_GIVEN, + order: Literal["asc", "desc"] | NotGiven = NOT_GIVEN, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> AsyncPaginator[ChatCompletionStoreMessage, AsyncCursorPage[ChatCompletionStoreMessage]]: + """Get the messages in a stored chat completion. + + Only Chat Completions that have + been created with the `store` parameter set to `true` will be returned. + + Args: + after: Identifier for the last message from the previous pagination request. + + limit: Number of messages to retrieve. + + order: Sort order for messages by timestamp. Use `asc` for ascending order or `desc` + for descending order. Defaults to `asc`. + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if not completion_id: + raise ValueError(f"Expected a non-empty value for `completion_id` but received {completion_id!r}") + return self._get_api_list( + f"/chat/completions/{completion_id}/messages", + page=AsyncCursorPage[ChatCompletionStoreMessage], + options=make_request_options( + extra_headers=extra_headers, + extra_query=extra_query, + extra_body=extra_body, + timeout=timeout, + query=maybe_transform( + { + "after": after, + "limit": limit, + "order": order, + }, + message_list_params.MessageListParams, + ), + ), + model=ChatCompletionStoreMessage, + ) + + +class MessagesWithRawResponse: + def __init__(self, messages: Messages) -> None: + self._messages = messages + + self.list = _legacy_response.to_raw_response_wrapper( + messages.list, + ) + + +class AsyncMessagesWithRawResponse: + def __init__(self, messages: AsyncMessages) -> None: + self._messages = messages + + self.list = _legacy_response.async_to_raw_response_wrapper( + messages.list, + ) + + +class MessagesWithStreamingResponse: + def __init__(self, messages: Messages) -> None: + self._messages = messages + + self.list = to_streamed_response_wrapper( + messages.list, + ) + + +class AsyncMessagesWithStreamingResponse: + def __init__(self, messages: AsyncMessages) -> None: + self._messages = messages + + self.list = async_to_streamed_response_wrapper( + messages.list, + ) diff --git a/lib/python3.12/site-packages/openai/resources/evals/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/openai/resources/evals/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ac7595004a25032395279acdf8af34344863522c Binary files /dev/null and b/lib/python3.12/site-packages/openai/resources/evals/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/resources/evals/__pycache__/evals.cpython-312.pyc b/lib/python3.12/site-packages/openai/resources/evals/__pycache__/evals.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8fe7d6e21f12271d67c7c1116d820eb3109b1810 Binary files /dev/null and b/lib/python3.12/site-packages/openai/resources/evals/__pycache__/evals.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/resources/evals/runs/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/openai/resources/evals/runs/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f648b7924381433ce27ead08f0b866bec129e7af Binary files /dev/null and b/lib/python3.12/site-packages/openai/resources/evals/runs/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/resources/evals/runs/__pycache__/runs.cpython-312.pyc b/lib/python3.12/site-packages/openai/resources/evals/runs/__pycache__/runs.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..697fbcdaa67724ef0f933a6e147287f1338c37c6 Binary files /dev/null and b/lib/python3.12/site-packages/openai/resources/evals/runs/__pycache__/runs.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/resources/fine_tuning/__init__.py b/lib/python3.12/site-packages/openai/resources/fine_tuning/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ed7db4f4e03c8a3ab4c5adcf162b12d65c013b99 --- /dev/null +++ b/lib/python3.12/site-packages/openai/resources/fine_tuning/__init__.py @@ -0,0 +1,47 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from .jobs import ( + Jobs, + AsyncJobs, + JobsWithRawResponse, + AsyncJobsWithRawResponse, + JobsWithStreamingResponse, + AsyncJobsWithStreamingResponse, +) +from .checkpoints import ( + Checkpoints, + AsyncCheckpoints, + CheckpointsWithRawResponse, + AsyncCheckpointsWithRawResponse, + CheckpointsWithStreamingResponse, + AsyncCheckpointsWithStreamingResponse, +) +from .fine_tuning import ( + FineTuning, + AsyncFineTuning, + FineTuningWithRawResponse, + AsyncFineTuningWithRawResponse, + FineTuningWithStreamingResponse, + AsyncFineTuningWithStreamingResponse, +) + +__all__ = [ + "Jobs", + "AsyncJobs", + "JobsWithRawResponse", + "AsyncJobsWithRawResponse", + "JobsWithStreamingResponse", + "AsyncJobsWithStreamingResponse", + "Checkpoints", + "AsyncCheckpoints", + "CheckpointsWithRawResponse", + "AsyncCheckpointsWithRawResponse", + "CheckpointsWithStreamingResponse", + "AsyncCheckpointsWithStreamingResponse", + "FineTuning", + "AsyncFineTuning", + "FineTuningWithRawResponse", + "AsyncFineTuningWithRawResponse", + "FineTuningWithStreamingResponse", + "AsyncFineTuningWithStreamingResponse", +] diff --git a/lib/python3.12/site-packages/openai/resources/fine_tuning/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/openai/resources/fine_tuning/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f4d5a89d4c3e6bef52f3775179572c0c2d309ab5 Binary files /dev/null and b/lib/python3.12/site-packages/openai/resources/fine_tuning/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/resources/fine_tuning/__pycache__/fine_tuning.cpython-312.pyc b/lib/python3.12/site-packages/openai/resources/fine_tuning/__pycache__/fine_tuning.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..22598bbdf3122558283bc3ad1e244f21edd3b33e Binary files /dev/null and b/lib/python3.12/site-packages/openai/resources/fine_tuning/__pycache__/fine_tuning.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/resources/fine_tuning/checkpoints/__init__.py b/lib/python3.12/site-packages/openai/resources/fine_tuning/checkpoints/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..fdc37940f98faf429968347ecfaa54a70e02407a --- /dev/null +++ b/lib/python3.12/site-packages/openai/resources/fine_tuning/checkpoints/__init__.py @@ -0,0 +1,33 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from .checkpoints import ( + Checkpoints, + AsyncCheckpoints, + CheckpointsWithRawResponse, + AsyncCheckpointsWithRawResponse, + CheckpointsWithStreamingResponse, + AsyncCheckpointsWithStreamingResponse, +) +from .permissions import ( + Permissions, + AsyncPermissions, + PermissionsWithRawResponse, + AsyncPermissionsWithRawResponse, + PermissionsWithStreamingResponse, + AsyncPermissionsWithStreamingResponse, +) + +__all__ = [ + "Permissions", + "AsyncPermissions", + "PermissionsWithRawResponse", + "AsyncPermissionsWithRawResponse", + "PermissionsWithStreamingResponse", + "AsyncPermissionsWithStreamingResponse", + "Checkpoints", + "AsyncCheckpoints", + "CheckpointsWithRawResponse", + "AsyncCheckpointsWithRawResponse", + "CheckpointsWithStreamingResponse", + "AsyncCheckpointsWithStreamingResponse", +] diff --git a/lib/python3.12/site-packages/openai/resources/fine_tuning/checkpoints/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/openai/resources/fine_tuning/checkpoints/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d015e392860912cd7789677fa86bf8a17a1086fc Binary files /dev/null and b/lib/python3.12/site-packages/openai/resources/fine_tuning/checkpoints/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/resources/fine_tuning/checkpoints/__pycache__/checkpoints.cpython-312.pyc b/lib/python3.12/site-packages/openai/resources/fine_tuning/checkpoints/__pycache__/checkpoints.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c6d3c9441a091cbdd6e62ac9505fc031308b8dda Binary files /dev/null and b/lib/python3.12/site-packages/openai/resources/fine_tuning/checkpoints/__pycache__/checkpoints.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/resources/fine_tuning/checkpoints/__pycache__/permissions.cpython-312.pyc b/lib/python3.12/site-packages/openai/resources/fine_tuning/checkpoints/__pycache__/permissions.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..913377ead6dd23387f981e20f6761f0b2276aa6d Binary files /dev/null and b/lib/python3.12/site-packages/openai/resources/fine_tuning/checkpoints/__pycache__/permissions.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/resources/fine_tuning/checkpoints/checkpoints.py b/lib/python3.12/site-packages/openai/resources/fine_tuning/checkpoints/checkpoints.py new file mode 100644 index 0000000000000000000000000000000000000000..f59976a2644b085352754cf1bbff3da47c909bb8 --- /dev/null +++ b/lib/python3.12/site-packages/openai/resources/fine_tuning/checkpoints/checkpoints.py @@ -0,0 +1,102 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from ...._compat import cached_property +from .permissions import ( + Permissions, + AsyncPermissions, + PermissionsWithRawResponse, + AsyncPermissionsWithRawResponse, + PermissionsWithStreamingResponse, + AsyncPermissionsWithStreamingResponse, +) +from ...._resource import SyncAPIResource, AsyncAPIResource + +__all__ = ["Checkpoints", "AsyncCheckpoints"] + + +class Checkpoints(SyncAPIResource): + @cached_property + def permissions(self) -> Permissions: + return Permissions(self._client) + + @cached_property + def with_raw_response(self) -> CheckpointsWithRawResponse: + """ + This property can be used as a prefix for any HTTP method call to return + the raw response object instead of the parsed content. + + For more information, see https://www.github.com/openai/openai-python#accessing-raw-response-data-eg-headers + """ + return CheckpointsWithRawResponse(self) + + @cached_property + def with_streaming_response(self) -> CheckpointsWithStreamingResponse: + """ + An alternative to `.with_raw_response` that doesn't eagerly read the response body. + + For more information, see https://www.github.com/openai/openai-python#with_streaming_response + """ + return CheckpointsWithStreamingResponse(self) + + +class AsyncCheckpoints(AsyncAPIResource): + @cached_property + def permissions(self) -> AsyncPermissions: + return AsyncPermissions(self._client) + + @cached_property + def with_raw_response(self) -> AsyncCheckpointsWithRawResponse: + """ + This property can be used as a prefix for any HTTP method call to return + the raw response object instead of the parsed content. + + For more information, see https://www.github.com/openai/openai-python#accessing-raw-response-data-eg-headers + """ + return AsyncCheckpointsWithRawResponse(self) + + @cached_property + def with_streaming_response(self) -> AsyncCheckpointsWithStreamingResponse: + """ + An alternative to `.with_raw_response` that doesn't eagerly read the response body. + + For more information, see https://www.github.com/openai/openai-python#with_streaming_response + """ + return AsyncCheckpointsWithStreamingResponse(self) + + +class CheckpointsWithRawResponse: + def __init__(self, checkpoints: Checkpoints) -> None: + self._checkpoints = checkpoints + + @cached_property + def permissions(self) -> PermissionsWithRawResponse: + return PermissionsWithRawResponse(self._checkpoints.permissions) + + +class AsyncCheckpointsWithRawResponse: + def __init__(self, checkpoints: AsyncCheckpoints) -> None: + self._checkpoints = checkpoints + + @cached_property + def permissions(self) -> AsyncPermissionsWithRawResponse: + return AsyncPermissionsWithRawResponse(self._checkpoints.permissions) + + +class CheckpointsWithStreamingResponse: + def __init__(self, checkpoints: Checkpoints) -> None: + self._checkpoints = checkpoints + + @cached_property + def permissions(self) -> PermissionsWithStreamingResponse: + return PermissionsWithStreamingResponse(self._checkpoints.permissions) + + +class AsyncCheckpointsWithStreamingResponse: + def __init__(self, checkpoints: AsyncCheckpoints) -> None: + self._checkpoints = checkpoints + + @cached_property + def permissions(self) -> AsyncPermissionsWithStreamingResponse: + return AsyncPermissionsWithStreamingResponse(self._checkpoints.permissions) diff --git a/lib/python3.12/site-packages/openai/resources/fine_tuning/checkpoints/permissions.py b/lib/python3.12/site-packages/openai/resources/fine_tuning/checkpoints/permissions.py new file mode 100644 index 0000000000000000000000000000000000000000..beb7b099d3bac3179a48eee054c17bb6a2efde73 --- /dev/null +++ b/lib/python3.12/site-packages/openai/resources/fine_tuning/checkpoints/permissions.py @@ -0,0 +1,416 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing import List +from typing_extensions import Literal + +import httpx + +from .... import _legacy_response +from ...._types import NOT_GIVEN, Body, Query, Headers, NotGiven +from ...._utils import ( + maybe_transform, + async_maybe_transform, +) +from ...._compat import cached_property +from ...._resource import SyncAPIResource, AsyncAPIResource +from ...._response import to_streamed_response_wrapper, async_to_streamed_response_wrapper +from ....pagination import SyncPage, AsyncPage +from ...._base_client import AsyncPaginator, make_request_options +from ....types.fine_tuning.checkpoints import permission_create_params, permission_retrieve_params +from ....types.fine_tuning.checkpoints.permission_create_response import PermissionCreateResponse +from ....types.fine_tuning.checkpoints.permission_delete_response import PermissionDeleteResponse +from ....types.fine_tuning.checkpoints.permission_retrieve_response import PermissionRetrieveResponse + +__all__ = ["Permissions", "AsyncPermissions"] + + +class Permissions(SyncAPIResource): + @cached_property + def with_raw_response(self) -> PermissionsWithRawResponse: + """ + This property can be used as a prefix for any HTTP method call to return + the raw response object instead of the parsed content. + + For more information, see https://www.github.com/openai/openai-python#accessing-raw-response-data-eg-headers + """ + return PermissionsWithRawResponse(self) + + @cached_property + def with_streaming_response(self) -> PermissionsWithStreamingResponse: + """ + An alternative to `.with_raw_response` that doesn't eagerly read the response body. + + For more information, see https://www.github.com/openai/openai-python#with_streaming_response + """ + return PermissionsWithStreamingResponse(self) + + def create( + self, + fine_tuned_model_checkpoint: str, + *, + project_ids: List[str], + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> SyncPage[PermissionCreateResponse]: + """ + **NOTE:** Calling this endpoint requires an [admin API key](../admin-api-keys). + + This enables organization owners to share fine-tuned models with other projects + in their organization. + + Args: + project_ids: The project identifiers to grant access to. + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if not fine_tuned_model_checkpoint: + raise ValueError( + f"Expected a non-empty value for `fine_tuned_model_checkpoint` but received {fine_tuned_model_checkpoint!r}" + ) + return self._get_api_list( + f"/fine_tuning/checkpoints/{fine_tuned_model_checkpoint}/permissions", + page=SyncPage[PermissionCreateResponse], + body=maybe_transform({"project_ids": project_ids}, permission_create_params.PermissionCreateParams), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + model=PermissionCreateResponse, + method="post", + ) + + def retrieve( + self, + fine_tuned_model_checkpoint: str, + *, + after: str | NotGiven = NOT_GIVEN, + limit: int | NotGiven = NOT_GIVEN, + order: Literal["ascending", "descending"] | NotGiven = NOT_GIVEN, + project_id: str | NotGiven = NOT_GIVEN, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> PermissionRetrieveResponse: + """ + **NOTE:** This endpoint requires an [admin API key](../admin-api-keys). + + Organization owners can use this endpoint to view all permissions for a + fine-tuned model checkpoint. + + Args: + after: Identifier for the last permission ID from the previous pagination request. + + limit: Number of permissions to retrieve. + + order: The order in which to retrieve permissions. + + project_id: The ID of the project to get permissions for. + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if not fine_tuned_model_checkpoint: + raise ValueError( + f"Expected a non-empty value for `fine_tuned_model_checkpoint` but received {fine_tuned_model_checkpoint!r}" + ) + return self._get( + f"/fine_tuning/checkpoints/{fine_tuned_model_checkpoint}/permissions", + options=make_request_options( + extra_headers=extra_headers, + extra_query=extra_query, + extra_body=extra_body, + timeout=timeout, + query=maybe_transform( + { + "after": after, + "limit": limit, + "order": order, + "project_id": project_id, + }, + permission_retrieve_params.PermissionRetrieveParams, + ), + ), + cast_to=PermissionRetrieveResponse, + ) + + def delete( + self, + fine_tuned_model_checkpoint: str, + *, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> PermissionDeleteResponse: + """ + **NOTE:** This endpoint requires an [admin API key](../admin-api-keys). + + Organization owners can use this endpoint to delete a permission for a + fine-tuned model checkpoint. + + Args: + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if not fine_tuned_model_checkpoint: + raise ValueError( + f"Expected a non-empty value for `fine_tuned_model_checkpoint` but received {fine_tuned_model_checkpoint!r}" + ) + return self._delete( + f"/fine_tuning/checkpoints/{fine_tuned_model_checkpoint}/permissions", + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=PermissionDeleteResponse, + ) + + +class AsyncPermissions(AsyncAPIResource): + @cached_property + def with_raw_response(self) -> AsyncPermissionsWithRawResponse: + """ + This property can be used as a prefix for any HTTP method call to return + the raw response object instead of the parsed content. + + For more information, see https://www.github.com/openai/openai-python#accessing-raw-response-data-eg-headers + """ + return AsyncPermissionsWithRawResponse(self) + + @cached_property + def with_streaming_response(self) -> AsyncPermissionsWithStreamingResponse: + """ + An alternative to `.with_raw_response` that doesn't eagerly read the response body. + + For more information, see https://www.github.com/openai/openai-python#with_streaming_response + """ + return AsyncPermissionsWithStreamingResponse(self) + + def create( + self, + fine_tuned_model_checkpoint: str, + *, + project_ids: List[str], + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> AsyncPaginator[PermissionCreateResponse, AsyncPage[PermissionCreateResponse]]: + """ + **NOTE:** Calling this endpoint requires an [admin API key](../admin-api-keys). + + This enables organization owners to share fine-tuned models with other projects + in their organization. + + Args: + project_ids: The project identifiers to grant access to. + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if not fine_tuned_model_checkpoint: + raise ValueError( + f"Expected a non-empty value for `fine_tuned_model_checkpoint` but received {fine_tuned_model_checkpoint!r}" + ) + return self._get_api_list( + f"/fine_tuning/checkpoints/{fine_tuned_model_checkpoint}/permissions", + page=AsyncPage[PermissionCreateResponse], + body=maybe_transform({"project_ids": project_ids}, permission_create_params.PermissionCreateParams), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + model=PermissionCreateResponse, + method="post", + ) + + async def retrieve( + self, + fine_tuned_model_checkpoint: str, + *, + after: str | NotGiven = NOT_GIVEN, + limit: int | NotGiven = NOT_GIVEN, + order: Literal["ascending", "descending"] | NotGiven = NOT_GIVEN, + project_id: str | NotGiven = NOT_GIVEN, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> PermissionRetrieveResponse: + """ + **NOTE:** This endpoint requires an [admin API key](../admin-api-keys). + + Organization owners can use this endpoint to view all permissions for a + fine-tuned model checkpoint. + + Args: + after: Identifier for the last permission ID from the previous pagination request. + + limit: Number of permissions to retrieve. + + order: The order in which to retrieve permissions. + + project_id: The ID of the project to get permissions for. + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if not fine_tuned_model_checkpoint: + raise ValueError( + f"Expected a non-empty value for `fine_tuned_model_checkpoint` but received {fine_tuned_model_checkpoint!r}" + ) + return await self._get( + f"/fine_tuning/checkpoints/{fine_tuned_model_checkpoint}/permissions", + options=make_request_options( + extra_headers=extra_headers, + extra_query=extra_query, + extra_body=extra_body, + timeout=timeout, + query=await async_maybe_transform( + { + "after": after, + "limit": limit, + "order": order, + "project_id": project_id, + }, + permission_retrieve_params.PermissionRetrieveParams, + ), + ), + cast_to=PermissionRetrieveResponse, + ) + + async def delete( + self, + fine_tuned_model_checkpoint: str, + *, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> PermissionDeleteResponse: + """ + **NOTE:** This endpoint requires an [admin API key](../admin-api-keys). + + Organization owners can use this endpoint to delete a permission for a + fine-tuned model checkpoint. + + Args: + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if not fine_tuned_model_checkpoint: + raise ValueError( + f"Expected a non-empty value for `fine_tuned_model_checkpoint` but received {fine_tuned_model_checkpoint!r}" + ) + return await self._delete( + f"/fine_tuning/checkpoints/{fine_tuned_model_checkpoint}/permissions", + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=PermissionDeleteResponse, + ) + + +class PermissionsWithRawResponse: + def __init__(self, permissions: Permissions) -> None: + self._permissions = permissions + + self.create = _legacy_response.to_raw_response_wrapper( + permissions.create, + ) + self.retrieve = _legacy_response.to_raw_response_wrapper( + permissions.retrieve, + ) + self.delete = _legacy_response.to_raw_response_wrapper( + permissions.delete, + ) + + +class AsyncPermissionsWithRawResponse: + def __init__(self, permissions: AsyncPermissions) -> None: + self._permissions = permissions + + self.create = _legacy_response.async_to_raw_response_wrapper( + permissions.create, + ) + self.retrieve = _legacy_response.async_to_raw_response_wrapper( + permissions.retrieve, + ) + self.delete = _legacy_response.async_to_raw_response_wrapper( + permissions.delete, + ) + + +class PermissionsWithStreamingResponse: + def __init__(self, permissions: Permissions) -> None: + self._permissions = permissions + + self.create = to_streamed_response_wrapper( + permissions.create, + ) + self.retrieve = to_streamed_response_wrapper( + permissions.retrieve, + ) + self.delete = to_streamed_response_wrapper( + permissions.delete, + ) + + +class AsyncPermissionsWithStreamingResponse: + def __init__(self, permissions: AsyncPermissions) -> None: + self._permissions = permissions + + self.create = async_to_streamed_response_wrapper( + permissions.create, + ) + self.retrieve = async_to_streamed_response_wrapper( + permissions.retrieve, + ) + self.delete = async_to_streamed_response_wrapper( + permissions.delete, + ) diff --git a/lib/python3.12/site-packages/openai/resources/fine_tuning/fine_tuning.py b/lib/python3.12/site-packages/openai/resources/fine_tuning/fine_tuning.py new file mode 100644 index 0000000000000000000000000000000000000000..1388c8230c5c529969cd58d8a223180ccf42ad23 --- /dev/null +++ b/lib/python3.12/site-packages/openai/resources/fine_tuning/fine_tuning.py @@ -0,0 +1,134 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from ..._compat import cached_property +from .jobs.jobs import ( + Jobs, + AsyncJobs, + JobsWithRawResponse, + AsyncJobsWithRawResponse, + JobsWithStreamingResponse, + AsyncJobsWithStreamingResponse, +) +from ..._resource import SyncAPIResource, AsyncAPIResource +from .checkpoints.checkpoints import ( + Checkpoints, + AsyncCheckpoints, + CheckpointsWithRawResponse, + AsyncCheckpointsWithRawResponse, + CheckpointsWithStreamingResponse, + AsyncCheckpointsWithStreamingResponse, +) + +__all__ = ["FineTuning", "AsyncFineTuning"] + + +class FineTuning(SyncAPIResource): + @cached_property + def jobs(self) -> Jobs: + return Jobs(self._client) + + @cached_property + def checkpoints(self) -> Checkpoints: + return Checkpoints(self._client) + + @cached_property + def with_raw_response(self) -> FineTuningWithRawResponse: + """ + This property can be used as a prefix for any HTTP method call to return + the raw response object instead of the parsed content. + + For more information, see https://www.github.com/openai/openai-python#accessing-raw-response-data-eg-headers + """ + return FineTuningWithRawResponse(self) + + @cached_property + def with_streaming_response(self) -> FineTuningWithStreamingResponse: + """ + An alternative to `.with_raw_response` that doesn't eagerly read the response body. + + For more information, see https://www.github.com/openai/openai-python#with_streaming_response + """ + return FineTuningWithStreamingResponse(self) + + +class AsyncFineTuning(AsyncAPIResource): + @cached_property + def jobs(self) -> AsyncJobs: + return AsyncJobs(self._client) + + @cached_property + def checkpoints(self) -> AsyncCheckpoints: + return AsyncCheckpoints(self._client) + + @cached_property + def with_raw_response(self) -> AsyncFineTuningWithRawResponse: + """ + This property can be used as a prefix for any HTTP method call to return + the raw response object instead of the parsed content. + + For more information, see https://www.github.com/openai/openai-python#accessing-raw-response-data-eg-headers + """ + return AsyncFineTuningWithRawResponse(self) + + @cached_property + def with_streaming_response(self) -> AsyncFineTuningWithStreamingResponse: + """ + An alternative to `.with_raw_response` that doesn't eagerly read the response body. + + For more information, see https://www.github.com/openai/openai-python#with_streaming_response + """ + return AsyncFineTuningWithStreamingResponse(self) + + +class FineTuningWithRawResponse: + def __init__(self, fine_tuning: FineTuning) -> None: + self._fine_tuning = fine_tuning + + @cached_property + def jobs(self) -> JobsWithRawResponse: + return JobsWithRawResponse(self._fine_tuning.jobs) + + @cached_property + def checkpoints(self) -> CheckpointsWithRawResponse: + return CheckpointsWithRawResponse(self._fine_tuning.checkpoints) + + +class AsyncFineTuningWithRawResponse: + def __init__(self, fine_tuning: AsyncFineTuning) -> None: + self._fine_tuning = fine_tuning + + @cached_property + def jobs(self) -> AsyncJobsWithRawResponse: + return AsyncJobsWithRawResponse(self._fine_tuning.jobs) + + @cached_property + def checkpoints(self) -> AsyncCheckpointsWithRawResponse: + return AsyncCheckpointsWithRawResponse(self._fine_tuning.checkpoints) + + +class FineTuningWithStreamingResponse: + def __init__(self, fine_tuning: FineTuning) -> None: + self._fine_tuning = fine_tuning + + @cached_property + def jobs(self) -> JobsWithStreamingResponse: + return JobsWithStreamingResponse(self._fine_tuning.jobs) + + @cached_property + def checkpoints(self) -> CheckpointsWithStreamingResponse: + return CheckpointsWithStreamingResponse(self._fine_tuning.checkpoints) + + +class AsyncFineTuningWithStreamingResponse: + def __init__(self, fine_tuning: AsyncFineTuning) -> None: + self._fine_tuning = fine_tuning + + @cached_property + def jobs(self) -> AsyncJobsWithStreamingResponse: + return AsyncJobsWithStreamingResponse(self._fine_tuning.jobs) + + @cached_property + def checkpoints(self) -> AsyncCheckpointsWithStreamingResponse: + return AsyncCheckpointsWithStreamingResponse(self._fine_tuning.checkpoints) diff --git a/lib/python3.12/site-packages/openai/resources/fine_tuning/jobs/__init__.py b/lib/python3.12/site-packages/openai/resources/fine_tuning/jobs/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..94cd1fb7e7a6ff9ba28805922a5e8e5a9d18c26d --- /dev/null +++ b/lib/python3.12/site-packages/openai/resources/fine_tuning/jobs/__init__.py @@ -0,0 +1,33 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from .jobs import ( + Jobs, + AsyncJobs, + JobsWithRawResponse, + AsyncJobsWithRawResponse, + JobsWithStreamingResponse, + AsyncJobsWithStreamingResponse, +) +from .checkpoints import ( + Checkpoints, + AsyncCheckpoints, + CheckpointsWithRawResponse, + AsyncCheckpointsWithRawResponse, + CheckpointsWithStreamingResponse, + AsyncCheckpointsWithStreamingResponse, +) + +__all__ = [ + "Checkpoints", + "AsyncCheckpoints", + "CheckpointsWithRawResponse", + "AsyncCheckpointsWithRawResponse", + "CheckpointsWithStreamingResponse", + "AsyncCheckpointsWithStreamingResponse", + "Jobs", + "AsyncJobs", + "JobsWithRawResponse", + "AsyncJobsWithRawResponse", + "JobsWithStreamingResponse", + "AsyncJobsWithStreamingResponse", +] diff --git a/lib/python3.12/site-packages/openai/resources/fine_tuning/jobs/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/openai/resources/fine_tuning/jobs/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b1622fffb891eaa5d539386e92942f44bc66695c Binary files /dev/null and b/lib/python3.12/site-packages/openai/resources/fine_tuning/jobs/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/resources/fine_tuning/jobs/__pycache__/checkpoints.cpython-312.pyc b/lib/python3.12/site-packages/openai/resources/fine_tuning/jobs/__pycache__/checkpoints.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..746100cf0d2d06f2d188e8fd2170865237667073 Binary files /dev/null and b/lib/python3.12/site-packages/openai/resources/fine_tuning/jobs/__pycache__/checkpoints.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/resources/fine_tuning/jobs/__pycache__/jobs.cpython-312.pyc b/lib/python3.12/site-packages/openai/resources/fine_tuning/jobs/__pycache__/jobs.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8fb3a6159496601abfc0ef60757047acf1db5513 Binary files /dev/null and b/lib/python3.12/site-packages/openai/resources/fine_tuning/jobs/__pycache__/jobs.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/resources/fine_tuning/jobs/checkpoints.py b/lib/python3.12/site-packages/openai/resources/fine_tuning/jobs/checkpoints.py new file mode 100644 index 0000000000000000000000000000000000000000..f86462e513b4a151637da2c00bd1b5548905e643 --- /dev/null +++ b/lib/python3.12/site-packages/openai/resources/fine_tuning/jobs/checkpoints.py @@ -0,0 +1,199 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +import httpx + +from .... import _legacy_response +from ...._types import NOT_GIVEN, Body, Query, Headers, NotGiven +from ...._utils import maybe_transform +from ...._compat import cached_property +from ...._resource import SyncAPIResource, AsyncAPIResource +from ...._response import to_streamed_response_wrapper, async_to_streamed_response_wrapper +from ....pagination import SyncCursorPage, AsyncCursorPage +from ...._base_client import ( + AsyncPaginator, + make_request_options, +) +from ....types.fine_tuning.jobs import checkpoint_list_params +from ....types.fine_tuning.jobs.fine_tuning_job_checkpoint import FineTuningJobCheckpoint + +__all__ = ["Checkpoints", "AsyncCheckpoints"] + + +class Checkpoints(SyncAPIResource): + @cached_property + def with_raw_response(self) -> CheckpointsWithRawResponse: + """ + This property can be used as a prefix for any HTTP method call to return + the raw response object instead of the parsed content. + + For more information, see https://www.github.com/openai/openai-python#accessing-raw-response-data-eg-headers + """ + return CheckpointsWithRawResponse(self) + + @cached_property + def with_streaming_response(self) -> CheckpointsWithStreamingResponse: + """ + An alternative to `.with_raw_response` that doesn't eagerly read the response body. + + For more information, see https://www.github.com/openai/openai-python#with_streaming_response + """ + return CheckpointsWithStreamingResponse(self) + + def list( + self, + fine_tuning_job_id: str, + *, + after: str | NotGiven = NOT_GIVEN, + limit: int | NotGiven = NOT_GIVEN, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> SyncCursorPage[FineTuningJobCheckpoint]: + """ + List checkpoints for a fine-tuning job. + + Args: + after: Identifier for the last checkpoint ID from the previous pagination request. + + limit: Number of checkpoints to retrieve. + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if not fine_tuning_job_id: + raise ValueError(f"Expected a non-empty value for `fine_tuning_job_id` but received {fine_tuning_job_id!r}") + return self._get_api_list( + f"/fine_tuning/jobs/{fine_tuning_job_id}/checkpoints", + page=SyncCursorPage[FineTuningJobCheckpoint], + options=make_request_options( + extra_headers=extra_headers, + extra_query=extra_query, + extra_body=extra_body, + timeout=timeout, + query=maybe_transform( + { + "after": after, + "limit": limit, + }, + checkpoint_list_params.CheckpointListParams, + ), + ), + model=FineTuningJobCheckpoint, + ) + + +class AsyncCheckpoints(AsyncAPIResource): + @cached_property + def with_raw_response(self) -> AsyncCheckpointsWithRawResponse: + """ + This property can be used as a prefix for any HTTP method call to return + the raw response object instead of the parsed content. + + For more information, see https://www.github.com/openai/openai-python#accessing-raw-response-data-eg-headers + """ + return AsyncCheckpointsWithRawResponse(self) + + @cached_property + def with_streaming_response(self) -> AsyncCheckpointsWithStreamingResponse: + """ + An alternative to `.with_raw_response` that doesn't eagerly read the response body. + + For more information, see https://www.github.com/openai/openai-python#with_streaming_response + """ + return AsyncCheckpointsWithStreamingResponse(self) + + def list( + self, + fine_tuning_job_id: str, + *, + after: str | NotGiven = NOT_GIVEN, + limit: int | NotGiven = NOT_GIVEN, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> AsyncPaginator[FineTuningJobCheckpoint, AsyncCursorPage[FineTuningJobCheckpoint]]: + """ + List checkpoints for a fine-tuning job. + + Args: + after: Identifier for the last checkpoint ID from the previous pagination request. + + limit: Number of checkpoints to retrieve. + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if not fine_tuning_job_id: + raise ValueError(f"Expected a non-empty value for `fine_tuning_job_id` but received {fine_tuning_job_id!r}") + return self._get_api_list( + f"/fine_tuning/jobs/{fine_tuning_job_id}/checkpoints", + page=AsyncCursorPage[FineTuningJobCheckpoint], + options=make_request_options( + extra_headers=extra_headers, + extra_query=extra_query, + extra_body=extra_body, + timeout=timeout, + query=maybe_transform( + { + "after": after, + "limit": limit, + }, + checkpoint_list_params.CheckpointListParams, + ), + ), + model=FineTuningJobCheckpoint, + ) + + +class CheckpointsWithRawResponse: + def __init__(self, checkpoints: Checkpoints) -> None: + self._checkpoints = checkpoints + + self.list = _legacy_response.to_raw_response_wrapper( + checkpoints.list, + ) + + +class AsyncCheckpointsWithRawResponse: + def __init__(self, checkpoints: AsyncCheckpoints) -> None: + self._checkpoints = checkpoints + + self.list = _legacy_response.async_to_raw_response_wrapper( + checkpoints.list, + ) + + +class CheckpointsWithStreamingResponse: + def __init__(self, checkpoints: Checkpoints) -> None: + self._checkpoints = checkpoints + + self.list = to_streamed_response_wrapper( + checkpoints.list, + ) + + +class AsyncCheckpointsWithStreamingResponse: + def __init__(self, checkpoints: AsyncCheckpoints) -> None: + self._checkpoints = checkpoints + + self.list = async_to_streamed_response_wrapper( + checkpoints.list, + ) diff --git a/lib/python3.12/site-packages/openai/resources/fine_tuning/jobs/jobs.py b/lib/python3.12/site-packages/openai/resources/fine_tuning/jobs/jobs.py new file mode 100644 index 0000000000000000000000000000000000000000..bbeff60bc686dadf051f777b6e95bff9fdf65bef --- /dev/null +++ b/lib/python3.12/site-packages/openai/resources/fine_tuning/jobs/jobs.py @@ -0,0 +1,761 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing import Dict, Union, Iterable, Optional +from typing_extensions import Literal + +import httpx + +from .... import _legacy_response +from ...._types import NOT_GIVEN, Body, Query, Headers, NotGiven +from ...._utils import ( + maybe_transform, + async_maybe_transform, +) +from ...._compat import cached_property +from .checkpoints import ( + Checkpoints, + AsyncCheckpoints, + CheckpointsWithRawResponse, + AsyncCheckpointsWithRawResponse, + CheckpointsWithStreamingResponse, + AsyncCheckpointsWithStreamingResponse, +) +from ...._resource import SyncAPIResource, AsyncAPIResource +from ...._response import to_streamed_response_wrapper, async_to_streamed_response_wrapper +from ....pagination import SyncCursorPage, AsyncCursorPage +from ...._base_client import ( + AsyncPaginator, + make_request_options, +) +from ....types.fine_tuning import job_list_params, job_create_params, job_list_events_params +from ....types.shared_params.metadata import Metadata +from ....types.fine_tuning.fine_tuning_job import FineTuningJob +from ....types.fine_tuning.fine_tuning_job_event import FineTuningJobEvent + +__all__ = ["Jobs", "AsyncJobs"] + + +class Jobs(SyncAPIResource): + @cached_property + def checkpoints(self) -> Checkpoints: + return Checkpoints(self._client) + + @cached_property + def with_raw_response(self) -> JobsWithRawResponse: + """ + This property can be used as a prefix for any HTTP method call to return + the raw response object instead of the parsed content. + + For more information, see https://www.github.com/openai/openai-python#accessing-raw-response-data-eg-headers + """ + return JobsWithRawResponse(self) + + @cached_property + def with_streaming_response(self) -> JobsWithStreamingResponse: + """ + An alternative to `.with_raw_response` that doesn't eagerly read the response body. + + For more information, see https://www.github.com/openai/openai-python#with_streaming_response + """ + return JobsWithStreamingResponse(self) + + def create( + self, + *, + model: Union[str, Literal["babbage-002", "davinci-002", "gpt-3.5-turbo", "gpt-4o-mini"]], + training_file: str, + hyperparameters: job_create_params.Hyperparameters | NotGiven = NOT_GIVEN, + integrations: Optional[Iterable[job_create_params.Integration]] | NotGiven = NOT_GIVEN, + metadata: Optional[Metadata] | NotGiven = NOT_GIVEN, + method: job_create_params.Method | NotGiven = NOT_GIVEN, + seed: Optional[int] | NotGiven = NOT_GIVEN, + suffix: Optional[str] | NotGiven = NOT_GIVEN, + validation_file: Optional[str] | NotGiven = NOT_GIVEN, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> FineTuningJob: + """ + Creates a fine-tuning job which begins the process of creating a new model from + a given dataset. + + Response includes details of the enqueued job including job status and the name + of the fine-tuned models once complete. + + [Learn more about fine-tuning](https://platform.openai.com/docs/guides/fine-tuning) + + Args: + model: The name of the model to fine-tune. You can select one of the + [supported models](https://platform.openai.com/docs/guides/fine-tuning#which-models-can-be-fine-tuned). + + training_file: The ID of an uploaded file that contains training data. + + See [upload file](https://platform.openai.com/docs/api-reference/files/create) + for how to upload a file. + + Your dataset must be formatted as a JSONL file. Additionally, you must upload + your file with the purpose `fine-tune`. + + The contents of the file should differ depending on if the model uses the + [chat](https://platform.openai.com/docs/api-reference/fine-tuning/chat-input), + [completions](https://platform.openai.com/docs/api-reference/fine-tuning/completions-input) + format, or if the fine-tuning method uses the + [preference](https://platform.openai.com/docs/api-reference/fine-tuning/preference-input) + format. + + See the [fine-tuning guide](https://platform.openai.com/docs/guides/fine-tuning) + for more details. + + hyperparameters: The hyperparameters used for the fine-tuning job. This value is now deprecated + in favor of `method`, and should be passed in under the `method` parameter. + + integrations: A list of integrations to enable for your fine-tuning job. + + metadata: Set of 16 key-value pairs that can be attached to an object. This can be useful + for storing additional information about the object in a structured format, and + querying for objects via API or the dashboard. + + Keys are strings with a maximum length of 64 characters. Values are strings with + a maximum length of 512 characters. + + method: The method used for fine-tuning. + + seed: The seed controls the reproducibility of the job. Passing in the same seed and + job parameters should produce the same results, but may differ in rare cases. If + a seed is not specified, one will be generated for you. + + suffix: A string of up to 64 characters that will be added to your fine-tuned model + name. + + For example, a `suffix` of "custom-model-name" would produce a model name like + `ft:gpt-4o-mini:openai:custom-model-name:7p4lURel`. + + validation_file: The ID of an uploaded file that contains validation data. + + If you provide this file, the data is used to generate validation metrics + periodically during fine-tuning. These metrics can be viewed in the fine-tuning + results file. The same data should not be present in both train and validation + files. + + Your dataset must be formatted as a JSONL file. You must upload your file with + the purpose `fine-tune`. + + See the [fine-tuning guide](https://platform.openai.com/docs/guides/fine-tuning) + for more details. + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + return self._post( + "/fine_tuning/jobs", + body=maybe_transform( + { + "model": model, + "training_file": training_file, + "hyperparameters": hyperparameters, + "integrations": integrations, + "metadata": metadata, + "method": method, + "seed": seed, + "suffix": suffix, + "validation_file": validation_file, + }, + job_create_params.JobCreateParams, + ), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=FineTuningJob, + ) + + def retrieve( + self, + fine_tuning_job_id: str, + *, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> FineTuningJob: + """ + Get info about a fine-tuning job. + + [Learn more about fine-tuning](https://platform.openai.com/docs/guides/fine-tuning) + + Args: + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if not fine_tuning_job_id: + raise ValueError(f"Expected a non-empty value for `fine_tuning_job_id` but received {fine_tuning_job_id!r}") + return self._get( + f"/fine_tuning/jobs/{fine_tuning_job_id}", + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=FineTuningJob, + ) + + def list( + self, + *, + after: str | NotGiven = NOT_GIVEN, + limit: int | NotGiven = NOT_GIVEN, + metadata: Optional[Dict[str, str]] | NotGiven = NOT_GIVEN, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> SyncCursorPage[FineTuningJob]: + """ + List your organization's fine-tuning jobs + + Args: + after: Identifier for the last job from the previous pagination request. + + limit: Number of fine-tuning jobs to retrieve. + + metadata: Optional metadata filter. To filter, use the syntax `metadata[k]=v`. + Alternatively, set `metadata=null` to indicate no metadata. + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + return self._get_api_list( + "/fine_tuning/jobs", + page=SyncCursorPage[FineTuningJob], + options=make_request_options( + extra_headers=extra_headers, + extra_query=extra_query, + extra_body=extra_body, + timeout=timeout, + query=maybe_transform( + { + "after": after, + "limit": limit, + "metadata": metadata, + }, + job_list_params.JobListParams, + ), + ), + model=FineTuningJob, + ) + + def cancel( + self, + fine_tuning_job_id: str, + *, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> FineTuningJob: + """ + Immediately cancel a fine-tune job. + + Args: + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if not fine_tuning_job_id: + raise ValueError(f"Expected a non-empty value for `fine_tuning_job_id` but received {fine_tuning_job_id!r}") + return self._post( + f"/fine_tuning/jobs/{fine_tuning_job_id}/cancel", + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=FineTuningJob, + ) + + def list_events( + self, + fine_tuning_job_id: str, + *, + after: str | NotGiven = NOT_GIVEN, + limit: int | NotGiven = NOT_GIVEN, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> SyncCursorPage[FineTuningJobEvent]: + """ + Get status updates for a fine-tuning job. + + Args: + after: Identifier for the last event from the previous pagination request. + + limit: Number of events to retrieve. + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if not fine_tuning_job_id: + raise ValueError(f"Expected a non-empty value for `fine_tuning_job_id` but received {fine_tuning_job_id!r}") + return self._get_api_list( + f"/fine_tuning/jobs/{fine_tuning_job_id}/events", + page=SyncCursorPage[FineTuningJobEvent], + options=make_request_options( + extra_headers=extra_headers, + extra_query=extra_query, + extra_body=extra_body, + timeout=timeout, + query=maybe_transform( + { + "after": after, + "limit": limit, + }, + job_list_events_params.JobListEventsParams, + ), + ), + model=FineTuningJobEvent, + ) + + +class AsyncJobs(AsyncAPIResource): + @cached_property + def checkpoints(self) -> AsyncCheckpoints: + return AsyncCheckpoints(self._client) + + @cached_property + def with_raw_response(self) -> AsyncJobsWithRawResponse: + """ + This property can be used as a prefix for any HTTP method call to return + the raw response object instead of the parsed content. + + For more information, see https://www.github.com/openai/openai-python#accessing-raw-response-data-eg-headers + """ + return AsyncJobsWithRawResponse(self) + + @cached_property + def with_streaming_response(self) -> AsyncJobsWithStreamingResponse: + """ + An alternative to `.with_raw_response` that doesn't eagerly read the response body. + + For more information, see https://www.github.com/openai/openai-python#with_streaming_response + """ + return AsyncJobsWithStreamingResponse(self) + + async def create( + self, + *, + model: Union[str, Literal["babbage-002", "davinci-002", "gpt-3.5-turbo", "gpt-4o-mini"]], + training_file: str, + hyperparameters: job_create_params.Hyperparameters | NotGiven = NOT_GIVEN, + integrations: Optional[Iterable[job_create_params.Integration]] | NotGiven = NOT_GIVEN, + metadata: Optional[Metadata] | NotGiven = NOT_GIVEN, + method: job_create_params.Method | NotGiven = NOT_GIVEN, + seed: Optional[int] | NotGiven = NOT_GIVEN, + suffix: Optional[str] | NotGiven = NOT_GIVEN, + validation_file: Optional[str] | NotGiven = NOT_GIVEN, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> FineTuningJob: + """ + Creates a fine-tuning job which begins the process of creating a new model from + a given dataset. + + Response includes details of the enqueued job including job status and the name + of the fine-tuned models once complete. + + [Learn more about fine-tuning](https://platform.openai.com/docs/guides/fine-tuning) + + Args: + model: The name of the model to fine-tune. You can select one of the + [supported models](https://platform.openai.com/docs/guides/fine-tuning#which-models-can-be-fine-tuned). + + training_file: The ID of an uploaded file that contains training data. + + See [upload file](https://platform.openai.com/docs/api-reference/files/create) + for how to upload a file. + + Your dataset must be formatted as a JSONL file. Additionally, you must upload + your file with the purpose `fine-tune`. + + The contents of the file should differ depending on if the model uses the + [chat](https://platform.openai.com/docs/api-reference/fine-tuning/chat-input), + [completions](https://platform.openai.com/docs/api-reference/fine-tuning/completions-input) + format, or if the fine-tuning method uses the + [preference](https://platform.openai.com/docs/api-reference/fine-tuning/preference-input) + format. + + See the [fine-tuning guide](https://platform.openai.com/docs/guides/fine-tuning) + for more details. + + hyperparameters: The hyperparameters used for the fine-tuning job. This value is now deprecated + in favor of `method`, and should be passed in under the `method` parameter. + + integrations: A list of integrations to enable for your fine-tuning job. + + metadata: Set of 16 key-value pairs that can be attached to an object. This can be useful + for storing additional information about the object in a structured format, and + querying for objects via API or the dashboard. + + Keys are strings with a maximum length of 64 characters. Values are strings with + a maximum length of 512 characters. + + method: The method used for fine-tuning. + + seed: The seed controls the reproducibility of the job. Passing in the same seed and + job parameters should produce the same results, but may differ in rare cases. If + a seed is not specified, one will be generated for you. + + suffix: A string of up to 64 characters that will be added to your fine-tuned model + name. + + For example, a `suffix` of "custom-model-name" would produce a model name like + `ft:gpt-4o-mini:openai:custom-model-name:7p4lURel`. + + validation_file: The ID of an uploaded file that contains validation data. + + If you provide this file, the data is used to generate validation metrics + periodically during fine-tuning. These metrics can be viewed in the fine-tuning + results file. The same data should not be present in both train and validation + files. + + Your dataset must be formatted as a JSONL file. You must upload your file with + the purpose `fine-tune`. + + See the [fine-tuning guide](https://platform.openai.com/docs/guides/fine-tuning) + for more details. + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + return await self._post( + "/fine_tuning/jobs", + body=await async_maybe_transform( + { + "model": model, + "training_file": training_file, + "hyperparameters": hyperparameters, + "integrations": integrations, + "metadata": metadata, + "method": method, + "seed": seed, + "suffix": suffix, + "validation_file": validation_file, + }, + job_create_params.JobCreateParams, + ), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=FineTuningJob, + ) + + async def retrieve( + self, + fine_tuning_job_id: str, + *, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> FineTuningJob: + """ + Get info about a fine-tuning job. + + [Learn more about fine-tuning](https://platform.openai.com/docs/guides/fine-tuning) + + Args: + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if not fine_tuning_job_id: + raise ValueError(f"Expected a non-empty value for `fine_tuning_job_id` but received {fine_tuning_job_id!r}") + return await self._get( + f"/fine_tuning/jobs/{fine_tuning_job_id}", + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=FineTuningJob, + ) + + def list( + self, + *, + after: str | NotGiven = NOT_GIVEN, + limit: int | NotGiven = NOT_GIVEN, + metadata: Optional[Dict[str, str]] | NotGiven = NOT_GIVEN, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> AsyncPaginator[FineTuningJob, AsyncCursorPage[FineTuningJob]]: + """ + List your organization's fine-tuning jobs + + Args: + after: Identifier for the last job from the previous pagination request. + + limit: Number of fine-tuning jobs to retrieve. + + metadata: Optional metadata filter. To filter, use the syntax `metadata[k]=v`. + Alternatively, set `metadata=null` to indicate no metadata. + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + return self._get_api_list( + "/fine_tuning/jobs", + page=AsyncCursorPage[FineTuningJob], + options=make_request_options( + extra_headers=extra_headers, + extra_query=extra_query, + extra_body=extra_body, + timeout=timeout, + query=maybe_transform( + { + "after": after, + "limit": limit, + "metadata": metadata, + }, + job_list_params.JobListParams, + ), + ), + model=FineTuningJob, + ) + + async def cancel( + self, + fine_tuning_job_id: str, + *, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> FineTuningJob: + """ + Immediately cancel a fine-tune job. + + Args: + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if not fine_tuning_job_id: + raise ValueError(f"Expected a non-empty value for `fine_tuning_job_id` but received {fine_tuning_job_id!r}") + return await self._post( + f"/fine_tuning/jobs/{fine_tuning_job_id}/cancel", + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=FineTuningJob, + ) + + def list_events( + self, + fine_tuning_job_id: str, + *, + after: str | NotGiven = NOT_GIVEN, + limit: int | NotGiven = NOT_GIVEN, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> AsyncPaginator[FineTuningJobEvent, AsyncCursorPage[FineTuningJobEvent]]: + """ + Get status updates for a fine-tuning job. + + Args: + after: Identifier for the last event from the previous pagination request. + + limit: Number of events to retrieve. + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if not fine_tuning_job_id: + raise ValueError(f"Expected a non-empty value for `fine_tuning_job_id` but received {fine_tuning_job_id!r}") + return self._get_api_list( + f"/fine_tuning/jobs/{fine_tuning_job_id}/events", + page=AsyncCursorPage[FineTuningJobEvent], + options=make_request_options( + extra_headers=extra_headers, + extra_query=extra_query, + extra_body=extra_body, + timeout=timeout, + query=maybe_transform( + { + "after": after, + "limit": limit, + }, + job_list_events_params.JobListEventsParams, + ), + ), + model=FineTuningJobEvent, + ) + + +class JobsWithRawResponse: + def __init__(self, jobs: Jobs) -> None: + self._jobs = jobs + + self.create = _legacy_response.to_raw_response_wrapper( + jobs.create, + ) + self.retrieve = _legacy_response.to_raw_response_wrapper( + jobs.retrieve, + ) + self.list = _legacy_response.to_raw_response_wrapper( + jobs.list, + ) + self.cancel = _legacy_response.to_raw_response_wrapper( + jobs.cancel, + ) + self.list_events = _legacy_response.to_raw_response_wrapper( + jobs.list_events, + ) + + @cached_property + def checkpoints(self) -> CheckpointsWithRawResponse: + return CheckpointsWithRawResponse(self._jobs.checkpoints) + + +class AsyncJobsWithRawResponse: + def __init__(self, jobs: AsyncJobs) -> None: + self._jobs = jobs + + self.create = _legacy_response.async_to_raw_response_wrapper( + jobs.create, + ) + self.retrieve = _legacy_response.async_to_raw_response_wrapper( + jobs.retrieve, + ) + self.list = _legacy_response.async_to_raw_response_wrapper( + jobs.list, + ) + self.cancel = _legacy_response.async_to_raw_response_wrapper( + jobs.cancel, + ) + self.list_events = _legacy_response.async_to_raw_response_wrapper( + jobs.list_events, + ) + + @cached_property + def checkpoints(self) -> AsyncCheckpointsWithRawResponse: + return AsyncCheckpointsWithRawResponse(self._jobs.checkpoints) + + +class JobsWithStreamingResponse: + def __init__(self, jobs: Jobs) -> None: + self._jobs = jobs + + self.create = to_streamed_response_wrapper( + jobs.create, + ) + self.retrieve = to_streamed_response_wrapper( + jobs.retrieve, + ) + self.list = to_streamed_response_wrapper( + jobs.list, + ) + self.cancel = to_streamed_response_wrapper( + jobs.cancel, + ) + self.list_events = to_streamed_response_wrapper( + jobs.list_events, + ) + + @cached_property + def checkpoints(self) -> CheckpointsWithStreamingResponse: + return CheckpointsWithStreamingResponse(self._jobs.checkpoints) + + +class AsyncJobsWithStreamingResponse: + def __init__(self, jobs: AsyncJobs) -> None: + self._jobs = jobs + + self.create = async_to_streamed_response_wrapper( + jobs.create, + ) + self.retrieve = async_to_streamed_response_wrapper( + jobs.retrieve, + ) + self.list = async_to_streamed_response_wrapper( + jobs.list, + ) + self.cancel = async_to_streamed_response_wrapper( + jobs.cancel, + ) + self.list_events = async_to_streamed_response_wrapper( + jobs.list_events, + ) + + @cached_property + def checkpoints(self) -> AsyncCheckpointsWithStreamingResponse: + return AsyncCheckpointsWithStreamingResponse(self._jobs.checkpoints) diff --git a/lib/python3.12/site-packages/openai/resources/uploads/__init__.py b/lib/python3.12/site-packages/openai/resources/uploads/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..12d1056f9ec2ae80dfa5862832d6446def23bf93 --- /dev/null +++ b/lib/python3.12/site-packages/openai/resources/uploads/__init__.py @@ -0,0 +1,33 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from .parts import ( + Parts, + AsyncParts, + PartsWithRawResponse, + AsyncPartsWithRawResponse, + PartsWithStreamingResponse, + AsyncPartsWithStreamingResponse, +) +from .uploads import ( + Uploads, + AsyncUploads, + UploadsWithRawResponse, + AsyncUploadsWithRawResponse, + UploadsWithStreamingResponse, + AsyncUploadsWithStreamingResponse, +) + +__all__ = [ + "Parts", + "AsyncParts", + "PartsWithRawResponse", + "AsyncPartsWithRawResponse", + "PartsWithStreamingResponse", + "AsyncPartsWithStreamingResponse", + "Uploads", + "AsyncUploads", + "UploadsWithRawResponse", + "AsyncUploadsWithRawResponse", + "UploadsWithStreamingResponse", + "AsyncUploadsWithStreamingResponse", +] diff --git a/lib/python3.12/site-packages/openai/resources/uploads/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/openai/resources/uploads/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6c164c7ce3a6717c6db11afc7df1de8c578ce011 Binary files /dev/null and b/lib/python3.12/site-packages/openai/resources/uploads/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/resources/uploads/__pycache__/parts.cpython-312.pyc b/lib/python3.12/site-packages/openai/resources/uploads/__pycache__/parts.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..43978798003e6a90b771ee2a9e0e52fd3bdce40f Binary files /dev/null and b/lib/python3.12/site-packages/openai/resources/uploads/__pycache__/parts.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/resources/uploads/__pycache__/uploads.cpython-312.pyc b/lib/python3.12/site-packages/openai/resources/uploads/__pycache__/uploads.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bab4ecce8ae60baf02b25b0d1ce6ca80217210e9 Binary files /dev/null and b/lib/python3.12/site-packages/openai/resources/uploads/__pycache__/uploads.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/openai/resources/uploads/parts.py b/lib/python3.12/site-packages/openai/resources/uploads/parts.py new file mode 100644 index 0000000000000000000000000000000000000000..777469ac8efdc1bd69929973c8192a130d6756b8 --- /dev/null +++ b/lib/python3.12/site-packages/openai/resources/uploads/parts.py @@ -0,0 +1,210 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing import Mapping, cast + +import httpx + +from ... import _legacy_response +from ..._types import NOT_GIVEN, Body, Query, Headers, NotGiven, FileTypes +from ..._utils import ( + extract_files, + maybe_transform, + deepcopy_minimal, + async_maybe_transform, +) +from ..._compat import cached_property +from ..._resource import SyncAPIResource, AsyncAPIResource +from ..._response import to_streamed_response_wrapper, async_to_streamed_response_wrapper +from ..._base_client import make_request_options +from ...types.uploads import part_create_params +from ...types.uploads.upload_part import UploadPart + +__all__ = ["Parts", "AsyncParts"] + + +class Parts(SyncAPIResource): + @cached_property + def with_raw_response(self) -> PartsWithRawResponse: + """ + This property can be used as a prefix for any HTTP method call to return + the raw response object instead of the parsed content. + + For more information, see https://www.github.com/openai/openai-python#accessing-raw-response-data-eg-headers + """ + return PartsWithRawResponse(self) + + @cached_property + def with_streaming_response(self) -> PartsWithStreamingResponse: + """ + An alternative to `.with_raw_response` that doesn't eagerly read the response body. + + For more information, see https://www.github.com/openai/openai-python#with_streaming_response + """ + return PartsWithStreamingResponse(self) + + def create( + self, + upload_id: str, + *, + data: FileTypes, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> UploadPart: + """ + Adds a + [Part](https://platform.openai.com/docs/api-reference/uploads/part-object) to an + [Upload](https://platform.openai.com/docs/api-reference/uploads/object) object. + A Part represents a chunk of bytes from the file you are trying to upload. + + Each Part can be at most 64 MB, and you can add Parts until you hit the Upload + maximum of 8 GB. + + It is possible to add multiple Parts in parallel. You can decide the intended + order of the Parts when you + [complete the Upload](https://platform.openai.com/docs/api-reference/uploads/complete). + + Args: + data: The chunk of bytes for this Part. + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if not upload_id: + raise ValueError(f"Expected a non-empty value for `upload_id` but received {upload_id!r}") + body = deepcopy_minimal({"data": data}) + files = extract_files(cast(Mapping[str, object], body), paths=[["data"]]) + # It should be noted that the actual Content-Type header that will be + # sent to the server will contain a `boundary` parameter, e.g. + # multipart/form-data; boundary=---abc-- + extra_headers = {"Content-Type": "multipart/form-data", **(extra_headers or {})} + return self._post( + f"/uploads/{upload_id}/parts", + body=maybe_transform(body, part_create_params.PartCreateParams), + files=files, + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=UploadPart, + ) + + +class AsyncParts(AsyncAPIResource): + @cached_property + def with_raw_response(self) -> AsyncPartsWithRawResponse: + """ + This property can be used as a prefix for any HTTP method call to return + the raw response object instead of the parsed content. + + For more information, see https://www.github.com/openai/openai-python#accessing-raw-response-data-eg-headers + """ + return AsyncPartsWithRawResponse(self) + + @cached_property + def with_streaming_response(self) -> AsyncPartsWithStreamingResponse: + """ + An alternative to `.with_raw_response` that doesn't eagerly read the response body. + + For more information, see https://www.github.com/openai/openai-python#with_streaming_response + """ + return AsyncPartsWithStreamingResponse(self) + + async def create( + self, + upload_id: str, + *, + data: FileTypes, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> UploadPart: + """ + Adds a + [Part](https://platform.openai.com/docs/api-reference/uploads/part-object) to an + [Upload](https://platform.openai.com/docs/api-reference/uploads/object) object. + A Part represents a chunk of bytes from the file you are trying to upload. + + Each Part can be at most 64 MB, and you can add Parts until you hit the Upload + maximum of 8 GB. + + It is possible to add multiple Parts in parallel. You can decide the intended + order of the Parts when you + [complete the Upload](https://platform.openai.com/docs/api-reference/uploads/complete). + + Args: + data: The chunk of bytes for this Part. + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if not upload_id: + raise ValueError(f"Expected a non-empty value for `upload_id` but received {upload_id!r}") + body = deepcopy_minimal({"data": data}) + files = extract_files(cast(Mapping[str, object], body), paths=[["data"]]) + # It should be noted that the actual Content-Type header that will be + # sent to the server will contain a `boundary` parameter, e.g. + # multipart/form-data; boundary=---abc-- + extra_headers = {"Content-Type": "multipart/form-data", **(extra_headers or {})} + return await self._post( + f"/uploads/{upload_id}/parts", + body=await async_maybe_transform(body, part_create_params.PartCreateParams), + files=files, + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=UploadPart, + ) + + +class PartsWithRawResponse: + def __init__(self, parts: Parts) -> None: + self._parts = parts + + self.create = _legacy_response.to_raw_response_wrapper( + parts.create, + ) + + +class AsyncPartsWithRawResponse: + def __init__(self, parts: AsyncParts) -> None: + self._parts = parts + + self.create = _legacy_response.async_to_raw_response_wrapper( + parts.create, + ) + + +class PartsWithStreamingResponse: + def __init__(self, parts: Parts) -> None: + self._parts = parts + + self.create = to_streamed_response_wrapper( + parts.create, + ) + + +class AsyncPartsWithStreamingResponse: + def __init__(self, parts: AsyncParts) -> None: + self._parts = parts + + self.create = async_to_streamed_response_wrapper( + parts.create, + ) diff --git a/lib/python3.12/site-packages/openai/resources/uploads/uploads.py b/lib/python3.12/site-packages/openai/resources/uploads/uploads.py new file mode 100644 index 0000000000000000000000000000000000000000..9297dbc2c370f1b37b2a100bbfee124cbb063ba6 --- /dev/null +++ b/lib/python3.12/site-packages/openai/resources/uploads/uploads.py @@ -0,0 +1,714 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +import io +import os +import logging +import builtins +from typing import List, overload +from pathlib import Path + +import anyio +import httpx + +from ... import _legacy_response +from .parts import ( + Parts, + AsyncParts, + PartsWithRawResponse, + AsyncPartsWithRawResponse, + PartsWithStreamingResponse, + AsyncPartsWithStreamingResponse, +) +from ...types import FilePurpose, upload_create_params, upload_complete_params +from ..._types import NOT_GIVEN, Body, Query, Headers, NotGiven +from ..._utils import ( + maybe_transform, + async_maybe_transform, +) +from ..._compat import cached_property +from ..._resource import SyncAPIResource, AsyncAPIResource +from ..._response import to_streamed_response_wrapper, async_to_streamed_response_wrapper +from ..._base_client import make_request_options +from ...types.upload import Upload +from ...types.file_purpose import FilePurpose + +__all__ = ["Uploads", "AsyncUploads"] + + +# 64MB +DEFAULT_PART_SIZE = 64 * 1024 * 1024 + +log: logging.Logger = logging.getLogger(__name__) + + +class Uploads(SyncAPIResource): + @cached_property + def parts(self) -> Parts: + return Parts(self._client) + + @cached_property + def with_raw_response(self) -> UploadsWithRawResponse: + """ + This property can be used as a prefix for any HTTP method call to return + the raw response object instead of the parsed content. + + For more information, see https://www.github.com/openai/openai-python#accessing-raw-response-data-eg-headers + """ + return UploadsWithRawResponse(self) + + @cached_property + def with_streaming_response(self) -> UploadsWithStreamingResponse: + """ + An alternative to `.with_raw_response` that doesn't eagerly read the response body. + + For more information, see https://www.github.com/openai/openai-python#with_streaming_response + """ + return UploadsWithStreamingResponse(self) + + @overload + def upload_file_chunked( + self, + *, + file: os.PathLike[str], + mime_type: str, + purpose: FilePurpose, + bytes: int | None = None, + part_size: int | None = None, + md5: str | NotGiven = NOT_GIVEN, + ) -> Upload: + """Splits a file into multiple 64MB parts and uploads them sequentially.""" + + @overload + def upload_file_chunked( + self, + *, + file: bytes, + filename: str, + bytes: int, + mime_type: str, + purpose: FilePurpose, + part_size: int | None = None, + md5: str | NotGiven = NOT_GIVEN, + ) -> Upload: + """Splits an in-memory file into multiple 64MB parts and uploads them sequentially.""" + + def upload_file_chunked( + self, + *, + file: os.PathLike[str] | bytes, + mime_type: str, + purpose: FilePurpose, + filename: str | None = None, + bytes: int | None = None, + part_size: int | None = None, + md5: str | NotGiven = NOT_GIVEN, + ) -> Upload: + """Splits the given file into multiple parts and uploads them sequentially. + + ```py + from pathlib import Path + + client.uploads.upload_file( + file=Path("my-paper.pdf"), + mime_type="pdf", + purpose="assistants", + ) + ``` + """ + if isinstance(file, builtins.bytes): + if filename is None: + raise TypeError("The `filename` argument must be given for in-memory files") + + if bytes is None: + raise TypeError("The `bytes` argument must be given for in-memory files") + else: + if not isinstance(file, Path): + file = Path(file) + + if not filename: + filename = file.name + + if bytes is None: + bytes = file.stat().st_size + + upload = self.create( + bytes=bytes, + filename=filename, + mime_type=mime_type, + purpose=purpose, + ) + + part_ids: list[str] = [] + + if part_size is None: + part_size = DEFAULT_PART_SIZE + + if isinstance(file, builtins.bytes): + buf: io.FileIO | io.BytesIO = io.BytesIO(file) + else: + buf = io.FileIO(file) + + try: + while True: + data = buf.read(part_size) + if not data: + # EOF + break + + part = self.parts.create(upload_id=upload.id, data=data) + log.info("Uploaded part %s for upload %s", part.id, upload.id) + part_ids.append(part.id) + except Exception: + buf.close() + raise + + return self.complete(upload_id=upload.id, part_ids=part_ids, md5=md5) + + def create( + self, + *, + bytes: int, + filename: str, + mime_type: str, + purpose: FilePurpose, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> Upload: + """ + Creates an intermediate + [Upload](https://platform.openai.com/docs/api-reference/uploads/object) object + that you can add + [Parts](https://platform.openai.com/docs/api-reference/uploads/part-object) to. + Currently, an Upload can accept at most 8 GB in total and expires after an hour + after you create it. + + Once you complete the Upload, we will create a + [File](https://platform.openai.com/docs/api-reference/files/object) object that + contains all the parts you uploaded. This File is usable in the rest of our + platform as a regular File object. + + For certain `purpose` values, the correct `mime_type` must be specified. Please + refer to documentation for the + [supported MIME types for your use case](https://platform.openai.com/docs/assistants/tools/file-search#supported-files). + + For guidance on the proper filename extensions for each purpose, please follow + the documentation on + [creating a File](https://platform.openai.com/docs/api-reference/files/create). + + Args: + bytes: The number of bytes in the file you are uploading. + + filename: The name of the file to upload. + + mime_type: The MIME type of the file. + + This must fall within the supported MIME types for your file purpose. See the + supported MIME types for assistants and vision. + + purpose: The intended purpose of the uploaded file. + + See the + [documentation on File purposes](https://platform.openai.com/docs/api-reference/files/create#files-create-purpose). + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + return self._post( + "/uploads", + body=maybe_transform( + { + "bytes": bytes, + "filename": filename, + "mime_type": mime_type, + "purpose": purpose, + }, + upload_create_params.UploadCreateParams, + ), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=Upload, + ) + + def cancel( + self, + upload_id: str, + *, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> Upload: + """Cancels the Upload. + + No Parts may be added after an Upload is cancelled. + + Args: + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if not upload_id: + raise ValueError(f"Expected a non-empty value for `upload_id` but received {upload_id!r}") + return self._post( + f"/uploads/{upload_id}/cancel", + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=Upload, + ) + + def complete( + self, + upload_id: str, + *, + part_ids: List[str], + md5: str | NotGiven = NOT_GIVEN, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> Upload: + """ + Completes the + [Upload](https://platform.openai.com/docs/api-reference/uploads/object). + + Within the returned Upload object, there is a nested + [File](https://platform.openai.com/docs/api-reference/files/object) object that + is ready to use in the rest of the platform. + + You can specify the order of the Parts by passing in an ordered list of the Part + IDs. + + The number of bytes uploaded upon completion must match the number of bytes + initially specified when creating the Upload object. No Parts may be added after + an Upload is completed. + + Args: + part_ids: The ordered list of Part IDs. + + md5: The optional md5 checksum for the file contents to verify if the bytes uploaded + matches what you expect. + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if not upload_id: + raise ValueError(f"Expected a non-empty value for `upload_id` but received {upload_id!r}") + return self._post( + f"/uploads/{upload_id}/complete", + body=maybe_transform( + { + "part_ids": part_ids, + "md5": md5, + }, + upload_complete_params.UploadCompleteParams, + ), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=Upload, + ) + + +class AsyncUploads(AsyncAPIResource): + @cached_property + def parts(self) -> AsyncParts: + return AsyncParts(self._client) + + @cached_property + def with_raw_response(self) -> AsyncUploadsWithRawResponse: + """ + This property can be used as a prefix for any HTTP method call to return + the raw response object instead of the parsed content. + + For more information, see https://www.github.com/openai/openai-python#accessing-raw-response-data-eg-headers + """ + return AsyncUploadsWithRawResponse(self) + + @cached_property + def with_streaming_response(self) -> AsyncUploadsWithStreamingResponse: + """ + An alternative to `.with_raw_response` that doesn't eagerly read the response body. + + For more information, see https://www.github.com/openai/openai-python#with_streaming_response + """ + return AsyncUploadsWithStreamingResponse(self) + + @overload + async def upload_file_chunked( + self, + *, + file: os.PathLike[str], + mime_type: str, + purpose: FilePurpose, + bytes: int | None = None, + part_size: int | None = None, + md5: str | NotGiven = NOT_GIVEN, + ) -> Upload: + """Splits a file into multiple 64MB parts and uploads them sequentially.""" + + @overload + async def upload_file_chunked( + self, + *, + file: bytes, + filename: str, + bytes: int, + mime_type: str, + purpose: FilePurpose, + part_size: int | None = None, + md5: str | NotGiven = NOT_GIVEN, + ) -> Upload: + """Splits an in-memory file into multiple 64MB parts and uploads them sequentially.""" + + async def upload_file_chunked( + self, + *, + file: os.PathLike[str] | bytes, + mime_type: str, + purpose: FilePurpose, + filename: str | None = None, + bytes: int | None = None, + part_size: int | None = None, + md5: str | NotGiven = NOT_GIVEN, + ) -> Upload: + """Splits the given file into multiple parts and uploads them sequentially. + + ```py + from pathlib import Path + + client.uploads.upload_file( + file=Path("my-paper.pdf"), + mime_type="pdf", + purpose="assistants", + ) + ``` + """ + if isinstance(file, builtins.bytes): + if filename is None: + raise TypeError("The `filename` argument must be given for in-memory files") + + if bytes is None: + raise TypeError("The `bytes` argument must be given for in-memory files") + else: + if not isinstance(file, anyio.Path): + file = anyio.Path(file) + + if not filename: + filename = file.name + + if bytes is None: + stat = await file.stat() + bytes = stat.st_size + + upload = await self.create( + bytes=bytes, + filename=filename, + mime_type=mime_type, + purpose=purpose, + ) + + part_ids: list[str] = [] + + if part_size is None: + part_size = DEFAULT_PART_SIZE + + if isinstance(file, anyio.Path): + fd = await file.open("rb") + async with fd: + while True: + data = await fd.read(part_size) + if not data: + # EOF + break + + part = await self.parts.create(upload_id=upload.id, data=data) + log.info("Uploaded part %s for upload %s", part.id, upload.id) + part_ids.append(part.id) + else: + buf = io.BytesIO(file) + + try: + while True: + data = buf.read(part_size) + if not data: + # EOF + break + + part = await self.parts.create(upload_id=upload.id, data=data) + log.info("Uploaded part %s for upload %s", part.id, upload.id) + part_ids.append(part.id) + except Exception: + buf.close() + raise + + return await self.complete(upload_id=upload.id, part_ids=part_ids, md5=md5) + + async def create( + self, + *, + bytes: int, + filename: str, + mime_type: str, + purpose: FilePurpose, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> Upload: + """ + Creates an intermediate + [Upload](https://platform.openai.com/docs/api-reference/uploads/object) object + that you can add + [Parts](https://platform.openai.com/docs/api-reference/uploads/part-object) to. + Currently, an Upload can accept at most 8 GB in total and expires after an hour + after you create it. + + Once you complete the Upload, we will create a + [File](https://platform.openai.com/docs/api-reference/files/object) object that + contains all the parts you uploaded. This File is usable in the rest of our + platform as a regular File object. + + For certain `purpose` values, the correct `mime_type` must be specified. Please + refer to documentation for the + [supported MIME types for your use case](https://platform.openai.com/docs/assistants/tools/file-search#supported-files). + + For guidance on the proper filename extensions for each purpose, please follow + the documentation on + [creating a File](https://platform.openai.com/docs/api-reference/files/create). + + Args: + bytes: The number of bytes in the file you are uploading. + + filename: The name of the file to upload. + + mime_type: The MIME type of the file. + + This must fall within the supported MIME types for your file purpose. See the + supported MIME types for assistants and vision. + + purpose: The intended purpose of the uploaded file. + + See the + [documentation on File purposes](https://platform.openai.com/docs/api-reference/files/create#files-create-purpose). + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + return await self._post( + "/uploads", + body=await async_maybe_transform( + { + "bytes": bytes, + "filename": filename, + "mime_type": mime_type, + "purpose": purpose, + }, + upload_create_params.UploadCreateParams, + ), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=Upload, + ) + + async def cancel( + self, + upload_id: str, + *, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> Upload: + """Cancels the Upload. + + No Parts may be added after an Upload is cancelled. + + Args: + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if not upload_id: + raise ValueError(f"Expected a non-empty value for `upload_id` but received {upload_id!r}") + return await self._post( + f"/uploads/{upload_id}/cancel", + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=Upload, + ) + + async def complete( + self, + upload_id: str, + *, + part_ids: List[str], + md5: str | NotGiven = NOT_GIVEN, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> Upload: + """ + Completes the + [Upload](https://platform.openai.com/docs/api-reference/uploads/object). + + Within the returned Upload object, there is a nested + [File](https://platform.openai.com/docs/api-reference/files/object) object that + is ready to use in the rest of the platform. + + You can specify the order of the Parts by passing in an ordered list of the Part + IDs. + + The number of bytes uploaded upon completion must match the number of bytes + initially specified when creating the Upload object. No Parts may be added after + an Upload is completed. + + Args: + part_ids: The ordered list of Part IDs. + + md5: The optional md5 checksum for the file contents to verify if the bytes uploaded + matches what you expect. + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if not upload_id: + raise ValueError(f"Expected a non-empty value for `upload_id` but received {upload_id!r}") + return await self._post( + f"/uploads/{upload_id}/complete", + body=await async_maybe_transform( + { + "part_ids": part_ids, + "md5": md5, + }, + upload_complete_params.UploadCompleteParams, + ), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=Upload, + ) + + +class UploadsWithRawResponse: + def __init__(self, uploads: Uploads) -> None: + self._uploads = uploads + + self.create = _legacy_response.to_raw_response_wrapper( + uploads.create, + ) + self.cancel = _legacy_response.to_raw_response_wrapper( + uploads.cancel, + ) + self.complete = _legacy_response.to_raw_response_wrapper( + uploads.complete, + ) + + @cached_property + def parts(self) -> PartsWithRawResponse: + return PartsWithRawResponse(self._uploads.parts) + + +class AsyncUploadsWithRawResponse: + def __init__(self, uploads: AsyncUploads) -> None: + self._uploads = uploads + + self.create = _legacy_response.async_to_raw_response_wrapper( + uploads.create, + ) + self.cancel = _legacy_response.async_to_raw_response_wrapper( + uploads.cancel, + ) + self.complete = _legacy_response.async_to_raw_response_wrapper( + uploads.complete, + ) + + @cached_property + def parts(self) -> AsyncPartsWithRawResponse: + return AsyncPartsWithRawResponse(self._uploads.parts) + + +class UploadsWithStreamingResponse: + def __init__(self, uploads: Uploads) -> None: + self._uploads = uploads + + self.create = to_streamed_response_wrapper( + uploads.create, + ) + self.cancel = to_streamed_response_wrapper( + uploads.cancel, + ) + self.complete = to_streamed_response_wrapper( + uploads.complete, + ) + + @cached_property + def parts(self) -> PartsWithStreamingResponse: + return PartsWithStreamingResponse(self._uploads.parts) + + +class AsyncUploadsWithStreamingResponse: + def __init__(self, uploads: AsyncUploads) -> None: + self._uploads = uploads + + self.create = async_to_streamed_response_wrapper( + uploads.create, + ) + self.cancel = async_to_streamed_response_wrapper( + uploads.cancel, + ) + self.complete = async_to_streamed_response_wrapper( + uploads.complete, + ) + + @cached_property + def parts(self) -> AsyncPartsWithStreamingResponse: + return AsyncPartsWithStreamingResponse(self._uploads.parts) diff --git a/lib/python3.12/site-packages/sentry_sdk-2.52.0.dist-info/INSTALLER b/lib/python3.12/site-packages/sentry_sdk-2.52.0.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68 --- /dev/null +++ b/lib/python3.12/site-packages/sentry_sdk-2.52.0.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/lib/python3.12/site-packages/sentry_sdk-2.52.0.dist-info/METADATA b/lib/python3.12/site-packages/sentry_sdk-2.52.0.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..09adeb4436fbcd3700c3600cf6e2e98b13ff1ed8 --- /dev/null +++ b/lib/python3.12/site-packages/sentry_sdk-2.52.0.dist-info/METADATA @@ -0,0 +1,268 @@ +Metadata-Version: 2.4 +Name: sentry-sdk +Version: 2.52.0 +Summary: Python client for Sentry (https://sentry.io) +Home-page: https://github.com/getsentry/sentry-python +Author: Sentry Team and Contributors +Author-email: hello@sentry.io +License: MIT +Project-URL: Documentation, https://docs.sentry.io/platforms/python/ +Project-URL: Changelog, https://github.com/getsentry/sentry-python/blob/master/CHANGELOG.md +Classifier: Development Status :: 5 - Production/Stable +Classifier: Environment :: Web Environment +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: BSD License +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: 3.13 +Classifier: Programming Language :: Python :: 3.14 +Classifier: Topic :: Software Development :: Libraries :: Python Modules +Requires-Python: >=3.6 +Description-Content-Type: text/markdown +License-File: LICENSE +Requires-Dist: urllib3>=1.26.11 +Requires-Dist: certifi +Provides-Extra: aiohttp +Requires-Dist: aiohttp>=3.5; extra == "aiohttp" +Provides-Extra: anthropic +Requires-Dist: anthropic>=0.16; extra == "anthropic" +Provides-Extra: arq +Requires-Dist: arq>=0.23; extra == "arq" +Provides-Extra: asyncpg +Requires-Dist: asyncpg>=0.23; extra == "asyncpg" +Provides-Extra: beam +Requires-Dist: apache-beam>=2.12; extra == "beam" +Provides-Extra: bottle +Requires-Dist: bottle>=0.12.13; extra == "bottle" +Provides-Extra: celery +Requires-Dist: celery>=3; extra == "celery" +Provides-Extra: celery-redbeat +Requires-Dist: celery-redbeat>=2; extra == "celery-redbeat" +Provides-Extra: chalice +Requires-Dist: chalice>=1.16.0; extra == "chalice" +Provides-Extra: clickhouse-driver +Requires-Dist: clickhouse-driver>=0.2.0; extra == "clickhouse-driver" +Provides-Extra: django +Requires-Dist: django>=1.8; extra == "django" +Provides-Extra: falcon +Requires-Dist: falcon>=1.4; extra == "falcon" +Provides-Extra: fastapi +Requires-Dist: fastapi>=0.79.0; extra == "fastapi" +Provides-Extra: flask +Requires-Dist: flask>=0.11; extra == "flask" +Requires-Dist: blinker>=1.1; extra == "flask" +Requires-Dist: markupsafe; extra == "flask" +Provides-Extra: grpcio +Requires-Dist: grpcio>=1.21.1; extra == "grpcio" +Requires-Dist: protobuf>=3.8.0; extra == "grpcio" +Provides-Extra: http2 +Requires-Dist: httpcore[http2]==1.*; extra == "http2" +Provides-Extra: httpx +Requires-Dist: httpx>=0.16.0; extra == "httpx" +Provides-Extra: huey +Requires-Dist: huey>=2; extra == "huey" +Provides-Extra: huggingface-hub +Requires-Dist: huggingface_hub>=0.22; extra == "huggingface-hub" +Provides-Extra: langchain +Requires-Dist: langchain>=0.0.210; extra == "langchain" +Provides-Extra: langgraph +Requires-Dist: langgraph>=0.6.6; extra == "langgraph" +Provides-Extra: launchdarkly +Requires-Dist: launchdarkly-server-sdk>=9.8.0; extra == "launchdarkly" +Provides-Extra: litellm +Requires-Dist: litellm>=1.77.5; extra == "litellm" +Provides-Extra: litestar +Requires-Dist: litestar>=2.0.0; extra == "litestar" +Provides-Extra: loguru +Requires-Dist: loguru>=0.5; extra == "loguru" +Provides-Extra: mcp +Requires-Dist: mcp>=1.15.0; extra == "mcp" +Provides-Extra: openai +Requires-Dist: openai>=1.0.0; extra == "openai" +Requires-Dist: tiktoken>=0.3.0; extra == "openai" +Provides-Extra: openfeature +Requires-Dist: openfeature-sdk>=0.7.1; extra == "openfeature" +Provides-Extra: opentelemetry +Requires-Dist: opentelemetry-distro>=0.35b0; extra == "opentelemetry" +Provides-Extra: opentelemetry-experimental +Requires-Dist: opentelemetry-distro; extra == "opentelemetry-experimental" +Provides-Extra: opentelemetry-otlp +Requires-Dist: opentelemetry-distro[otlp]>=0.35b0; extra == "opentelemetry-otlp" +Provides-Extra: pure-eval +Requires-Dist: pure_eval; extra == "pure-eval" +Requires-Dist: executing; extra == "pure-eval" +Requires-Dist: asttokens; extra == "pure-eval" +Provides-Extra: pydantic-ai +Requires-Dist: pydantic-ai>=1.0.0; extra == "pydantic-ai" +Provides-Extra: pymongo +Requires-Dist: pymongo>=3.1; extra == "pymongo" +Provides-Extra: pyspark +Requires-Dist: pyspark>=2.4.4; extra == "pyspark" +Provides-Extra: quart +Requires-Dist: quart>=0.16.1; extra == "quart" +Requires-Dist: blinker>=1.1; extra == "quart" +Provides-Extra: rq +Requires-Dist: rq>=0.6; extra == "rq" +Provides-Extra: sanic +Requires-Dist: sanic>=0.8; extra == "sanic" +Provides-Extra: sqlalchemy +Requires-Dist: sqlalchemy>=1.2; extra == "sqlalchemy" +Provides-Extra: starlette +Requires-Dist: starlette>=0.19.1; extra == "starlette" +Provides-Extra: starlite +Requires-Dist: starlite>=1.48; extra == "starlite" +Provides-Extra: statsig +Requires-Dist: statsig>=0.55.3; extra == "statsig" +Provides-Extra: tornado +Requires-Dist: tornado>=6; extra == "tornado" +Provides-Extra: unleash +Requires-Dist: UnleashClient>=6.0.1; extra == "unleash" +Provides-Extra: google-genai +Requires-Dist: google-genai>=1.29.0; extra == "google-genai" +Dynamic: author +Dynamic: author-email +Dynamic: classifier +Dynamic: description +Dynamic: description-content-type +Dynamic: home-page +Dynamic: license +Dynamic: license-file +Dynamic: project-url +Dynamic: provides-extra +Dynamic: requires-dist +Dynamic: requires-python +Dynamic: summary + + + Sentry for Python + +
+ +_Bad software is everywhere, and we're tired of it. Sentry is on a mission to help developers write better software faster, so we can get back to enjoying technology. If you want to join us +[**Check out our open positions**](https://sentry.io/careers/)_. + +[![Discord](https://img.shields.io/discord/621778831602221064?logo=discord&labelColor=%20%235462eb&logoColor=%20%23f5f5f5&color=%20%235462eb)](https://discord.com/invite/Ww9hbqr) +[![X Follow](https://img.shields.io/twitter/follow/sentry?label=sentry&style=social)](https://x.com/intent/follow?screen_name=sentry) +[![PyPi page link -- version](https://img.shields.io/pypi/v/sentry-sdk.svg)](https://pypi.python.org/pypi/sentry-sdk) +python +[![Build Status](https://github.com/getsentry/sentry-python/actions/workflows/ci.yml/badge.svg)](https://github.com/getsentry/sentry-python/actions/workflows/ci.yml) + +
+ +
+ + +# Official Sentry SDK for Python + +Welcome to the official Python SDK for **[Sentry](http://sentry.io/)**. + + +## 📦 Getting Started + +### Prerequisites + +You need a Sentry [account](https://sentry.io/signup/) and [project](https://docs.sentry.io/product/projects/). + +### Installation + +Getting Sentry into your project is straightforward. Just run this command in your terminal: + +```bash +pip install --upgrade sentry-sdk +``` + +### Basic Configuration + +Here's a quick configuration example to get Sentry up and running: + +```python +import sentry_sdk + +sentry_sdk.init( + "https://12927b5f211046b575ee51fd8b1ac34f@o1.ingest.sentry.io/1", # Your DSN here + + # Set traces_sample_rate to 1.0 to capture 100% + # of traces for performance monitoring. + traces_sample_rate=1.0, +) +``` + +With this configuration, Sentry will monitor for exceptions and performance issues. + +### Quick Usage Example + +To generate some events that will show up in Sentry, you can log messages or capture errors: + +```python +import sentry_sdk +sentry_sdk.init(...) # same as above + +sentry_sdk.capture_message("Hello Sentry!") # You'll see this in your Sentry dashboard. + +raise ValueError("Oops, something went wrong!") # This will create an error event in Sentry. +``` + + +## 📚 Documentation + +For more details on advanced usage, integrations, and customization, check out the full documentation on [https://docs.sentry.io](https://docs.sentry.io/). + + +## 🧩 Integrations + +Sentry integrates with a ton of popular Python libraries and frameworks, including [FastAPI](https://docs.sentry.io/platforms/python/integrations/fastapi/), [Django](https://docs.sentry.io/platforms/python/integrations/django/), [Celery](https://docs.sentry.io/platforms/python/integrations/celery/), [OpenAI](https://docs.sentry.io/platforms/python/integrations/openai/) and many, many more. Check out the [full list of integrations](https://docs.sentry.io/platforms/python/integrations/) to get the full picture. + + +## 🚧 Migrating Between Versions? + +### From `1.x` to `2.x` + +If you're using the older `1.x` version of the SDK, now's the time to upgrade to `2.x`. It includes significant upgrades and new features. Check our [migration guide](https://docs.sentry.io/platforms/python/migration/1.x-to-2.x) for assistance. + +### From `raven-python` + +Using the legacy `raven-python` client? It's now in maintenance mode, and we recommend migrating to the new SDK for an improved experience. Get all the details in our [migration guide](https://docs.sentry.io/platforms/python/migration/raven-to-sentry-sdk/). + + +## 🙌 Want to Contribute? + +We'd love your help in improving the Sentry SDK! Whether it's fixing bugs, adding features, writing new integrations, or enhancing documentation, every contribution is valuable. + +For details on how to contribute, please read our [contribution guide](CONTRIBUTING.md) and explore the [open issues](https://github.com/getsentry/sentry-python/issues). + + +## 🛟 Need Help? + +If you encounter issues or need help setting up or configuring the SDK, don't hesitate to reach out to the [Sentry Community on Discord](https://discord.com/invite/Ww9hbqr). There is a ton of great people there ready to help! + + +## 🔗 Resources + +Here are all resources to help you make the most of Sentry: + +- [Documentation](https://docs.sentry.io/platforms/python/) - Official documentation to get started. +- [Discord](https://discord.com/invite/Ww9hbqr) - Join our Discord community. +- [X/Twitter](https://x.com/intent/follow?screen_name=sentry) - Follow us on X (Twitter) for updates. +- [Stack Overflow](https://stackoverflow.com/questions/tagged/sentry) - Questions and answers related to Sentry. + + +## 📃 License + +The SDK is open-source and available under the MIT license. Check out the [LICENSE](LICENSE) file for more information. + + +## 😘 Contributors + +Thanks to everyone who has helped improve the SDK! + + + + diff --git a/lib/python3.12/site-packages/sentry_sdk-2.52.0.dist-info/RECORD b/lib/python3.12/site-packages/sentry_sdk-2.52.0.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..b8551aa2549e60467ab1a0b9f80feab718e44723 --- /dev/null +++ b/lib/python3.12/site-packages/sentry_sdk-2.52.0.dist-info/RECORD @@ -0,0 +1,386 @@ +sentry_sdk-2.52.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +sentry_sdk-2.52.0.dist-info/METADATA,sha256=mYkrIa3jj1mHhoOiUqJQJ_70ocgZ1yiN2CxGChBEvDY,10815 +sentry_sdk-2.52.0.dist-info/RECORD,, +sentry_sdk-2.52.0.dist-info/WHEEL,sha256=Mk1ST5gDzEO5il5kYREiBnzzM469m5sI8ESPl7TRhJY,110 +sentry_sdk-2.52.0.dist-info/entry_points.txt,sha256=qacZEz40UspQZD1IukCXykx0JtImqGDOctS5KfOLTko,91 +sentry_sdk-2.52.0.dist-info/licenses/LICENSE,sha256=KhQNZg9GKBL6KQvHQNBGMxJsXsRdhLebVp4Sew7t3Qs,1093 +sentry_sdk-2.52.0.dist-info/top_level.txt,sha256=XrQz30XE9FKXSY_yGLrd9bsv2Rk390GTDJOSujYaMxI,11 +sentry_sdk/__init__.py,sha256=cnZoQ9y329brs-cdzIVtxbO1-o9AIrKk8VTVyZNJs1A,1410 +sentry_sdk/__pycache__/__init__.cpython-312.pyc,, +sentry_sdk/__pycache__/_batcher.cpython-312.pyc,, +sentry_sdk/__pycache__/_compat.cpython-312.pyc,, +sentry_sdk/__pycache__/_init_implementation.cpython-312.pyc,, +sentry_sdk/__pycache__/_log_batcher.cpython-312.pyc,, +sentry_sdk/__pycache__/_lru_cache.cpython-312.pyc,, +sentry_sdk/__pycache__/_metrics_batcher.cpython-312.pyc,, +sentry_sdk/__pycache__/_queue.cpython-312.pyc,, +sentry_sdk/__pycache__/_span_batcher.cpython-312.pyc,, +sentry_sdk/__pycache__/_types.cpython-312.pyc,, +sentry_sdk/__pycache__/_werkzeug.cpython-312.pyc,, +sentry_sdk/__pycache__/api.cpython-312.pyc,, +sentry_sdk/__pycache__/attachments.cpython-312.pyc,, +sentry_sdk/__pycache__/client.cpython-312.pyc,, +sentry_sdk/__pycache__/consts.cpython-312.pyc,, +sentry_sdk/__pycache__/debug.cpython-312.pyc,, +sentry_sdk/__pycache__/envelope.cpython-312.pyc,, +sentry_sdk/__pycache__/feature_flags.cpython-312.pyc,, +sentry_sdk/__pycache__/hub.cpython-312.pyc,, +sentry_sdk/__pycache__/logger.cpython-312.pyc,, +sentry_sdk/__pycache__/metrics.cpython-312.pyc,, +sentry_sdk/__pycache__/monitor.cpython-312.pyc,, +sentry_sdk/__pycache__/scope.cpython-312.pyc,, +sentry_sdk/__pycache__/scrubber.cpython-312.pyc,, +sentry_sdk/__pycache__/serializer.cpython-312.pyc,, +sentry_sdk/__pycache__/session.cpython-312.pyc,, +sentry_sdk/__pycache__/sessions.cpython-312.pyc,, +sentry_sdk/__pycache__/spotlight.cpython-312.pyc,, +sentry_sdk/__pycache__/traces.cpython-312.pyc,, +sentry_sdk/__pycache__/tracing.cpython-312.pyc,, +sentry_sdk/__pycache__/tracing_utils.cpython-312.pyc,, +sentry_sdk/__pycache__/transport.cpython-312.pyc,, +sentry_sdk/__pycache__/types.cpython-312.pyc,, +sentry_sdk/__pycache__/utils.cpython-312.pyc,, +sentry_sdk/__pycache__/worker.cpython-312.pyc,, +sentry_sdk/_batcher.py,sha256=3Elbey1gpWzYSo-Dr5xxP00QpYcRYb8FGYqei9U_mdE,4004 +sentry_sdk/_compat.py,sha256=tSI9LTAlSw7x_putvg7qg5SSdo7W1EwpbYiB_godUMQ,3065 +sentry_sdk/_init_implementation.py,sha256=xualjTVpMB8XbJUX-30eAnuiyPfDeoeZoer3iIl0mH4,2491 +sentry_sdk/_log_batcher.py,sha256=6Y3VNP6a65t1gzND9Rz9356xj2ZXOI1rB8Wq2De7Cpc,1827 +sentry_sdk/_lru_cache.py,sha256=YSnCmL1qArB6_dsfcwKZgO4eJhBak87HqjqbUzBNQ7A,1167 +sentry_sdk/_metrics_batcher.py,sha256=c7aw1sG7rldcS4XrpKAX59AGMXZ74Vx9Ja7peQo7MPc,1273 +sentry_sdk/_queue.py,sha256=bluPs51jz8d3w74aSolOZYc3jnwmyTGPKvKxv7Z2LL0,11244 +sentry_sdk/_span_batcher.py,sha256=pIMrpMf9yJUi0hfD-i8S6GM5mLH4dovKuIDpurrR86g,4332 +sentry_sdk/_types.py,sha256=bcDjVByhfqJoYB2deda3oZBYv73gRanJ3WfFM2WagXM,11018 +sentry_sdk/_werkzeug.py,sha256=vQL3Z_q3OBQ8vEby2ktiY7Ey8szMDWZsUe-BGJ9A3oY,3716 +sentry_sdk/ai/__init__.py,sha256=L2EAYEx9075dUVZGEB_CBxSgwaLeYLFWhBYfIEeBDRg,218 +sentry_sdk/ai/__pycache__/__init__.cpython-312.pyc,, +sentry_sdk/ai/__pycache__/_openai_completions_api.cpython-312.pyc,, +sentry_sdk/ai/__pycache__/_openai_responses_api.cpython-312.pyc,, +sentry_sdk/ai/__pycache__/monitoring.cpython-312.pyc,, +sentry_sdk/ai/__pycache__/utils.cpython-312.pyc,, +sentry_sdk/ai/_openai_completions_api.py,sha256=MeLmyCCd-PQLWRN4RHgCm7bK6QXBjU74Df0ZGHDDDCg,1821 +sentry_sdk/ai/_openai_responses_api.py,sha256=xN2fbP2hNibB-kW0mbA7L9s_g_JrYhyra7V4S9xF8kU,656 +sentry_sdk/ai/monitoring.py,sha256=PRYQXdKh2CdIttIKcCWhQKebGHlBHn6b0lKWAmhK7MQ,5539 +sentry_sdk/ai/utils.py,sha256=VtyghTEHI_3KRrEltYpLlIOO5FX5eD_Og1l7sqA9mx4,23837 +sentry_sdk/api.py,sha256=p4ABg2gX0Lx5T6nbi-D8LIo-p3lB37IcI91I_bEoJYg,14668 +sentry_sdk/attachments.py,sha256=fvmTzYvG2a2mjNzTzcja28ize87kEEZMtXwK1rmXpBc,3023 +sentry_sdk/client.py,sha256=jhdrYnioKRqwZy66VgqrwGNyTewajkd1stPU0iZqZLA,38867 +sentry_sdk/consts.py,sha256=Mik-gvgw_UARQ6cljsDuM2_Bs0E9_ApaPSBKajjEmLg,55042 +sentry_sdk/crons/__init__.py,sha256=3Zt6g1-pZZ12uRKKsC8QLm3XgJ4K1VYxgVpNNUygOZY,221 +sentry_sdk/crons/__pycache__/__init__.cpython-312.pyc,, +sentry_sdk/crons/__pycache__/api.cpython-312.pyc,, +sentry_sdk/crons/__pycache__/consts.cpython-312.pyc,, +sentry_sdk/crons/__pycache__/decorator.cpython-312.pyc,, +sentry_sdk/crons/api.py,sha256=gwtNjHZxZy51piEwYrnrYg3bZkifrdkSRlDAZYC30Kg,1679 +sentry_sdk/crons/consts.py,sha256=dXqJk5meBSu5rjlGpqAOlkpACnuUi7svQnAFoy1ZNUU,87 +sentry_sdk/crons/decorator.py,sha256=4fYEHYlhdtKyhzJr1HZZ7g2ierEJ3ULZe4N7a3jqbYE,3834 +sentry_sdk/debug.py,sha256=63f8uUowpbZ75E8q-r_8Tdymsv9jXZn1VMh287VcWhk,959 +sentry_sdk/envelope.py,sha256=q0CtNI9IgYPTlviQyp8HB8aeIxLVcklMIqaQ-LU_HpU,9652 +sentry_sdk/feature_flags.py,sha256=J9G-nnQiXimYbhkPNDGz_CjAccYlbXS9KgybExBZjUI,2136 +sentry_sdk/hub.py,sha256=sVkOrpjU0cNhxOFtWfxA8Nc_3T8hiGH-mVFZa4tX1Us,25147 +sentry_sdk/integrations/__init__.py,sha256=U52rckcGzXoe73On-T70R64feEIg1ir1O1NSSkQvHzg,12674 +sentry_sdk/integrations/__pycache__/__init__.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/_asgi_common.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/_wsgi_common.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/aiohttp.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/anthropic.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/argv.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/ariadne.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/arq.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/asgi.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/asyncio.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/asyncpg.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/atexit.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/aws_lambda.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/beam.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/boto3.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/bottle.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/chalice.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/clickhouse_driver.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/cloud_resource_context.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/cohere.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/dedupe.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/dramatiq.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/excepthook.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/executing.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/falcon.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/fastapi.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/flask.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/gcp.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/gnu_backtrace.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/gql.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/graphene.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/httpx.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/huey.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/huggingface_hub.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/langchain.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/langgraph.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/launchdarkly.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/litellm.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/litestar.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/logging.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/loguru.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/mcp.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/modules.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/openai.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/openfeature.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/otlp.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/pure_eval.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/pymongo.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/pyramid.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/quart.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/ray.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/rq.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/rust_tracing.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/sanic.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/serverless.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/socket.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/sqlalchemy.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/starlette.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/starlite.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/statsig.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/stdlib.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/strawberry.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/sys_exit.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/threading.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/tornado.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/trytond.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/typer.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/unleash.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/unraisablehook.cpython-312.pyc,, +sentry_sdk/integrations/__pycache__/wsgi.cpython-312.pyc,, +sentry_sdk/integrations/_asgi_common.py,sha256=qY2nH21YGtQ1EufzdrovHtGZ-ZaMbSzYNKCo8z-kD44,3145 +sentry_sdk/integrations/_wsgi_common.py,sha256=ffyvbYSYtVNo8wkzx7tt1T7anuMiu2ixApZmC0O9NH4,7282 +sentry_sdk/integrations/aiohttp.py,sha256=vQrqPm_IVgvgQo4PLEyDNrvA5oloz1xALOFtVDTtOpE,12930 +sentry_sdk/integrations/anthropic.py,sha256=CDLZf73v8HBj-OxDzJHlgmecXFHeIIYKoem2kdHmKP4,21222 +sentry_sdk/integrations/argv.py,sha256=3dyTYcNVhx-EoCxzZI8-JO1mxydBo_zHxIKHpLJgJcc,877 +sentry_sdk/integrations/ariadne.py,sha256=u1X6QEcXfwBXa4mmHkgcj8XMoowwBGIWQQYAsItpjwg,5792 +sentry_sdk/integrations/arq.py,sha256=3hs8Y41X3rjxCd769CHjUBYQe3YgULtbX6WQQB5-kwQ,7906 +sentry_sdk/integrations/asgi.py,sha256=AoJcWDRkmm5vqSMH44V2zx00BiT9Kzqfo3pd3Eyi6D8,12687 +sentry_sdk/integrations/asyncio.py,sha256=nszg5HkxWLoTzfS3EZ9hBzsCC7JM8JKOBFXbNXnCDpI,6994 +sentry_sdk/integrations/asyncpg.py,sha256=T0Ijpln-xoQCuY0AS15dYZK2_1XCdKYT1J9EfdSQURo,6578 +sentry_sdk/integrations/atexit.py,sha256=eUcgC6f7WoqJadxDbn76dBv6zoS97kgyxshjGBNK1us,1567 +sentry_sdk/integrations/aws_lambda.py,sha256=HDvJk2EpOjS0_2wkTXc9lQr2w-huT7v3j4_-Pxq0KPw,17895 +sentry_sdk/integrations/beam.py,sha256=j-BzEzAHXk12HJ8izUyIOaM02d29BZUpYldBMCq8vdI,5092 +sentry_sdk/integrations/boto3.py,sha256=4HLGehOoU8rc-85yL5ATUUZL9x_TGYjPkNYHOcT4Z2w,4341 +sentry_sdk/integrations/bottle.py,sha256=Im8xqoQvrygqKHsK-rtaX41gOaY_hxb4anjgJnZQGu4,6404 +sentry_sdk/integrations/celery/__init__.py,sha256=HU-EfhtJxnrcEowxkjSUC_ODkEmDlW8eEkZZJjPwO3U,18442 +sentry_sdk/integrations/celery/__pycache__/__init__.cpython-312.pyc,, +sentry_sdk/integrations/celery/__pycache__/beat.cpython-312.pyc,, +sentry_sdk/integrations/celery/__pycache__/utils.cpython-312.pyc,, +sentry_sdk/integrations/celery/beat.py,sha256=RCG2yAyJSyWtzDGw6B1M_w3TXDvPnznAt1dhfFcLBDI,8839 +sentry_sdk/integrations/celery/utils.py,sha256=KyaM868RGak0JeFHkQVYtR1CkKRNpSD4_bbpisF5jqo,1152 +sentry_sdk/integrations/chalice.py,sha256=nd_J4MnPZzoVdH4M5HV_-ljeS8fbxe_5DXmKJm6zqSI,4663 +sentry_sdk/integrations/clickhouse_driver.py,sha256=mNYW99LlIOCk45aUeELY9cuHTfBorhwV0rGn2c_25EY,5905 +sentry_sdk/integrations/cloud_resource_context.py,sha256=J_3Zd9czyI2NuWy09BEwGKzw80rQlVlJ1CQqTvJ5m18,7638 +sentry_sdk/integrations/cohere.py,sha256=QxgvN7fy385-EwUPYFup98UiMxxg_zrE8h0yCfI5iT8,9495 +sentry_sdk/integrations/dedupe.py,sha256=EApcjbAp_uEmsOSb0m64JNGaKUEqB46YCYX8AgTDO24,1903 +sentry_sdk/integrations/django/__init__.py,sha256=T2N0ZnN4aI4AGJvoyNtYBBXtAKPrN3suuQPQi1Ks_r8,26421 +sentry_sdk/integrations/django/__pycache__/__init__.cpython-312.pyc,, +sentry_sdk/integrations/django/__pycache__/asgi.cpython-312.pyc,, +sentry_sdk/integrations/django/__pycache__/caching.cpython-312.pyc,, +sentry_sdk/integrations/django/__pycache__/middleware.cpython-312.pyc,, +sentry_sdk/integrations/django/__pycache__/signals_handlers.cpython-312.pyc,, +sentry_sdk/integrations/django/__pycache__/tasks.cpython-312.pyc,, +sentry_sdk/integrations/django/__pycache__/templates.cpython-312.pyc,, +sentry_sdk/integrations/django/__pycache__/transactions.cpython-312.pyc,, +sentry_sdk/integrations/django/__pycache__/views.cpython-312.pyc,, +sentry_sdk/integrations/django/asgi.py,sha256=xi4ceuIgjFsvyBwzGOwHj_siLHqeYHuL5oLfPqDxPKQ,8436 +sentry_sdk/integrations/django/caching.py,sha256=gXRDSZigV1SETmxDUe8GA53Tha2fo0ZAFOofuorQD30,6980 +sentry_sdk/integrations/django/middleware.py,sha256=kSN4ORVUuO8Yvt0sC6YS8br2gk6gLmBwysIFsN9z2js,5917 +sentry_sdk/integrations/django/signals_handlers.py,sha256=E1r1sYzKQHko6fD8bPC7_rXGH5kBVDwmE2sutWuGAwQ,3062 +sentry_sdk/integrations/django/tasks.py,sha256=Bu_5jxvFNcAk9N8PedxhWP5Fq-Mj8zVRYdDRsd-6eJA,1151 +sentry_sdk/integrations/django/templates.py,sha256=NZuSVDU0srkF_La5BpEIGqonCbK-qLKMMI3bqj2za3w,5695 +sentry_sdk/integrations/django/transactions.py,sha256=zjsJgVFJZX-c5H_I-TXeLrvrw3OvgAWgmxYO5nTdhfQ,4918 +sentry_sdk/integrations/django/views.py,sha256=Cz2H_yeZkPixOksVoi5urp6Xeq5p0YdDsBWz5aEJan4,3256 +sentry_sdk/integrations/dramatiq.py,sha256=OIq0lIUbabt9widLKGkyjhvx9MKXNF9atP1gTsLheuQ,7438 +sentry_sdk/integrations/excepthook.py,sha256=lhbIwr1eIj3k8shtmUGDOWI7Qr-OaT5kbgeHj-F217g,2371 +sentry_sdk/integrations/executing.py,sha256=O5R4pw999PIisy7uksrMkRx0SsmfBFKI1BEpI3cTDz4,1981 +sentry_sdk/integrations/falcon.py,sha256=zWKFTzoOY92BryGKsjKDAhVxFDhyop4G02js6OlmIXY,9277 +sentry_sdk/integrations/fastapi.py,sha256=eebHCt2FiR7YQCaRJVOnXurltMZ4phRCZaL6BXSEpVI,4485 +sentry_sdk/integrations/flask.py,sha256=KGG2Jbcw9SdwADJLg5GmBnVq3VywK7qQwdPv97RldmA,8519 +sentry_sdk/integrations/gcp.py,sha256=Fao0nrrBpiG8_GHygzh6VvmM8bpJq0mkvgTwu5LT1a4,8400 +sentry_sdk/integrations/gnu_backtrace.py,sha256=XND4pkVeprhDdvPgOi8pBgQQ9g3CtPA-wJK0e0ex1XE,2771 +sentry_sdk/integrations/google_genai/__init__.py,sha256=Z-NX8wKGkSwlQupCcCgsarGV3qtpmeiT_wGNqpxweSM,14093 +sentry_sdk/integrations/google_genai/__pycache__/__init__.cpython-312.pyc,, +sentry_sdk/integrations/google_genai/__pycache__/consts.cpython-312.pyc,, +sentry_sdk/integrations/google_genai/__pycache__/streaming.cpython-312.pyc,, +sentry_sdk/integrations/google_genai/__pycache__/utils.cpython-312.pyc,, +sentry_sdk/integrations/google_genai/consts.py,sha256=nqHKKSyGixrSoozA06BGVBFaUCsvZlvGoubUZGI1kB8,559 +sentry_sdk/integrations/google_genai/streaming.py,sha256=JqhTcAX3DtEsZ033AXsnA0KAzQe_jqca-7cztTKDcRI,5416 +sentry_sdk/integrations/google_genai/utils.py,sha256=WCTyevkQF2CdGpdlidONJSWSdDmnqEa0dcaiRX2luuk,34850 +sentry_sdk/integrations/gql.py,sha256=oLssOmHYQ4e0OQ1ITer8TynvyLXtDUoM9X29-IsVAkI,5003 +sentry_sdk/integrations/graphene.py,sha256=11XRvAsSgvCLcfF1xvdSeAf0vvJm72hfcE3toAS8jZI,5066 +sentry_sdk/integrations/grpc/__init__.py,sha256=0Y3neOWwES1PsWJsluPF_2_VatFlVvg7IUjTYYdCs-U,6253 +sentry_sdk/integrations/grpc/__pycache__/__init__.cpython-312.pyc,, +sentry_sdk/integrations/grpc/__pycache__/client.cpython-312.pyc,, +sentry_sdk/integrations/grpc/__pycache__/consts.cpython-312.pyc,, +sentry_sdk/integrations/grpc/__pycache__/server.cpython-312.pyc,, +sentry_sdk/integrations/grpc/aio/__init__.py,sha256=2rgrliowpPfLLw40_2YU6ixSzIu_3f8NN3TRplzc8S8,141 +sentry_sdk/integrations/grpc/aio/__pycache__/__init__.cpython-312.pyc,, +sentry_sdk/integrations/grpc/aio/__pycache__/client.cpython-312.pyc,, +sentry_sdk/integrations/grpc/aio/__pycache__/server.cpython-312.pyc,, +sentry_sdk/integrations/grpc/aio/client.py,sha256=2GNrTWJt4iZ9dK-M3_0-sVWlUcfqKItHZI5u-uzesUM,3507 +sentry_sdk/integrations/grpc/aio/server.py,sha256=cRBvz9GOJemvNOsCB9ymTUX70nhjljfzvX0bfR8Wc_o,3954 +sentry_sdk/integrations/grpc/client.py,sha256=s7yLcbYgnq1-bdh6jvSOv6NwKTx0m9tLPt0q4qKI7kw,3404 +sentry_sdk/integrations/grpc/consts.py,sha256=NpsN5gKWDmtGtVK_L5HscgFZBHqjOpmLJLGKyh8GZBA,31 +sentry_sdk/integrations/grpc/server.py,sha256=h6vEibthsZB6hWNmqT50UlBaddEZSdbXio_rYhighW0,2470 +sentry_sdk/integrations/httpx.py,sha256=MxoT_D8n902awI2hq-lHpxXkOvNwGZFuG9ap4l4VxL4,5315 +sentry_sdk/integrations/huey.py,sha256=GOslGxuYfrdPLqx_4MWOH_tdwwnbD72SkTdc-P8uF80,5348 +sentry_sdk/integrations/huggingface_hub.py,sha256=axCqdDlmYn7wANHgVBo3MDv9QgBr0MWj3lBYhPJJX7E,14966 +sentry_sdk/integrations/langchain.py,sha256=Oa-YJ2WoHX8Huyrzs4aQ2xlaupFR5Ft8FC4_0b_zvBs,42903 +sentry_sdk/integrations/langgraph.py,sha256=xb9InjYjd4jAQECWddl30QGRoHRbbhoYBeHscsM7C1A,13301 +sentry_sdk/integrations/launchdarkly.py,sha256=RSTCAUOJCo6Npr4V3-7-vSOaZ531Jj1LNU5IftXL6_w,1918 +sentry_sdk/integrations/litellm.py,sha256=1CLM3CBdIv0YZSloDPCfemIdlufNG41eWvCBRG_AFow,11550 +sentry_sdk/integrations/litestar.py,sha256=oJIz1vv2JgIT_AriKXDTQvQjkMOmwKfT68DFrCfdJ-8,11698 +sentry_sdk/integrations/logging.py,sha256=eLjza7xDrfBszOH4gDHxPtSM3WfiWsw9WGR3GUfnA-A,13739 +sentry_sdk/integrations/loguru.py,sha256=2UaLLzXVEOV0cYxOU5n9GpkmNtC_bA1KzCzEnrqIilE,6505 +sentry_sdk/integrations/mcp.py,sha256=4t7cVGwqjoqyNSd4HrwzjfM7fw2B5HYWRJoT6wOw6AA,22192 +sentry_sdk/integrations/modules.py,sha256=7kmS9T-werRkKN0w51skyDmTMuZbdz9L28RgXPsoDeU,786 +sentry_sdk/integrations/openai.py,sha256=8UquHKCu17tS6Afs90HLDa4mpx-7ErRl4nUh1OKwdbE,33167 +sentry_sdk/integrations/openai_agents/__init__.py,sha256=EzJ9SQgB22IeXAXVVFRHKYoegKef-IN33FX1O6qG_LQ,1999 +sentry_sdk/integrations/openai_agents/__pycache__/__init__.cpython-312.pyc,, +sentry_sdk/integrations/openai_agents/__pycache__/consts.cpython-312.pyc,, +sentry_sdk/integrations/openai_agents/__pycache__/utils.cpython-312.pyc,, +sentry_sdk/integrations/openai_agents/consts.py,sha256=PTb3vlqkuMPktu21ALK72o5WMIX4-cewTEiTRdHKFdQ,38 +sentry_sdk/integrations/openai_agents/patches/__init__.py,sha256=kistNkqrQsQdxd39bm7PCOjfqX8pBJ0XsfFJOAE-jKU,323 +sentry_sdk/integrations/openai_agents/patches/__pycache__/__init__.cpython-312.pyc,, +sentry_sdk/integrations/openai_agents/patches/__pycache__/agent_run.cpython-312.pyc,, +sentry_sdk/integrations/openai_agents/patches/__pycache__/error_tracing.cpython-312.pyc,, +sentry_sdk/integrations/openai_agents/patches/__pycache__/models.cpython-312.pyc,, +sentry_sdk/integrations/openai_agents/patches/__pycache__/runner.cpython-312.pyc,, +sentry_sdk/integrations/openai_agents/patches/__pycache__/tools.cpython-312.pyc,, +sentry_sdk/integrations/openai_agents/patches/agent_run.py,sha256=I1zOUv2yGDxLYmcUhBWy3WtL_FgAAGqDwn8O-lppFrs,9667 +sentry_sdk/integrations/openai_agents/patches/error_tracing.py,sha256=Fzs4oAyGWnmbaeLMMRnr9m6F6M2N9Cm0i4GSPsJbNqI,2237 +sentry_sdk/integrations/openai_agents/patches/models.py,sha256=FutMf_-VYDmsrJXWwwE8DA2nqtTkhwU4riR8w78VBmw,7868 +sentry_sdk/integrations/openai_agents/patches/runner.py,sha256=DlG9Qb6dzjWjbYSylQ9KlL2T-LJr5dimWE45yLrAR1A,6643 +sentry_sdk/integrations/openai_agents/patches/tools.py,sha256=k0DHyusZEHN6Ei0iPgHM3457N-DAzaePek2O9X61IEU,3297 +sentry_sdk/integrations/openai_agents/spans/__init__.py,sha256=uzqInMjqfA_MtdTJoRRH10Z4cVtVESbi7qc5sC_zr8U,393 +sentry_sdk/integrations/openai_agents/spans/__pycache__/__init__.cpython-312.pyc,, +sentry_sdk/integrations/openai_agents/spans/__pycache__/agent_workflow.cpython-312.pyc,, +sentry_sdk/integrations/openai_agents/spans/__pycache__/ai_client.cpython-312.pyc,, +sentry_sdk/integrations/openai_agents/spans/__pycache__/execute_tool.cpython-312.pyc,, +sentry_sdk/integrations/openai_agents/spans/__pycache__/handoff.cpython-312.pyc,, +sentry_sdk/integrations/openai_agents/spans/__pycache__/invoke_agent.cpython-312.pyc,, +sentry_sdk/integrations/openai_agents/spans/agent_workflow.py,sha256=lqr87hbaV1GCXX40cGQLSlf3U6V-nmD2zj5kr2w9kmk,459 +sentry_sdk/integrations/openai_agents/spans/ai_client.py,sha256=RbJNDn9DXM4NJLHJBIodRp1GK-kQmclYCD5DgOV2WTI,2265 +sentry_sdk/integrations/openai_agents/spans/execute_tool.py,sha256=ePm9PMnEzoYZW8M04dnDRINV7dRnDKtHa4YFn1InqZY,1612 +sentry_sdk/integrations/openai_agents/spans/handoff.py,sha256=icCbs6HOFrVXY4V1LILCoyV-qxNntIBhgx8AFHW6MGo,723 +sentry_sdk/integrations/openai_agents/spans/invoke_agent.py,sha256=VWB1HmR3oyoTqi3ZrHIHY37ageQnPsap5C81bLcxrOA,3592 +sentry_sdk/integrations/openai_agents/utils.py,sha256=MvbPG-dh5UJzE50y-Q9xUeSuuu_Ot642SvD1H0hIcuY,8910 +sentry_sdk/integrations/openfeature.py,sha256=w6bYjrHJdjIpziEvEBocCbxdryO2eDZGGOoc7g8jNEU,1102 +sentry_sdk/integrations/opentelemetry/__init__.py,sha256=emNL5aAq_NhK0PZmfX_g4GIdvBS6nHqGrjrIgrdC5m8,229 +sentry_sdk/integrations/opentelemetry/__pycache__/__init__.cpython-312.pyc,, +sentry_sdk/integrations/opentelemetry/__pycache__/consts.cpython-312.pyc,, +sentry_sdk/integrations/opentelemetry/__pycache__/integration.cpython-312.pyc,, +sentry_sdk/integrations/opentelemetry/__pycache__/propagator.cpython-312.pyc,, +sentry_sdk/integrations/opentelemetry/__pycache__/span_processor.cpython-312.pyc,, +sentry_sdk/integrations/opentelemetry/consts.py,sha256=6yhH65VVzzIjpU8MRYItLKkskkBsLlbsO-_2MULlEHQ,275 +sentry_sdk/integrations/opentelemetry/integration.py,sha256=VPQlj-tDBgjSRXOxtrxeFpv9Mc_OqX7cC997gMy4CkY,1742 +sentry_sdk/integrations/opentelemetry/propagator.py,sha256=ofVK4IJ9zhSN0cRmw_Cr15IHK0XckmBy8WqA_oaGa5Q,3973 +sentry_sdk/integrations/opentelemetry/span_processor.py,sha256=YK-k88VMoJNrOEXMqYaVM_Y-FywN2hQjdrodLeqGegw,13222 +sentry_sdk/integrations/otlp.py,sha256=5h0hFY19myYSg_buSAutomC_8_BfRtDyHyLE9XD_A8A,7713 +sentry_sdk/integrations/pure_eval.py,sha256=SLN-omknLQWyeH1_3v3Qi-puFCAliYKQqi0wdDt3Keg,4560 +sentry_sdk/integrations/pydantic_ai/__init__.py,sha256=YrUxvNgPIznxzVaniyLTorkkTmlYyosAbA37tFOTR68,1465 +sentry_sdk/integrations/pydantic_ai/__pycache__/__init__.cpython-312.pyc,, +sentry_sdk/integrations/pydantic_ai/__pycache__/consts.cpython-312.pyc,, +sentry_sdk/integrations/pydantic_ai/__pycache__/utils.cpython-312.pyc,, +sentry_sdk/integrations/pydantic_ai/consts.py,sha256=fxOQ5n_Do8EqqqxtOJm5zyvhQmOV75HACNrt_-zGngs,36 +sentry_sdk/integrations/pydantic_ai/patches/__init__.py,sha256=_RHvjc3436KSwPjzrAdnyascgggxg5e0MQpdHhmiS-U,229 +sentry_sdk/integrations/pydantic_ai/patches/__pycache__/__init__.cpython-312.pyc,, +sentry_sdk/integrations/pydantic_ai/patches/__pycache__/agent_run.cpython-312.pyc,, +sentry_sdk/integrations/pydantic_ai/patches/__pycache__/graph_nodes.cpython-312.pyc,, +sentry_sdk/integrations/pydantic_ai/patches/__pycache__/model_request.cpython-312.pyc,, +sentry_sdk/integrations/pydantic_ai/patches/__pycache__/tools.cpython-312.pyc,, +sentry_sdk/integrations/pydantic_ai/patches/agent_run.py,sha256=iGafWMXefgs_1HQfV_rK4Pe5wJex4vLbAdX-O83fxQQ,7345 +sentry_sdk/integrations/pydantic_ai/patches/graph_nodes.py,sha256=TDkIjRUxGGt0UbC3FzgaCJNJN1s682IXoVFO07biL6E,3793 +sentry_sdk/integrations/pydantic_ai/patches/model_request.py,sha256=15uGRv0UXjZcEc9b6Jzx7o9yhToUQfd4KRLZm_OWMm8,1240 +sentry_sdk/integrations/pydantic_ai/patches/tools.py,sha256=soSA-ugtq-EdkkbfXV_mAPC-AfEaCM_6E_c1kvg_IQk,3822 +sentry_sdk/integrations/pydantic_ai/spans/__init__.py,sha256=dTUjvkw7VMOAiSasuAq37q_njvANsUlgfZxgXRKJDDo,243 +sentry_sdk/integrations/pydantic_ai/spans/__pycache__/__init__.cpython-312.pyc,, +sentry_sdk/integrations/pydantic_ai/spans/__pycache__/ai_client.cpython-312.pyc,, +sentry_sdk/integrations/pydantic_ai/spans/__pycache__/execute_tool.cpython-312.pyc,, +sentry_sdk/integrations/pydantic_ai/spans/__pycache__/invoke_agent.cpython-312.pyc,, +sentry_sdk/integrations/pydantic_ai/spans/__pycache__/utils.cpython-312.pyc,, +sentry_sdk/integrations/pydantic_ai/spans/ai_client.py,sha256=aAEjW2eh317ZFQJwJi2_jJHnWaKtdWdJFkuaU-BFbj8,10508 +sentry_sdk/integrations/pydantic_ai/spans/execute_tool.py,sha256=0k-eIN1tltSOQf2kFdUMGKbyWhKYTNZE8dJF9vkOfnE,1542 +sentry_sdk/integrations/pydantic_ai/spans/invoke_agent.py,sha256=Mbq_7NkqD5oyjY-_AOhblliYNT_Ij6z-MEMfZl4xyyY,5742 +sentry_sdk/integrations/pydantic_ai/spans/utils.py,sha256=VSWLcgjrUwNSR_31TFM5BRG0EsGrc3X8BWIxM4v0HZI,1763 +sentry_sdk/integrations/pydantic_ai/utils.py,sha256=OWFa15_jQ7H4gHfMAUQHmSVclLVrF_DBb7heaPZ89HY,7091 +sentry_sdk/integrations/pymongo.py,sha256=whIHxnsiz245UNT7yKMP1OqfAsBdC7ITy7kuice0uwE,6275 +sentry_sdk/integrations/pyramid.py,sha256=EzK6W2VB4DdDfV_VXnp5ERibM27mVK-JGib3k9Khc4E,7208 +sentry_sdk/integrations/quart.py,sha256=4GTx6E7auhLeYXyY8WoloqSghbeU14Jcz-gfDL4o3aE,7258 +sentry_sdk/integrations/ray.py,sha256=8zLUo2ZFl9KEt88X-yJYVVoDphJUTWRZRT0I5OQ1PCk,5884 +sentry_sdk/integrations/redis/__init__.py,sha256=TzeEWrYcDO9u3q5WXT8J9Zk5Ft9nrWW1PsE3i0ULGUg,1662 +sentry_sdk/integrations/redis/__pycache__/__init__.cpython-312.pyc,, +sentry_sdk/integrations/redis/__pycache__/_async_common.cpython-312.pyc,, +sentry_sdk/integrations/redis/__pycache__/_sync_common.cpython-312.pyc,, +sentry_sdk/integrations/redis/__pycache__/consts.cpython-312.pyc,, +sentry_sdk/integrations/redis/__pycache__/rb.cpython-312.pyc,, +sentry_sdk/integrations/redis/__pycache__/redis.cpython-312.pyc,, +sentry_sdk/integrations/redis/__pycache__/redis_cluster.cpython-312.pyc,, +sentry_sdk/integrations/redis/__pycache__/redis_py_cluster_legacy.cpython-312.pyc,, +sentry_sdk/integrations/redis/__pycache__/utils.cpython-312.pyc,, +sentry_sdk/integrations/redis/_async_common.py,sha256=V05_CgSyWBKBLQ4Udkg4LT5ovPAnP5iqsvfeqsmLEwU,4058 +sentry_sdk/integrations/redis/_sync_common.py,sha256=3seE5lTc2fDNLXutpe41Qv-3pdRVbLHFGCQaT9OXzvc,3776 +sentry_sdk/integrations/redis/consts.py,sha256=y2f-FJ7TIkzto01tyjXvbKVSVELVkjZaxj3FG5DZ0hA,480 +sentry_sdk/integrations/redis/modules/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +sentry_sdk/integrations/redis/modules/__pycache__/__init__.cpython-312.pyc,, +sentry_sdk/integrations/redis/modules/__pycache__/caches.cpython-312.pyc,, +sentry_sdk/integrations/redis/modules/__pycache__/queries.cpython-312.pyc,, +sentry_sdk/integrations/redis/modules/caches.py,sha256=cV64P6UWHXolvTtog5jQFHtR4XFd5IqA5vVYXc8xO2E,4020 +sentry_sdk/integrations/redis/modules/queries.py,sha256=lRwn72AUvk5a29n43UEpx6l9ix_xC2htHcl4cHKsdcw,1941 +sentry_sdk/integrations/redis/rb.py,sha256=we3oh4fEhiirglutAllZe55sAoz6vKtaQt1tvLPGoX4,791 +sentry_sdk/integrations/redis/redis.py,sha256=NaaefF_0poC8-9citN6etyAC6WOabxBVOdH_J8FJ7ec,1684 +sentry_sdk/integrations/redis/redis_cluster.py,sha256=rBbj1WRa_-NAR6tTQHKQMF40WjvNb6o6FoojImIE03Y,3530 +sentry_sdk/integrations/redis/redis_py_cluster_legacy.py,sha256=YREwCLBkx_SDVkzNLGNcgCr4yi_mdomlwgKlVU8bgSU,1570 +sentry_sdk/integrations/redis/utils.py,sha256=i818myPF61yl4EIS-t3ptyYCa2IrsaAWeFR0cqS0QNY,3954 +sentry_sdk/integrations/rq.py,sha256=wyerb5iiAJbekL5-kdOpwQzQiJiWIRlX49il1aEnUWg,5278 +sentry_sdk/integrations/rust_tracing.py,sha256=p3aFy8Gs8Dgvvlv9PSrbl7JDYmA5zus-fTichRzTiLM,9101 +sentry_sdk/integrations/sanic.py,sha256=OJ570xdV4luxJY9_b4gutVf8gQhNpqyZHxyT3evydLM,12700 +sentry_sdk/integrations/serverless.py,sha256=eZSd0NPYBm3ZGdYFmDYlXDtAD_Pe_aUGV77bCzJhoEI,1621 +sentry_sdk/integrations/socket.py,sha256=fj-4yzw7uecKDa4Jl_8AN_Nb9KSIvCh3iR8oqaLsYaQ,3164 +sentry_sdk/integrations/spark/__init__.py,sha256=oOewMErnZk2rzNvIlZO6URxQexu9bUJuSLM2m_zECy8,208 +sentry_sdk/integrations/spark/__pycache__/__init__.cpython-312.pyc,, +sentry_sdk/integrations/spark/__pycache__/spark_driver.cpython-312.pyc,, +sentry_sdk/integrations/spark/__pycache__/spark_worker.cpython-312.pyc,, +sentry_sdk/integrations/spark/spark_driver.py,sha256=mYX1ohjUD7wrc7F16OhvNyyOGqgQ7QM6bFzfBb_MUZY,8896 +sentry_sdk/integrations/spark/spark_worker.py,sha256=PtRzOBjrG0__Au-Uidd8cO5zpx-EJY_D13H9CG900RE,3632 +sentry_sdk/integrations/sqlalchemy.py,sha256=Q6JQxDcwo2U-3ggB_wUhQZps1gzzXz6-Yf20dcCMUWU,4347 +sentry_sdk/integrations/starlette.py,sha256=1cSqN0IARPGi6kLitW93ASlTJIAFFUxsyWnS8QMMzJY,26130 +sentry_sdk/integrations/starlite.py,sha256=GC_FaL-cyjeUtsRIJ29jMR4HlYXRwI-JRYy7GFn5QAI,10451 +sentry_sdk/integrations/statsig.py,sha256=ceSjGymN4Fqdnk8Tz-L5teHz0eNOJtptB_W0Y1VZu2o,1214 +sentry_sdk/integrations/stdlib.py,sha256=2yN0_v-fokZwgfnsKBvcIdl4tQ-QpLUnoQzYyOX_gH4,9376 +sentry_sdk/integrations/strawberry.py,sha256=cTY1L8ihyR5mCiF2H5ZEM59t0BMwNmWqwNkTIuIrFUA,14120 +sentry_sdk/integrations/sys_exit.py,sha256=nLbBnbyPjUgyhsI1VkmTxVAOgdlenf-dFtIqzFTZNOI,2408 +sentry_sdk/integrations/threading.py,sha256=jUsmnGEPoi6XGhuLZP3pWKJCDksul-e7uLwAyPK_vUI,7107 +sentry_sdk/integrations/tornado.py,sha256=jh_tm6UR-m-OPFlTKD4eOYEsyex2Lm9rw7_UpZ-1dCQ,7212 +sentry_sdk/integrations/trytond.py,sha256=FjAKIKDTRmEc1JUFsHkcZhslCOD7RsT-pKYoSg1Qyuk,1750 +sentry_sdk/integrations/typer.py,sha256=hvLL7llTverbhjdC8eU5r0ps39XVf2qI27C35VWyXrE,1833 +sentry_sdk/integrations/unleash.py,sha256=Gz3qAKB0fTtOgzCOJV24ocW1NbdcDkkqijrWDE6Oi7E,1045 +sentry_sdk/integrations/unraisablehook.py,sha256=zThW8f6uFGFfwpPHFXq_pLbnaQRT-1tmltyOPVfTPzM,1717 +sentry_sdk/integrations/wsgi.py,sha256=w6BKB-aCaGKl6tfWlKL2SlifY4ZErQaEH5oCXg_bcFg,10503 +sentry_sdk/logger.py,sha256=3hfleFTl9n6ZlZeb-F8jVVMGoae_SU8fOZd0N6HWbuA,2683 +sentry_sdk/metrics.py,sha256=z8futlRltjsPeTkU_We4HAanxeopDaWtfJTkAumwL68,1470 +sentry_sdk/monitor.py,sha256=YRmomlJySjuuCHLJPgGMhRbHXun651He5QZXUvKWZp0,3443 +sentry_sdk/profiler/__init__.py,sha256=3PI3bHk9RSkkOXZKN84DDedk_7M65EiqqaIGo-DYs0E,1291 +sentry_sdk/profiler/__pycache__/__init__.cpython-312.pyc,, +sentry_sdk/profiler/__pycache__/continuous_profiler.cpython-312.pyc,, +sentry_sdk/profiler/__pycache__/transaction_profiler.cpython-312.pyc,, +sentry_sdk/profiler/__pycache__/utils.cpython-312.pyc,, +sentry_sdk/profiler/continuous_profiler.py,sha256=lRZa2yF81ByIh1yQVHdsK-Ur5jRMR6aKJuPCkuHn0pI,22550 +sentry_sdk/profiler/transaction_profiler.py,sha256=AJicxLx2aPcnTBm5E6GGx_34j-y2yD907_OsHtN6xyQ,27248 +sentry_sdk/profiler/utils.py,sha256=Y-GN2SoAKa12nxwUQRvI5LTYEWfIkhOvoCPG6B4K_N0,6397 +sentry_sdk/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +sentry_sdk/scope.py,sha256=HtA_ivY_Syuw2sxXiXCgp1PsRg46CetAuVeth_slfzk,68615 +sentry_sdk/scrubber.py,sha256=pEJECHaWzg0vf93S_uYe9PgStTImDXjLFu0pzIJ6XAM,5953 +sentry_sdk/serializer.py,sha256=W3HrcoJ9iMzHa6j9Nmu5Y7Qopbkon3ovxuPxRM5R4pU,13199 +sentry_sdk/session.py,sha256=ajYuxKt5cKfxl_OjcF0-g5IpowrJhvMe9rjTq1CaQac,5274 +sentry_sdk/sessions.py,sha256=-k0OGB28DoWbb7nIW4ML8R_fiF5nLnboWqObnfyZnWQ,8932 +sentry_sdk/spotlight.py,sha256=0bJrMpoT47xS2LJzpDfUZ5z5YEhdUla78r0YDtVm2Zo,12191 +sentry_sdk/traces.py,sha256=KVPac65D2T0hqllgR6ESNjoh6nVVkFtzkXNKKshRuD0,1821 +sentry_sdk/tracing.py,sha256=OkIHt-BYfYhLe9MyEcTUlgN_n0CjqQn-BJ0K_9gnxaY,50512 +sentry_sdk/tracing_utils.py,sha256=aItaWdi76Q0-HGsILrkNesctXp76BkriTol-jXSZjfI,43300 +sentry_sdk/transport.py,sha256=P21HMCcD58PJfRNPA7Y39hlMqIFK5RsPCMwsy-QzXAQ,32141 +sentry_sdk/types.py,sha256=A92AqvfrGQZ9KY6FaUjKfL9F1HK7Ui3heQilVzfzYCs,1269 +sentry_sdk/utils.py,sha256=h5Srrup8vTtS4tQ7U9l_0265H2gGXlifUjbjV6a0Wv4,65339 +sentry_sdk/worker.py,sha256=djRCygOJFRmdXBS4lRSF4RTljVZ47BWEal85WlTD0uo,4257 diff --git a/lib/python3.12/site-packages/sentry_sdk-2.52.0.dist-info/WHEEL b/lib/python3.12/site-packages/sentry_sdk-2.52.0.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..798dd6195b88f8618eca3eb9b05be33268fa3dac --- /dev/null +++ b/lib/python3.12/site-packages/sentry_sdk-2.52.0.dist-info/WHEEL @@ -0,0 +1,6 @@ +Wheel-Version: 1.0 +Generator: setuptools (80.10.2) +Root-Is-Purelib: true +Tag: py2-none-any +Tag: py3-none-any + diff --git a/lib/python3.12/site-packages/sentry_sdk-2.52.0.dist-info/entry_points.txt b/lib/python3.12/site-packages/sentry_sdk-2.52.0.dist-info/entry_points.txt new file mode 100644 index 0000000000000000000000000000000000000000..4b128a843cbaf61581876070bb4bbe709ce1c0b5 --- /dev/null +++ b/lib/python3.12/site-packages/sentry_sdk-2.52.0.dist-info/entry_points.txt @@ -0,0 +1,2 @@ +[opentelemetry_propagator] +sentry = sentry_sdk.integrations.opentelemetry:SentryPropagator diff --git a/lib/python3.12/site-packages/sentry_sdk-2.52.0.dist-info/licenses/LICENSE b/lib/python3.12/site-packages/sentry_sdk-2.52.0.dist-info/licenses/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..016323bd8da88e224808bebf892dabc485c85387 --- /dev/null +++ b/lib/python3.12/site-packages/sentry_sdk-2.52.0.dist-info/licenses/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018 Functional Software, Inc. dba Sentry + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/lib/python3.12/site-packages/sentry_sdk-2.52.0.dist-info/top_level.txt b/lib/python3.12/site-packages/sentry_sdk-2.52.0.dist-info/top_level.txt new file mode 100644 index 0000000000000000000000000000000000000000..5051901ecb46b7905ec9fcb8dfa5c2711d494b8d --- /dev/null +++ b/lib/python3.12/site-packages/sentry_sdk-2.52.0.dist-info/top_level.txt @@ -0,0 +1 @@ +sentry_sdk