ZhengyangZhang commited on
Commit
a102c6e
·
verified ·
1 Parent(s): 9babaa7

Add files using upload-large-folder tool

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. lib/python3.12/site-packages/datasets/__init__.py +47 -0
  2. lib/python3.12/site-packages/datasets/arrow_dataset.py +0 -0
  3. lib/python3.12/site-packages/datasets/arrow_reader.py +620 -0
  4. lib/python3.12/site-packages/datasets/arrow_writer.py +677 -0
  5. lib/python3.12/site-packages/datasets/builder.py +1903 -0
  6. lib/python3.12/site-packages/datasets/combine.py +215 -0
  7. lib/python3.12/site-packages/datasets/commands/__init__.py +13 -0
  8. lib/python3.12/site-packages/datasets/commands/__pycache__/__init__.cpython-312.pyc +0 -0
  9. lib/python3.12/site-packages/datasets/commands/__pycache__/convert.cpython-312.pyc +0 -0
  10. lib/python3.12/site-packages/datasets/commands/__pycache__/convert_to_parquet.cpython-312.pyc +0 -0
  11. lib/python3.12/site-packages/datasets/commands/__pycache__/datasets_cli.cpython-312.pyc +0 -0
  12. lib/python3.12/site-packages/datasets/commands/__pycache__/delete_from_hub.cpython-312.pyc +0 -0
  13. lib/python3.12/site-packages/datasets/commands/__pycache__/env.cpython-312.pyc +0 -0
  14. lib/python3.12/site-packages/datasets/commands/__pycache__/test.cpython-312.pyc +0 -0
  15. lib/python3.12/site-packages/datasets/commands/convert.py +195 -0
  16. lib/python3.12/site-packages/datasets/commands/convert_to_parquet.py +46 -0
  17. lib/python3.12/site-packages/datasets/commands/datasets_cli.py +43 -0
  18. lib/python3.12/site-packages/datasets/commands/delete_from_hub.py +42 -0
  19. lib/python3.12/site-packages/datasets/commands/env.py +41 -0
  20. lib/python3.12/site-packages/datasets/commands/test.py +207 -0
  21. lib/python3.12/site-packages/datasets/config.py +269 -0
  22. lib/python3.12/site-packages/datasets/data_files.py +795 -0
  23. lib/python3.12/site-packages/datasets/dataset_dict.py +0 -0
  24. lib/python3.12/site-packages/datasets/distributed.py +39 -0
  25. lib/python3.12/site-packages/datasets/exceptions.py +119 -0
  26. lib/python3.12/site-packages/datasets/fingerprint.py +454 -0
  27. lib/python3.12/site-packages/datasets/hub.py +230 -0
  28. lib/python3.12/site-packages/datasets/info.py +430 -0
  29. lib/python3.12/site-packages/datasets/inspect.py +363 -0
  30. lib/python3.12/site-packages/datasets/iterable_dataset.py +0 -0
  31. lib/python3.12/site-packages/datasets/keyhash.py +104 -0
  32. lib/python3.12/site-packages/datasets/load.py +0 -0
  33. lib/python3.12/site-packages/datasets/naming.py +84 -0
  34. lib/python3.12/site-packages/datasets/packaged_modules/__init__.py +104 -0
  35. lib/python3.12/site-packages/datasets/packaged_modules/csv/__init__.py +0 -0
  36. lib/python3.12/site-packages/datasets/packaged_modules/csv/__pycache__/__init__.cpython-312.pyc +0 -0
  37. lib/python3.12/site-packages/datasets/packaged_modules/csv/__pycache__/csv.cpython-312.pyc +0 -0
  38. lib/python3.12/site-packages/datasets/packaged_modules/csv/csv.py +198 -0
  39. lib/python3.12/site-packages/datasets/packaged_modules/folder_based_builder/__init__.py +0 -0
  40. lib/python3.12/site-packages/datasets/packaged_modules/folder_based_builder/__pycache__/__init__.cpython-312.pyc +0 -0
  41. lib/python3.12/site-packages/datasets/packaged_modules/folder_based_builder/__pycache__/folder_based_builder.cpython-312.pyc +0 -0
  42. lib/python3.12/site-packages/datasets/packaged_modules/folder_based_builder/folder_based_builder.py +417 -0
  43. lib/python3.12/site-packages/datasets/packaged_modules/generator/__init__.py +0 -0
  44. lib/python3.12/site-packages/datasets/packaged_modules/generator/__pycache__/generator.cpython-312.pyc +0 -0
  45. lib/python3.12/site-packages/datasets/packaged_modules/generator/generator.py +33 -0
  46. lib/python3.12/site-packages/datasets/packaged_modules/imagefolder/__init__.py +0 -0
  47. lib/python3.12/site-packages/datasets/packaged_modules/imagefolder/__pycache__/__init__.cpython-312.pyc +0 -0
  48. lib/python3.12/site-packages/datasets/packaged_modules/imagefolder/__pycache__/imagefolder.cpython-312.pyc +0 -0
  49. lib/python3.12/site-packages/datasets/packaged_modules/imagefolder/imagefolder.py +103 -0
  50. lib/python3.12/site-packages/datasets/packaged_modules/sql/__init__.py +0 -0
lib/python3.12/site-packages/datasets/__init__.py ADDED
@@ -0,0 +1,47 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ __version__ = "3.6.0"
16
+
17
+ from .arrow_dataset import Dataset
18
+ from .arrow_reader import ReadInstruction
19
+ from .builder import ArrowBasedBuilder, BuilderConfig, DatasetBuilder, GeneratorBasedBuilder
20
+ from .combine import concatenate_datasets, interleave_datasets
21
+ from .dataset_dict import DatasetDict, IterableDatasetDict
22
+ from .download import *
23
+ from .features import *
24
+ from .fingerprint import disable_caching, enable_caching, is_caching_enabled
25
+ from .info import DatasetInfo
26
+ from .inspect import (
27
+ get_dataset_config_info,
28
+ get_dataset_config_names,
29
+ get_dataset_default_config_name,
30
+ get_dataset_infos,
31
+ get_dataset_split_names,
32
+ )
33
+ from .iterable_dataset import IterableDataset
34
+ from .load import load_dataset, load_dataset_builder, load_from_disk
35
+ from .splits import (
36
+ NamedSplit,
37
+ NamedSplitAll,
38
+ Split,
39
+ SplitBase,
40
+ SplitDict,
41
+ SplitGenerator,
42
+ SplitInfo,
43
+ SubSplitInfo,
44
+ percent,
45
+ )
46
+ from .utils import *
47
+ from .utils import logging
lib/python3.12/site-packages/datasets/arrow_dataset.py ADDED
The diff for this file is too large to render. See raw diff
 
lib/python3.12/site-packages/datasets/arrow_reader.py ADDED
@@ -0,0 +1,620 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ # Lint as: python3
16
+ """Arrow ArrowReader."""
17
+
18
+ import copy
19
+ import math
20
+ import os
21
+ import re
22
+ from dataclasses import dataclass
23
+ from functools import partial
24
+ from typing import TYPE_CHECKING, Optional, Union
25
+
26
+ import pyarrow as pa
27
+ import pyarrow.parquet as pq
28
+ from tqdm.contrib.concurrent import thread_map
29
+
30
+ from .download.download_config import DownloadConfig # noqa: F401
31
+ from .naming import _split_re, filenames_for_dataset_split
32
+ from .table import InMemoryTable, MemoryMappedTable, Table, concat_tables
33
+ from .utils import logging
34
+ from .utils import tqdm as hf_tqdm
35
+
36
+
37
+ if TYPE_CHECKING:
38
+ from .info import DatasetInfo # noqa: F401
39
+ from .splits import Split, SplitInfo # noqa: F401
40
+
41
+
42
+ logger = logging.get_logger(__name__)
43
+
44
+ HF_GCP_BASE_URL = "https://storage.googleapis.com/huggingface-nlp/cache/datasets"
45
+
46
+ _SUB_SPEC_RE = re.compile(
47
+ rf"""
48
+ ^
49
+ (?P<split>{_split_re[1:-1]})
50
+ (\[
51
+ ((?P<from>-?[\d_]+)
52
+ (?P<from_pct>%)?)?
53
+ :
54
+ ((?P<to>-?[\d_]+)
55
+ (?P<to_pct>%)?)?
56
+ \])?(\((?P<rounding>[^\)]*)\))?
57
+ $
58
+ """, # remove ^ and $
59
+ re.X,
60
+ )
61
+
62
+ _ADDITION_SEP_RE = re.compile(r"\s*\+\s*")
63
+
64
+
65
+ class DatasetNotOnHfGcsError(ConnectionError):
66
+ """When you can't get the dataset from the Hf google cloud storage"""
67
+
68
+ pass
69
+
70
+
71
+ class MissingFilesOnHfGcsError(ConnectionError):
72
+ """When some files are missing on the Hf oogle cloud storage"""
73
+
74
+ pass
75
+
76
+
77
+ @dataclass(frozen=True)
78
+ class FileInstructions:
79
+ """The file instructions associated with a split ReadInstruction.
80
+
81
+ Attributes:
82
+ num_examples: `int`, The total number of examples
83
+ file_instructions: List[dict(filename, skip, take)], the files information.
84
+ The filenames contains the relative path, not absolute.
85
+ skip/take indicates which example read in the file: `ds.slice(skip, take)`
86
+ """
87
+
88
+ num_examples: int
89
+ file_instructions: list[dict]
90
+
91
+
92
+ def make_file_instructions(
93
+ name: str,
94
+ split_infos: list["SplitInfo"],
95
+ instruction: Union[str, "ReadInstruction"],
96
+ filetype_suffix: Optional[str] = None,
97
+ prefix_path: Optional[str] = None,
98
+ ) -> FileInstructions:
99
+ """Returns instructions of the split dict.
100
+
101
+ Args:
102
+ name (`str`): Name of the dataset.
103
+ split_infos (`list` of `[SplitInfo]`): Dataset splits information.
104
+ instruction ([`ReadInstruction`] or `str`): Reading instruction for a dataset.
105
+ filetype_suffix (`str`, *optional*): Suffix of dataset files, e.g. 'arrow' or 'parquet'.
106
+ prefix_path (`str`, *optional*): Prefix of dataset files, e.g. directory name.
107
+
108
+ Returns:
109
+ [`FileInstructions`]
110
+ """
111
+ if not isinstance(name, str):
112
+ raise TypeError(f"Expected str 'name', but got: {type(name).__name__}")
113
+ elif not name:
114
+ raise ValueError("Expected non-empty str 'name'")
115
+ name2len = {info.name: info.num_examples for info in split_infos}
116
+ name2shard_lengths = {info.name: info.shard_lengths for info in split_infos}
117
+ name2filenames = {
118
+ info.name: filenames_for_dataset_split(
119
+ path=prefix_path,
120
+ dataset_name=name,
121
+ split=info.name,
122
+ filetype_suffix=filetype_suffix,
123
+ shard_lengths=name2shard_lengths[info.name],
124
+ )
125
+ for info in split_infos
126
+ }
127
+ if not isinstance(instruction, ReadInstruction):
128
+ instruction = ReadInstruction.from_spec(instruction)
129
+ # Create the absolute instruction (per split)
130
+ absolute_instructions = instruction.to_absolute(name2len)
131
+
132
+ # For each split, return the files instruction (skip/take)
133
+ file_instructions = []
134
+ num_examples = 0
135
+ for abs_instr in absolute_instructions:
136
+ split_length = name2len[abs_instr.splitname]
137
+ filenames = name2filenames[abs_instr.splitname]
138
+ shard_lengths = name2shard_lengths[abs_instr.splitname]
139
+ from_ = 0 if abs_instr.from_ is None else abs_instr.from_
140
+ to = split_length if abs_instr.to is None else abs_instr.to
141
+ if shard_lengths is None: # not sharded
142
+ for filename in filenames:
143
+ take = to - from_
144
+ if take == 0:
145
+ continue
146
+ num_examples += take
147
+ file_instructions.append({"filename": filename, "skip": from_, "take": take})
148
+ else: # sharded
149
+ index_start = 0 # Beginning (included) of moving window.
150
+ index_end = 0 # End (excluded) of moving window.
151
+ for filename, shard_length in zip(filenames, shard_lengths):
152
+ index_end += shard_length
153
+ if from_ < index_end and to > index_start: # There is something to take.
154
+ skip = from_ - index_start if from_ > index_start else 0
155
+ take = to - index_start - skip if to < index_end else -1
156
+ if take == 0:
157
+ continue
158
+ file_instructions.append({"filename": filename, "skip": skip, "take": take})
159
+ num_examples += shard_length - skip if take == -1 else take
160
+ index_start += shard_length
161
+ return FileInstructions(
162
+ num_examples=num_examples,
163
+ file_instructions=file_instructions,
164
+ )
165
+
166
+
167
+ class BaseReader:
168
+ """
169
+ Build a Dataset object out of Instruction instance(s).
170
+ """
171
+
172
+ def __init__(self, path: str, info: Optional["DatasetInfo"]):
173
+ """Initializes ArrowReader.
174
+
175
+ Args:
176
+ path (str): path where tfrecords are stored.
177
+ info (DatasetInfo): info about the dataset.
178
+ """
179
+ self._path: str = path
180
+ self._info: Optional["DatasetInfo"] = info
181
+ self._filetype_suffix: Optional[str] = None
182
+
183
+ def _get_table_from_filename(self, filename_skip_take, in_memory=False) -> Table:
184
+ """Returns a Dataset instance from given (filename, skip, take)."""
185
+ raise NotImplementedError
186
+
187
+ def _read_files(self, files, in_memory=False) -> Table:
188
+ """Returns Dataset for given file instructions.
189
+
190
+ Args:
191
+ files: List[dict(filename, skip, take)], the files information.
192
+ The filenames contain the absolute path, not relative.
193
+ skip/take indicates which example read in the file: `ds.slice(skip, take)`
194
+ in_memory (bool, default False): Whether to copy the data in-memory.
195
+ """
196
+ if len(files) == 0 or not all(isinstance(f, dict) for f in files):
197
+ raise ValueError("please provide valid file informations")
198
+ files = copy.deepcopy(files)
199
+ for f in files:
200
+ f["filename"] = os.path.join(self._path, f["filename"])
201
+
202
+ pa_tables = thread_map(
203
+ partial(self._get_table_from_filename, in_memory=in_memory),
204
+ files,
205
+ tqdm_class=hf_tqdm,
206
+ desc="Loading dataset shards",
207
+ # set `disable=None` rather than `disable=False` by default to disable progress bar when no TTY attached
208
+ disable=len(files) <= 16 or None,
209
+ )
210
+ pa_tables = [t for t in pa_tables if len(t) > 0]
211
+ if not pa_tables and (self._info is None or self._info.features is None):
212
+ raise ValueError(
213
+ "Tried to read an empty table. Please specify at least info.features to create an empty table with the right type."
214
+ )
215
+ pa_tables = pa_tables or [InMemoryTable.from_batches([], schema=pa.schema(self._info.features.type))]
216
+ pa_table = concat_tables(pa_tables) if len(pa_tables) != 1 else pa_tables[0]
217
+ return pa_table
218
+
219
+ def get_file_instructions(self, name, instruction, split_infos):
220
+ """Return list of dict {'filename': str, 'skip': int, 'take': int}"""
221
+ file_instructions = make_file_instructions(
222
+ name, split_infos, instruction, filetype_suffix=self._filetype_suffix, prefix_path=self._path
223
+ )
224
+ files = file_instructions.file_instructions
225
+ return files
226
+
227
+ def read(
228
+ self,
229
+ name,
230
+ instructions,
231
+ split_infos,
232
+ in_memory=False,
233
+ ):
234
+ """Returns Dataset instance(s).
235
+
236
+ Args:
237
+ name (str): name of the dataset.
238
+ instructions (ReadInstruction): instructions to read.
239
+ Instruction can be string and will then be passed to the Instruction
240
+ constructor as it.
241
+ split_infos (list of SplitInfo proto): the available splits for dataset.
242
+ in_memory (bool, default False): Whether to copy the data in-memory.
243
+
244
+ Returns:
245
+ kwargs to build a single Dataset instance.
246
+ """
247
+
248
+ files = self.get_file_instructions(name, instructions, split_infos)
249
+ if not files:
250
+ msg = f'Instruction "{instructions}" corresponds to no data!'
251
+ raise ValueError(msg)
252
+ return self.read_files(files=files, original_instructions=instructions, in_memory=in_memory)
253
+
254
+ def read_files(
255
+ self,
256
+ files: list[dict],
257
+ original_instructions: Union[None, "ReadInstruction", "Split"] = None,
258
+ in_memory=False,
259
+ ):
260
+ """Returns single Dataset instance for the set of file instructions.
261
+
262
+ Args:
263
+ files: List[dict(filename, skip, take)], the files information.
264
+ The filenames contains the relative path, not absolute.
265
+ skip/take indicates which example read in the file: `ds.skip().take()`
266
+ original_instructions: store the original instructions used to build the dataset split in the dataset.
267
+ in_memory (bool, default False): Whether to copy the data in-memory.
268
+
269
+ Returns:
270
+ kwargs to build a Dataset instance.
271
+ """
272
+ # Prepend path to filename
273
+ pa_table = self._read_files(files, in_memory=in_memory)
274
+ # If original_instructions is not None, convert it to a human-readable NamedSplit
275
+ if original_instructions is not None:
276
+ from .splits import Split # noqa
277
+
278
+ split = Split(str(original_instructions))
279
+ else:
280
+ split = None
281
+ dataset_kwargs = {"arrow_table": pa_table, "info": self._info, "split": split}
282
+ return dataset_kwargs
283
+
284
+
285
+ class ArrowReader(BaseReader):
286
+ """
287
+ Build a Dataset object out of Instruction instance(s).
288
+ This Reader uses either memory mapping or file descriptors (in-memory) on arrow files.
289
+ """
290
+
291
+ def __init__(self, path: str, info: Optional["DatasetInfo"]):
292
+ """Initializes ArrowReader.
293
+
294
+ Args:
295
+ path (str): path where Arrow files are stored.
296
+ info (DatasetInfo): info about the dataset.
297
+ """
298
+ super().__init__(path, info)
299
+ self._filetype_suffix = "arrow"
300
+
301
+ def _get_table_from_filename(self, filename_skip_take, in_memory=False) -> Table:
302
+ """Returns a Dataset instance from given (filename, skip, take)."""
303
+ filename, skip, take = (
304
+ filename_skip_take["filename"],
305
+ filename_skip_take["skip"] if "skip" in filename_skip_take else None,
306
+ filename_skip_take["take"] if "take" in filename_skip_take else None,
307
+ )
308
+ table = ArrowReader.read_table(filename, in_memory=in_memory)
309
+ if take == -1:
310
+ take = len(table) - skip
311
+ # here we don't want to slice an empty table, or it may segfault
312
+ if skip is not None and take is not None and not (skip == 0 and take == len(table)):
313
+ table = table.slice(skip, take)
314
+ return table
315
+
316
+ @staticmethod
317
+ def read_table(filename, in_memory=False) -> Table:
318
+ """
319
+ Read table from file.
320
+
321
+ Args:
322
+ filename (str): File name of the table.
323
+ in_memory (bool, default=False): Whether to copy the data in-memory.
324
+
325
+ Returns:
326
+ pyarrow.Table
327
+ """
328
+ table_cls = InMemoryTable if in_memory else MemoryMappedTable
329
+ return table_cls.from_file(filename)
330
+
331
+
332
+ class ParquetReader(BaseReader):
333
+ """
334
+ Build a Dataset object out of Instruction instance(s).
335
+ This Reader uses memory mapping on parquet files.
336
+ """
337
+
338
+ def __init__(self, path: str, info: Optional["DatasetInfo"]):
339
+ """Initializes ParquetReader.
340
+
341
+ Args:
342
+ path (str): path where tfrecords are stored.
343
+ info (DatasetInfo): info about the dataset.
344
+ """
345
+ super().__init__(path, info)
346
+ self._filetype_suffix = "parquet"
347
+
348
+ def _get_table_from_filename(self, filename_skip_take, **kwargs):
349
+ """Returns a Dataset instance from given (filename, skip, take)."""
350
+ filename, skip, take = (
351
+ filename_skip_take["filename"],
352
+ filename_skip_take["skip"] if "skip" in filename_skip_take else None,
353
+ filename_skip_take["take"] if "take" in filename_skip_take else None,
354
+ )
355
+ # Parquet read_table always loads data in memory, independently of memory_map
356
+ pa_table = pq.read_table(filename, memory_map=True)
357
+ # here we don't want to slice an empty table, or it may segfault
358
+ if skip is not None and take is not None and not (skip == 0 and take == len(pa_table)):
359
+ pa_table = pa_table.slice(skip, take)
360
+ return pa_table
361
+
362
+
363
+ @dataclass(frozen=True)
364
+ class _AbsoluteInstruction:
365
+ """A machine friendly slice: defined absolute positive boundaries."""
366
+
367
+ splitname: str
368
+ from_: int # uint (starting index).
369
+ to: int # uint (ending index).
370
+
371
+
372
+ @dataclass(frozen=True)
373
+ class _RelativeInstruction:
374
+ """Represents a single parsed slicing instruction, can use % and negatives."""
375
+
376
+ splitname: str
377
+ from_: Optional[int] = None # int (starting index) or None if no lower boundary.
378
+ to: Optional[int] = None # int (ending index) or None if no upper boundary.
379
+ unit: Optional[str] = None
380
+ rounding: Optional[str] = None
381
+
382
+ def __post_init__(self):
383
+ if self.unit is not None and self.unit not in ["%", "abs"]:
384
+ raise ValueError("unit must be either % or abs")
385
+ if self.rounding is not None and self.rounding not in ["closest", "pct1_dropremainder"]:
386
+ raise ValueError("rounding must be either closest or pct1_dropremainder")
387
+ if self.unit != "%" and self.rounding is not None:
388
+ raise ValueError("It is forbidden to specify rounding if not using percent slicing.")
389
+ if self.unit == "%" and self.from_ is not None and abs(self.from_) > 100:
390
+ raise ValueError("Percent slice boundaries must be > -100 and < 100.")
391
+ if self.unit == "%" and self.to is not None and abs(self.to) > 100:
392
+ raise ValueError("Percent slice boundaries must be > -100 and < 100.")
393
+ # Update via __dict__ due to instance being "frozen"
394
+ self.__dict__["rounding"] = "closest" if self.rounding is None and self.unit == "%" else self.rounding
395
+
396
+
397
+ def _str_to_read_instruction(spec):
398
+ """Returns ReadInstruction for given string."""
399
+ res = _SUB_SPEC_RE.match(spec)
400
+ if not res:
401
+ raise ValueError(f"Unrecognized instruction format: {spec}")
402
+ unit = "%" if res.group("from_pct") or res.group("to_pct") else "abs"
403
+ return ReadInstruction(
404
+ split_name=res.group("split"),
405
+ rounding=res.group("rounding"),
406
+ from_=int(res.group("from")) if res.group("from") else None,
407
+ to=int(res.group("to")) if res.group("to") else None,
408
+ unit=unit,
409
+ )
410
+
411
+
412
+ def _pct_to_abs_pct1(boundary, num_examples):
413
+ # Using math.trunc here, since -99.5% should give -99%, not -100%.
414
+ if num_examples < 100:
415
+ msg = (
416
+ 'Using "pct1_dropremainder" rounding on a split with less than 100 '
417
+ "elements is forbidden: it always results in an empty dataset."
418
+ )
419
+ raise ValueError(msg)
420
+ return boundary * math.trunc(num_examples / 100.0)
421
+
422
+
423
+ def _pct_to_abs_closest(boundary, num_examples):
424
+ return int(round(boundary * num_examples / 100.0))
425
+
426
+
427
+ def _rel_to_abs_instr(rel_instr, name2len):
428
+ """Returns _AbsoluteInstruction instance for given RelativeInstruction.
429
+
430
+ Args:
431
+ rel_instr: RelativeInstruction instance.
432
+ name2len: dict {split_name: num_examples}.
433
+ """
434
+ pct_to_abs = _pct_to_abs_closest if rel_instr.rounding == "closest" else _pct_to_abs_pct1
435
+ split = rel_instr.splitname
436
+ if split not in name2len:
437
+ raise ValueError(f'Unknown split "{split}". Should be one of {list(name2len)}.')
438
+ num_examples = name2len[split]
439
+ from_ = rel_instr.from_
440
+ to = rel_instr.to
441
+ if rel_instr.unit == "%":
442
+ from_ = 0 if from_ is None else pct_to_abs(from_, num_examples)
443
+ to = num_examples if to is None else pct_to_abs(to, num_examples)
444
+ else:
445
+ from_ = 0 if from_ is None else from_
446
+ to = num_examples if to is None else to
447
+ if from_ < 0:
448
+ from_ = max(num_examples + from_, 0)
449
+ if to < 0:
450
+ to = max(num_examples + to, 0)
451
+ from_ = min(from_, num_examples)
452
+ to = min(to, num_examples)
453
+ return _AbsoluteInstruction(split, from_, to)
454
+
455
+
456
+ class ReadInstruction:
457
+ """Reading instruction for a dataset.
458
+
459
+ Examples::
460
+
461
+ # The following lines are equivalent:
462
+ ds = datasets.load_dataset('mnist', split='test[:33%]')
463
+ ds = datasets.load_dataset('mnist', split=datasets.ReadInstruction.from_spec('test[:33%]'))
464
+ ds = datasets.load_dataset('mnist', split=datasets.ReadInstruction('test', to=33, unit='%'))
465
+ ds = datasets.load_dataset('mnist', split=datasets.ReadInstruction(
466
+ 'test', from_=0, to=33, unit='%'))
467
+
468
+ # The following lines are equivalent:
469
+ ds = datasets.load_dataset('mnist', split='test[:33%]+train[1:-1]')
470
+ ds = datasets.load_dataset('mnist', split=datasets.ReadInstruction.from_spec(
471
+ 'test[:33%]+train[1:-1]'))
472
+ ds = datasets.load_dataset('mnist', split=(
473
+ datasets.ReadInstruction('test', to=33, unit='%') +
474
+ datasets.ReadInstruction('train', from_=1, to=-1, unit='abs')))
475
+
476
+ # The following lines are equivalent:
477
+ ds = datasets.load_dataset('mnist', split='test[:33%](pct1_dropremainder)')
478
+ ds = datasets.load_dataset('mnist', split=datasets.ReadInstruction.from_spec(
479
+ 'test[:33%](pct1_dropremainder)'))
480
+ ds = datasets.load_dataset('mnist', split=datasets.ReadInstruction(
481
+ 'test', from_=0, to=33, unit='%', rounding="pct1_dropremainder"))
482
+
483
+ # 10-fold validation:
484
+ tests = datasets.load_dataset(
485
+ 'mnist',
486
+ [datasets.ReadInstruction('train', from_=k, to=k+10, unit='%')
487
+ for k in range(0, 100, 10)])
488
+ trains = datasets.load_dataset(
489
+ 'mnist',
490
+ [datasets.ReadInstruction('train', to=k, unit='%') + datasets.ReadInstruction('train', from_=k+10, unit='%')
491
+ for k in range(0, 100, 10)])
492
+
493
+ """
494
+
495
+ def _init(self, relative_instructions):
496
+ # Private initializer.
497
+ self._relative_instructions = relative_instructions
498
+
499
+ @classmethod
500
+ def _read_instruction_from_relative_instructions(cls, relative_instructions):
501
+ """Returns ReadInstruction obj initialized with relative_instructions."""
502
+ # Use __new__ to bypass __init__ used by public API and not conveniant here.
503
+ result = cls.__new__(cls)
504
+ result._init(relative_instructions) # pylint: disable=protected-access
505
+ return result
506
+
507
+ def __init__(self, split_name, rounding=None, from_=None, to=None, unit=None):
508
+ """Initialize ReadInstruction.
509
+
510
+ Args:
511
+ split_name (str): name of the split to read. Eg: 'train'.
512
+ rounding (str, optional): The rounding behaviour to use when percent slicing is
513
+ used. Ignored when slicing with absolute indices.
514
+ Possible values:
515
+ - 'closest' (default): The specified percentages are rounded to the
516
+ closest value. Use this if you want specified percents to be as
517
+ much exact as possible.
518
+ - 'pct1_dropremainder': the specified percentages are treated as
519
+ multiple of 1%. Use this option if you want consistency. Eg:
520
+ len(5%) == 5 * len(1%).
521
+ Using this option, one might not be able to use the full set of
522
+ examples, if the number of those is not a multiple of 100.
523
+ from_ (int):
524
+ to (int): alternative way of specifying slicing boundaries. If any of
525
+ {from_, to, unit} argument is used, slicing cannot be specified as
526
+ string.
527
+ unit (str): optional, one of:
528
+ '%': to set the slicing unit as percents of the split size.
529
+ 'abs': to set the slicing unit as absolute numbers.
530
+ """
531
+ # This constructor is not always called. See factory method
532
+ # `_read_instruction_from_relative_instructions`. Common init instructions
533
+ # MUST be placed in the _init method.
534
+ self._init([_RelativeInstruction(split_name, from_, to, unit, rounding)])
535
+
536
+ @classmethod
537
+ def from_spec(cls, spec):
538
+ """Creates a `ReadInstruction` instance out of a string spec.
539
+
540
+ Args:
541
+ spec (`str`):
542
+ Split(s) + optional slice(s) to read + optional rounding
543
+ if percents are used as the slicing unit. A slice can be specified,
544
+ using absolute numbers (`int`) or percentages (`int`).
545
+
546
+ Examples:
547
+
548
+ ```
549
+ test: test split.
550
+ test + validation: test split + validation split.
551
+ test[10:]: test split, minus its first 10 records.
552
+ test[:10%]: first 10% records of test split.
553
+ test[:20%](pct1_dropremainder): first 10% records, rounded with the pct1_dropremainder rounding.
554
+ test[:-5%]+train[40%:60%]: first 95% of test + middle 20% of train.
555
+ ```
556
+
557
+ Returns:
558
+ ReadInstruction instance.
559
+ """
560
+ spec = str(spec) # Need to convert to str in case of NamedSplit instance.
561
+ subs = _ADDITION_SEP_RE.split(spec)
562
+ if not subs:
563
+ raise ValueError(f"No instructions could be built out of {spec}")
564
+ instruction = _str_to_read_instruction(subs[0])
565
+ return sum((_str_to_read_instruction(sub) for sub in subs[1:]), instruction)
566
+
567
+ def to_spec(self):
568
+ rel_instr_specs = []
569
+ for rel_instr in self._relative_instructions:
570
+ rel_instr_spec = rel_instr.splitname
571
+ if rel_instr.from_ is not None or rel_instr.to is not None:
572
+ from_ = rel_instr.from_
573
+ to = rel_instr.to
574
+ unit = rel_instr.unit
575
+ rounding = rel_instr.rounding
576
+ unit = unit if unit == "%" else ""
577
+ from_ = str(from_) + unit if from_ is not None else ""
578
+ to = str(to) + unit if to is not None else ""
579
+ slice_str = f"[{from_}:{to}]"
580
+ rounding_str = (
581
+ f"({rounding})" if unit == "%" and rounding is not None and rounding != "closest" else ""
582
+ )
583
+ rel_instr_spec += slice_str + rounding_str
584
+ rel_instr_specs.append(rel_instr_spec)
585
+ return "+".join(rel_instr_specs)
586
+
587
+ def __add__(self, other):
588
+ """Returns a new ReadInstruction obj, result of appending other to self."""
589
+ if not isinstance(other, ReadInstruction):
590
+ msg = "ReadInstruction can only be added to another ReadInstruction obj."
591
+ raise TypeError(msg)
592
+ self_ris = self._relative_instructions
593
+ other_ris = other._relative_instructions # pylint: disable=protected-access
594
+ if (
595
+ self_ris[0].unit != "abs"
596
+ and other_ris[0].unit != "abs"
597
+ and self._relative_instructions[0].rounding != other_ris[0].rounding
598
+ ):
599
+ raise ValueError("It is forbidden to sum ReadInstruction instances with different rounding values.")
600
+ return self._read_instruction_from_relative_instructions(self_ris + other_ris)
601
+
602
+ def __str__(self):
603
+ return self.to_spec()
604
+
605
+ def __repr__(self):
606
+ return f"ReadInstruction({self._relative_instructions})"
607
+
608
+ def to_absolute(self, name2len):
609
+ """Translate instruction into a list of absolute instructions.
610
+
611
+ Those absolute instructions are then to be added together.
612
+
613
+ Args:
614
+ name2len (`dict`):
615
+ Associating split names to number of examples.
616
+
617
+ Returns:
618
+ list of _AbsoluteInstruction instances (corresponds to the + in spec).
619
+ """
620
+ return [_rel_to_abs_instr(rel_instr, name2len) for rel_instr in self._relative_instructions]
lib/python3.12/site-packages/datasets/arrow_writer.py ADDED
@@ -0,0 +1,677 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # Unless required by applicable law or agreed to in writing, software
8
+ # distributed under the License is distributed on an "AS IS" BASIS,
9
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10
+ # See the License for the specific language governing permissions and
11
+ # limitations under the License.
12
+
13
+ # Lint as: python3
14
+ """To write records into Parquet files."""
15
+
16
+ import json
17
+ import sys
18
+ from collections.abc import Iterable
19
+ from typing import Any, Optional, Union
20
+
21
+ import fsspec
22
+ import numpy as np
23
+ import pyarrow as pa
24
+ import pyarrow.parquet as pq
25
+ from fsspec.core import url_to_fs
26
+
27
+ from . import config
28
+ from .features import Audio, Features, Image, Pdf, Value, Video
29
+ from .features.features import (
30
+ FeatureType,
31
+ _ArrayXDExtensionType,
32
+ _visit,
33
+ cast_to_python_objects,
34
+ generate_from_arrow_type,
35
+ get_nested_type,
36
+ list_of_np_array_to_pyarrow_listarray,
37
+ numpy_to_pyarrow_listarray,
38
+ to_pyarrow_listarray,
39
+ )
40
+ from .filesystems import is_remote_filesystem
41
+ from .info import DatasetInfo
42
+ from .keyhash import DuplicatedKeysError, KeyHasher
43
+ from .table import array_cast, cast_array_to_feature, embed_table_storage, table_cast
44
+ from .utils import logging
45
+ from .utils.py_utils import asdict, first_non_null_non_empty_value
46
+
47
+
48
+ logger = logging.get_logger(__name__)
49
+
50
+ type_ = type # keep python's type function
51
+
52
+
53
+ def get_writer_batch_size(features: Optional[Features]) -> Optional[int]:
54
+ """
55
+ Get the writer_batch_size that defines the maximum row group size in the parquet files.
56
+ The default in `datasets` is 1,000 but we lower it to 100 for image/audio datasets and 10 for videos.
57
+ This allows to optimize random access to parquet file, since accessing 1 row requires
58
+ to read its entire row group.
59
+
60
+ This can be improved to get optimized size for querying/iterating
61
+ but at least it matches the dataset viewer expectations on HF.
62
+
63
+ Args:
64
+ features (`datasets.Features` or `None`):
65
+ Dataset Features from `datasets`.
66
+ Returns:
67
+ writer_batch_size (`Optional[int]`):
68
+ Writer batch size to pass to a dataset builder.
69
+ If `None`, then it will use the `datasets` default.
70
+ """
71
+ if not features:
72
+ return None
73
+
74
+ batch_size = np.inf
75
+
76
+ def set_batch_size(feature: FeatureType) -> None:
77
+ nonlocal batch_size
78
+ if isinstance(feature, Image):
79
+ batch_size = min(batch_size, config.PARQUET_ROW_GROUP_SIZE_FOR_IMAGE_DATASETS)
80
+ elif isinstance(feature, Audio):
81
+ batch_size = min(batch_size, config.PARQUET_ROW_GROUP_SIZE_FOR_AUDIO_DATASETS)
82
+ elif isinstance(feature, Video):
83
+ batch_size = min(batch_size, config.PARQUET_ROW_GROUP_SIZE_FOR_VIDEO_DATASETS)
84
+ elif isinstance(feature, Value) and feature.dtype == "binary":
85
+ batch_size = min(batch_size, config.PARQUET_ROW_GROUP_SIZE_FOR_BINARY_DATASETS)
86
+
87
+ _visit(features, set_batch_size)
88
+
89
+ return None if batch_size is np.inf else batch_size
90
+
91
+
92
+ class SchemaInferenceError(ValueError):
93
+ pass
94
+
95
+
96
+ class TypedSequence:
97
+ """
98
+ This data container generalizes the typing when instantiating pyarrow arrays, tables or batches.
99
+
100
+ More specifically it adds several features:
101
+ - Support extension types like ``datasets.features.Array2DExtensionType``:
102
+ By default pyarrow arrays don't return extension arrays. One has to call
103
+ ``pa.ExtensionArray.from_storage(type, pa.array(data, type.storage_type))``
104
+ in order to get an extension array.
105
+ - Support for ``try_type`` parameter that can be used instead of ``type``:
106
+ When an array is transformed, we like to keep the same type as before if possible.
107
+ For example when calling :func:`datasets.Dataset.map`, we don't want to change the type
108
+ of each column by default.
109
+ - Better error message when a pyarrow array overflows.
110
+
111
+ Example::
112
+
113
+ from datasets.features import Array2D, Array2DExtensionType, Value
114
+ from datasets.arrow_writer import TypedSequence
115
+ import pyarrow as pa
116
+
117
+ arr = pa.array(TypedSequence([1, 2, 3], type=Value("int32")))
118
+ assert arr.type == pa.int32()
119
+
120
+ arr = pa.array(TypedSequence([1, 2, 3], try_type=Value("int32")))
121
+ assert arr.type == pa.int32()
122
+
123
+ arr = pa.array(TypedSequence(["foo", "bar"], try_type=Value("int32")))
124
+ assert arr.type == pa.string()
125
+
126
+ arr = pa.array(TypedSequence([[[1, 2, 3]]], type=Array2D((1, 3), "int64")))
127
+ assert arr.type == Array2DExtensionType((1, 3), "int64")
128
+
129
+ table = pa.Table.from_pydict({
130
+ "image": TypedSequence([[[1, 2, 3]]], type=Array2D((1, 3), "int64"))
131
+ })
132
+ assert table["image"].type == Array2DExtensionType((1, 3), "int64")
133
+
134
+ """
135
+
136
+ def __init__(
137
+ self,
138
+ data: Iterable,
139
+ type: Optional[FeatureType] = None,
140
+ try_type: Optional[FeatureType] = None,
141
+ optimized_int_type: Optional[FeatureType] = None,
142
+ ):
143
+ # assert type is None or try_type is None,
144
+ if type is not None and try_type is not None:
145
+ raise ValueError("You cannot specify both type and try_type")
146
+ # set attributes
147
+ self.data = data
148
+ self.type = type
149
+ self.try_type = try_type # is ignored if it doesn't match the data
150
+ self.optimized_int_type = optimized_int_type
151
+ # when trying a type (is ignored if data is not compatible)
152
+ self.trying_type = self.try_type is not None
153
+ self.trying_int_optimization = optimized_int_type is not None and type is None and try_type is None
154
+ # used to get back the inferred type after __arrow_array__() is called once
155
+ self._inferred_type = None
156
+
157
+ def get_inferred_type(self) -> FeatureType:
158
+ """Return the inferred feature type.
159
+ This is done by converting the sequence to an Arrow array, and getting the corresponding
160
+ feature type.
161
+
162
+ Since building the Arrow array can be expensive, the value of the inferred type is cached
163
+ as soon as pa.array is called on the typed sequence.
164
+
165
+ Returns:
166
+ FeatureType: inferred feature type of the sequence.
167
+ """
168
+ if self._inferred_type is None:
169
+ self._inferred_type = generate_from_arrow_type(pa.array(self).type)
170
+ return self._inferred_type
171
+
172
+ @staticmethod
173
+ def _infer_custom_type_and_encode(data: Iterable) -> tuple[Iterable, Optional[FeatureType]]:
174
+ """Implement type inference for custom objects like PIL.Image.Image -> Image type.
175
+
176
+ This function is only used for custom python objects that can't be directly passed to build
177
+ an Arrow array. In such cases is infers the feature type to use, and it encodes the data so
178
+ that they can be passed to an Arrow array.
179
+
180
+ Args:
181
+ data (Iterable): array of data to infer the type, e.g. a list of PIL images.
182
+
183
+ Returns:
184
+ Tuple[Iterable, Optional[FeatureType]]: a tuple with:
185
+ - the (possibly encoded) array, if the inferred feature type requires encoding
186
+ - the inferred feature type if the array is made of supported custom objects like
187
+ PIL images, else None.
188
+ """
189
+ if config.PIL_AVAILABLE and "PIL" in sys.modules:
190
+ import PIL.Image
191
+
192
+ non_null_idx, non_null_value = first_non_null_non_empty_value(data)
193
+ if isinstance(non_null_value, PIL.Image.Image):
194
+ return [Image().encode_example(value) if value is not None else None for value in data], Image()
195
+ if isinstance(non_null_value, list) and isinstance(non_null_value[0], PIL.Image.Image):
196
+ return [[Image().encode_example(x) for x in value] if value is not None else None for value in data], [
197
+ Image()
198
+ ]
199
+ if config.PDFPLUMBER_AVAILABLE and "pdfplumber" in sys.modules:
200
+ import pdfplumber
201
+
202
+ non_null_idx, non_null_value = first_non_null_non_empty_value(data)
203
+ if isinstance(non_null_value, pdfplumber.pdf.PDF):
204
+ return [Pdf().encode_example(value) if value is not None else None for value in data], Pdf()
205
+ if isinstance(non_null_value, list) and isinstance(non_null_value[0], pdfplumber.pdf.PDF):
206
+ return [[Pdf().encode_example(x) for x in value] if value is not None else None for value in data], [
207
+ Pdf()
208
+ ]
209
+ return data, None
210
+
211
+ def __arrow_array__(self, type: Optional[pa.DataType] = None):
212
+ """This function is called when calling pa.array(typed_sequence)"""
213
+
214
+ if type is not None:
215
+ raise ValueError("TypedSequence is supposed to be used with pa.array(typed_sequence, type=None)")
216
+ del type # make sure we don't use it
217
+ data = self.data
218
+ # automatic type inference for custom objects
219
+ if self.type is None and self.try_type is None:
220
+ data, self._inferred_type = self._infer_custom_type_and_encode(data)
221
+ if self._inferred_type is None:
222
+ type = self.try_type if self.trying_type else self.type
223
+ else:
224
+ type = self._inferred_type
225
+ pa_type = get_nested_type(type) if type is not None else None
226
+ optimized_int_pa_type = (
227
+ get_nested_type(self.optimized_int_type) if self.optimized_int_type is not None else None
228
+ )
229
+ trying_cast_to_python_objects = False
230
+ try:
231
+ # custom pyarrow types
232
+ if isinstance(pa_type, _ArrayXDExtensionType):
233
+ storage = to_pyarrow_listarray(data, pa_type)
234
+ return pa.ExtensionArray.from_storage(pa_type, storage)
235
+
236
+ # efficient np array to pyarrow array
237
+ if isinstance(data, np.ndarray):
238
+ out = numpy_to_pyarrow_listarray(data)
239
+ elif isinstance(data, list) and data and isinstance(first_non_null_non_empty_value(data)[1], np.ndarray):
240
+ out = list_of_np_array_to_pyarrow_listarray(data)
241
+ else:
242
+ trying_cast_to_python_objects = True
243
+ out = pa.array(cast_to_python_objects(data, only_1d_for_numpy=True))
244
+ # use smaller integer precisions if possible
245
+ if self.trying_int_optimization:
246
+ if pa.types.is_int64(out.type):
247
+ out = out.cast(optimized_int_pa_type)
248
+ elif pa.types.is_list(out.type):
249
+ if pa.types.is_int64(out.type.value_type):
250
+ out = array_cast(out, pa.list_(optimized_int_pa_type))
251
+ elif pa.types.is_list(out.type.value_type) and pa.types.is_int64(out.type.value_type.value_type):
252
+ out = array_cast(out, pa.list_(pa.list_(optimized_int_pa_type)))
253
+ # otherwise we can finally use the user's type
254
+ elif type is not None:
255
+ # We use cast_array_to_feature to support casting to custom types like Audio and Image
256
+ # Also, when trying type "string", we don't want to convert integers or floats to "string".
257
+ # We only do it if trying_type is False - since this is what the user asks for.
258
+ out = cast_array_to_feature(
259
+ out, type, allow_primitive_to_str=not self.trying_type, allow_decimal_to_str=not self.trying_type
260
+ )
261
+ return out
262
+ except (
263
+ TypeError,
264
+ pa.lib.ArrowInvalid,
265
+ pa.lib.ArrowNotImplementedError,
266
+ ) as e: # handle type errors and overflows
267
+ # Ignore ArrowNotImplementedError caused by trying type, otherwise re-raise
268
+ if not self.trying_type and isinstance(e, pa.lib.ArrowNotImplementedError):
269
+ raise
270
+
271
+ if self.trying_type:
272
+ try: # second chance
273
+ if isinstance(data, np.ndarray):
274
+ return numpy_to_pyarrow_listarray(data)
275
+ elif isinstance(data, list) and data and any(isinstance(value, np.ndarray) for value in data):
276
+ return list_of_np_array_to_pyarrow_listarray(data)
277
+ else:
278
+ trying_cast_to_python_objects = True
279
+ return pa.array(cast_to_python_objects(data, only_1d_for_numpy=True))
280
+ except pa.lib.ArrowInvalid as e:
281
+ if "overflow" in str(e):
282
+ raise OverflowError(
283
+ f"There was an overflow with type {type_(data)}. Try to reduce writer_batch_size to have batches smaller than 2GB.\n({e})"
284
+ ) from None
285
+ elif self.trying_int_optimization and "not in range" in str(e):
286
+ optimized_int_pa_type_str = np.dtype(optimized_int_pa_type.to_pandas_dtype()).name
287
+ logger.info(
288
+ f"Failed to cast a sequence to {optimized_int_pa_type_str}. Falling back to int64."
289
+ )
290
+ return out
291
+ elif trying_cast_to_python_objects and "Could not convert" in str(e):
292
+ out = pa.array(
293
+ cast_to_python_objects(data, only_1d_for_numpy=True, optimize_list_casting=False)
294
+ )
295
+ if type is not None:
296
+ out = cast_array_to_feature(
297
+ out, type, allow_primitive_to_str=True, allow_decimal_to_str=True
298
+ )
299
+ return out
300
+ else:
301
+ raise
302
+ elif "overflow" in str(e):
303
+ raise OverflowError(
304
+ f"There was an overflow with type {type_(data)}. Try to reduce writer_batch_size to have batches smaller than 2GB.\n({e})"
305
+ ) from None
306
+ elif self.trying_int_optimization and "not in range" in str(e):
307
+ optimized_int_pa_type_str = np.dtype(optimized_int_pa_type.to_pandas_dtype()).name
308
+ logger.info(f"Failed to cast a sequence to {optimized_int_pa_type_str}. Falling back to int64.")
309
+ return out
310
+ elif trying_cast_to_python_objects and "Could not convert" in str(e):
311
+ out = pa.array(cast_to_python_objects(data, only_1d_for_numpy=True, optimize_list_casting=False))
312
+ if type is not None:
313
+ out = cast_array_to_feature(out, type, allow_primitive_to_str=True, allow_decimal_to_str=True)
314
+ return out
315
+ else:
316
+ raise
317
+
318
+
319
+ class OptimizedTypedSequence(TypedSequence):
320
+ def __init__(
321
+ self,
322
+ data,
323
+ type: Optional[FeatureType] = None,
324
+ try_type: Optional[FeatureType] = None,
325
+ col: Optional[str] = None,
326
+ optimized_int_type: Optional[FeatureType] = None,
327
+ ):
328
+ optimized_int_type_by_col = {
329
+ "attention_mask": Value("int8"), # binary tensor
330
+ "special_tokens_mask": Value("int8"),
331
+ "input_ids": Value("int32"), # typical vocab size: 0-50k (max ~500k, never > 1M)
332
+ "token_type_ids": Value(
333
+ "int8"
334
+ ), # binary mask; some (XLNetModel) use an additional token represented by a 2
335
+ }
336
+ if type is None and try_type is None:
337
+ optimized_int_type = optimized_int_type_by_col.get(col, None)
338
+ super().__init__(data, type=type, try_type=try_type, optimized_int_type=optimized_int_type)
339
+
340
+
341
+ class ArrowWriter:
342
+ """Shuffles and writes Examples to Arrow files."""
343
+
344
+ _WRITER_CLASS = pa.RecordBatchStreamWriter
345
+
346
+ def __init__(
347
+ self,
348
+ schema: Optional[pa.Schema] = None,
349
+ features: Optional[Features] = None,
350
+ path: Optional[str] = None,
351
+ stream: Optional[pa.NativeFile] = None,
352
+ fingerprint: Optional[str] = None,
353
+ writer_batch_size: Optional[int] = None,
354
+ hash_salt: Optional[str] = None,
355
+ check_duplicates: Optional[bool] = False,
356
+ disable_nullable: bool = False,
357
+ update_features: bool = False,
358
+ with_metadata: bool = True,
359
+ unit: str = "examples",
360
+ embed_local_files: bool = False,
361
+ storage_options: Optional[dict] = None,
362
+ ):
363
+ if path is None and stream is None:
364
+ raise ValueError("At least one of path and stream must be provided.")
365
+ if features is not None:
366
+ self._features = features
367
+ self._schema = None
368
+ elif schema is not None:
369
+ self._schema: pa.Schema = schema
370
+ self._features = Features.from_arrow_schema(self._schema)
371
+ else:
372
+ self._features = None
373
+ self._schema = None
374
+
375
+ if hash_salt is not None:
376
+ # Create KeyHasher instance using split name as hash salt
377
+ self._hasher = KeyHasher(hash_salt)
378
+ else:
379
+ self._hasher = KeyHasher("")
380
+
381
+ self._check_duplicates = check_duplicates
382
+ self._disable_nullable = disable_nullable
383
+
384
+ if stream is None:
385
+ fs, path = url_to_fs(path, **(storage_options or {}))
386
+ self._fs: fsspec.AbstractFileSystem = fs
387
+ self._path = path if not is_remote_filesystem(self._fs) else self._fs.unstrip_protocol(path)
388
+ self.stream = self._fs.open(path, "wb")
389
+ self._closable_stream = True
390
+ else:
391
+ self._fs = None
392
+ self._path = None
393
+ self.stream = stream
394
+ self._closable_stream = False
395
+
396
+ self.fingerprint = fingerprint
397
+ self.disable_nullable = disable_nullable
398
+ self.writer_batch_size = (
399
+ writer_batch_size or get_writer_batch_size(self._features) or config.DEFAULT_MAX_BATCH_SIZE
400
+ )
401
+ self.update_features = update_features
402
+ self.with_metadata = with_metadata
403
+ self.unit = unit
404
+ self.embed_local_files = embed_local_files
405
+
406
+ self._num_examples = 0
407
+ self._num_bytes = 0
408
+ self.current_examples: list[tuple[dict[str, Any], str]] = []
409
+ self.current_rows: list[pa.Table] = []
410
+ self.pa_writer: Optional[pa.RecordBatchStreamWriter] = None
411
+ self.hkey_record = []
412
+
413
+ def __len__(self):
414
+ """Return the number of writed and staged examples"""
415
+ return self._num_examples + len(self.current_examples) + len(self.current_rows)
416
+
417
+ def __enter__(self):
418
+ return self
419
+
420
+ def __exit__(self, exc_type, exc_val, exc_tb):
421
+ self.close()
422
+
423
+ def close(self):
424
+ # Try closing if opened; if closed: pyarrow.lib.ArrowInvalid: Invalid operation on closed file
425
+ if self.pa_writer: # it might be None
426
+ try:
427
+ self.pa_writer.close()
428
+ except Exception: # pyarrow.lib.ArrowInvalid, OSError
429
+ pass
430
+ if self._closable_stream and not self.stream.closed:
431
+ self.stream.close() # This also closes self.pa_writer if it is opened
432
+
433
+ def _build_writer(self, inferred_schema: pa.Schema):
434
+ schema = self.schema
435
+ inferred_features = Features.from_arrow_schema(inferred_schema)
436
+ if self._features is not None:
437
+ if self.update_features: # keep original features it they match, or update them
438
+ fields = {field.name: field for field in self._features.type}
439
+ for inferred_field in inferred_features.type:
440
+ name = inferred_field.name
441
+ if name in fields:
442
+ if inferred_field == fields[name]:
443
+ inferred_features[name] = self._features[name]
444
+ self._features = inferred_features
445
+ schema: pa.Schema = inferred_schema
446
+ else:
447
+ self._features = inferred_features
448
+ schema: pa.Schema = inferred_features.arrow_schema
449
+ if self.disable_nullable:
450
+ schema = pa.schema(pa.field(field.name, field.type, nullable=False) for field in schema)
451
+ if self.with_metadata:
452
+ schema = schema.with_metadata(self._build_metadata(DatasetInfo(features=self._features), self.fingerprint))
453
+ else:
454
+ schema = schema.with_metadata({})
455
+ self._schema = schema
456
+ self.pa_writer = self._WRITER_CLASS(self.stream, schema)
457
+
458
+ @property
459
+ def schema(self):
460
+ _schema = (
461
+ self._schema
462
+ if self._schema is not None
463
+ else (pa.schema(self._features.type) if self._features is not None else None)
464
+ )
465
+ if self._disable_nullable and _schema is not None:
466
+ _schema = pa.schema(pa.field(field.name, field.type, nullable=False) for field in _schema)
467
+ return _schema if _schema is not None else []
468
+
469
+ @staticmethod
470
+ def _build_metadata(info: DatasetInfo, fingerprint: Optional[str] = None) -> dict[str, str]:
471
+ info_keys = ["features"] # we can add support for more DatasetInfo keys in the future
472
+ info_as_dict = asdict(info)
473
+ metadata = {}
474
+ metadata["info"] = {key: info_as_dict[key] for key in info_keys}
475
+ if fingerprint is not None:
476
+ metadata["fingerprint"] = fingerprint
477
+ return {"huggingface": json.dumps(metadata)}
478
+
479
+ def write_examples_on_file(self):
480
+ """Write stored examples from the write-pool of examples. It makes a table out of the examples and write it."""
481
+ if not self.current_examples:
482
+ return
483
+ # preserve the order the columns
484
+ if self.schema:
485
+ schema_cols = set(self.schema.names)
486
+ examples_cols = self.current_examples[0][0].keys() # .keys() preserves the order (unlike set)
487
+ common_cols = [col for col in self.schema.names if col in examples_cols]
488
+ extra_cols = [col for col in examples_cols if col not in schema_cols]
489
+ cols = common_cols + extra_cols
490
+ else:
491
+ cols = list(self.current_examples[0][0])
492
+ batch_examples = {}
493
+ for col in cols:
494
+ # We use row[0][col] since current_examples contains (example, key) tuples.
495
+ # Moreover, examples could be Arrow arrays of 1 element.
496
+ # This can happen in `.map()` when we want to re-write the same Arrow data
497
+ if all(isinstance(row[0][col], (pa.Array, pa.ChunkedArray)) for row in self.current_examples):
498
+ arrays = [row[0][col] for row in self.current_examples]
499
+ arrays = [
500
+ chunk
501
+ for array in arrays
502
+ for chunk in (array.chunks if isinstance(array, pa.ChunkedArray) else [array])
503
+ ]
504
+ batch_examples[col] = pa.concat_arrays(arrays)
505
+ else:
506
+ batch_examples[col] = [
507
+ row[0][col].to_pylist()[0] if isinstance(row[0][col], (pa.Array, pa.ChunkedArray)) else row[0][col]
508
+ for row in self.current_examples
509
+ ]
510
+ self.write_batch(batch_examples=batch_examples)
511
+ self.current_examples = []
512
+
513
+ def write_rows_on_file(self):
514
+ """Write stored rows from the write-pool of rows. It concatenates the single-row tables and it writes the resulting table."""
515
+ if not self.current_rows:
516
+ return
517
+ table = pa.concat_tables(self.current_rows)
518
+ self.write_table(table)
519
+ self.current_rows = []
520
+
521
+ def write(
522
+ self,
523
+ example: dict[str, Any],
524
+ key: Optional[Union[str, int, bytes]] = None,
525
+ writer_batch_size: Optional[int] = None,
526
+ ):
527
+ """Add a given (Example,Key) pair to the write-pool of examples which is written to file.
528
+
529
+ Args:
530
+ example: the Example to add.
531
+ key: Optional, a unique identifier(str, int or bytes) associated with each example
532
+ """
533
+ # Utilize the keys and duplicate checking when `self._check_duplicates` is passed True
534
+ if self._check_duplicates:
535
+ # Create unique hash from key and store as (key, example) pairs
536
+ hash = self._hasher.hash(key)
537
+ self.current_examples.append((example, hash))
538
+ # Maintain record of keys and their respective hashes for checking duplicates
539
+ self.hkey_record.append((hash, key))
540
+ else:
541
+ # Store example as a tuple so as to keep the structure of `self.current_examples` uniform
542
+ self.current_examples.append((example, ""))
543
+
544
+ if writer_batch_size is None:
545
+ writer_batch_size = self.writer_batch_size
546
+ if writer_batch_size is not None and len(self.current_examples) >= writer_batch_size:
547
+ if self._check_duplicates:
548
+ self.check_duplicate_keys()
549
+ # Re-initializing to empty list for next batch
550
+ self.hkey_record = []
551
+
552
+ self.write_examples_on_file()
553
+
554
+ def check_duplicate_keys(self):
555
+ """Raises error if duplicates found in a batch"""
556
+ tmp_record = set()
557
+ for hash, key in self.hkey_record:
558
+ if hash in tmp_record:
559
+ duplicate_key_indices = [
560
+ str(self._num_examples + index)
561
+ for index, (duplicate_hash, _) in enumerate(self.hkey_record)
562
+ if duplicate_hash == hash
563
+ ]
564
+
565
+ raise DuplicatedKeysError(key, duplicate_key_indices)
566
+ else:
567
+ tmp_record.add(hash)
568
+
569
+ def write_row(self, row: pa.Table, writer_batch_size: Optional[int] = None):
570
+ """Add a given single-row Table to the write-pool of rows which is written to file.
571
+
572
+ Args:
573
+ row: the row to add.
574
+ """
575
+ if len(row) != 1:
576
+ raise ValueError(f"Only single-row pyarrow tables are allowed but got table with {len(row)} rows.")
577
+ self.current_rows.append(row)
578
+ if writer_batch_size is None:
579
+ writer_batch_size = self.writer_batch_size
580
+ if writer_batch_size is not None and len(self.current_rows) >= writer_batch_size:
581
+ self.write_rows_on_file()
582
+
583
+ def write_batch(
584
+ self,
585
+ batch_examples: dict[str, list],
586
+ writer_batch_size: Optional[int] = None,
587
+ try_original_type: Optional[bool] = True,
588
+ ):
589
+ """Write a batch of Example to file.
590
+ Ignores the batch if it appears to be empty,
591
+ preventing a potential schema update of unknown types.
592
+
593
+ Args:
594
+ batch_examples: the batch of examples to add.
595
+ try_original_type: use `try_type` when instantiating OptimizedTypedSequence if `True`, otherwise `try_type = None`.
596
+ """
597
+ if batch_examples and len(next(iter(batch_examples.values()))) == 0:
598
+ return
599
+ features = None if self.pa_writer is None and self.update_features else self._features
600
+ try_features = self._features if self.pa_writer is None and self.update_features else None
601
+ arrays = []
602
+ inferred_features = Features()
603
+ # preserve the order the columns
604
+ if self.schema:
605
+ schema_cols = set(self.schema.names)
606
+ batch_cols = batch_examples.keys() # .keys() preserves the order (unlike set)
607
+ common_cols = [col for col in self.schema.names if col in batch_cols]
608
+ extra_cols = [col for col in batch_cols if col not in schema_cols]
609
+ cols = common_cols + extra_cols
610
+ else:
611
+ cols = list(batch_examples)
612
+ for col in cols:
613
+ col_values = batch_examples[col]
614
+ col_type = features[col] if features else None
615
+ if isinstance(col_values, (pa.Array, pa.ChunkedArray)):
616
+ array = cast_array_to_feature(col_values, col_type) if col_type is not None else col_values
617
+ arrays.append(array)
618
+ inferred_features[col] = generate_from_arrow_type(col_values.type)
619
+ else:
620
+ col_try_type = (
621
+ try_features[col]
622
+ if try_features is not None and col in try_features and try_original_type
623
+ else None
624
+ )
625
+ typed_sequence = OptimizedTypedSequence(col_values, type=col_type, try_type=col_try_type, col=col)
626
+ arrays.append(pa.array(typed_sequence))
627
+ inferred_features[col] = typed_sequence.get_inferred_type()
628
+ schema = inferred_features.arrow_schema if self.pa_writer is None else self.schema
629
+ pa_table = pa.Table.from_arrays(arrays, schema=schema)
630
+ self.write_table(pa_table, writer_batch_size)
631
+
632
+ def write_table(self, pa_table: pa.Table, writer_batch_size: Optional[int] = None):
633
+ """Write a Table to file.
634
+
635
+ Args:
636
+ example: the Table to add.
637
+ """
638
+ if writer_batch_size is None:
639
+ writer_batch_size = self.writer_batch_size
640
+ if self.pa_writer is None:
641
+ self._build_writer(inferred_schema=pa_table.schema)
642
+ pa_table = pa_table.combine_chunks()
643
+ pa_table = table_cast(pa_table, self._schema)
644
+ if self.embed_local_files:
645
+ pa_table = embed_table_storage(pa_table)
646
+ self._num_bytes += pa_table.nbytes
647
+ self._num_examples += pa_table.num_rows
648
+ self.pa_writer.write_table(pa_table, writer_batch_size)
649
+
650
+ def finalize(self, close_stream=True):
651
+ self.write_rows_on_file()
652
+ # In case current_examples < writer_batch_size, but user uses finalize()
653
+ if self._check_duplicates:
654
+ self.check_duplicate_keys()
655
+ # Re-initializing to empty list for next batch
656
+ self.hkey_record = []
657
+ self.write_examples_on_file()
658
+ # If schema is known, infer features even if no examples were written
659
+ if self.pa_writer is None and self.schema:
660
+ self._build_writer(self.schema)
661
+ if self.pa_writer is not None:
662
+ self.pa_writer.close()
663
+ self.pa_writer = None
664
+ if close_stream:
665
+ self.stream.close()
666
+ else:
667
+ if close_stream:
668
+ self.stream.close()
669
+ raise SchemaInferenceError("Please pass `features` or at least one example when writing data")
670
+ logger.debug(
671
+ f"Done writing {self._num_examples} {self.unit} in {self._num_bytes} bytes {self._path if self._path else ''}."
672
+ )
673
+ return self._num_examples, self._num_bytes
674
+
675
+
676
+ class ParquetWriter(ArrowWriter):
677
+ _WRITER_CLASS = pq.ParquetWriter
lib/python3.12/site-packages/datasets/builder.py ADDED
@@ -0,0 +1,1903 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ # Lint as: python3
16
+ """DatasetBuilder base class."""
17
+
18
+ import abc
19
+ import contextlib
20
+ import copy
21
+ import inspect
22
+ import os
23
+ import posixpath
24
+ import shutil
25
+ import textwrap
26
+ import time
27
+ import urllib
28
+ from collections.abc import Iterable, Mapping
29
+ from dataclasses import dataclass
30
+ from functools import partial
31
+ from pathlib import Path
32
+ from typing import TYPE_CHECKING, Optional, Union
33
+ from unittest.mock import patch
34
+
35
+ import fsspec
36
+ from fsspec.core import url_to_fs
37
+ from multiprocess import Pool
38
+ from tqdm.contrib.concurrent import thread_map
39
+
40
+ from . import config, utils
41
+ from .arrow_dataset import Dataset
42
+ from .arrow_reader import (
43
+ ArrowReader,
44
+ ReadInstruction,
45
+ )
46
+ from .arrow_writer import ArrowWriter, ParquetWriter, SchemaInferenceError
47
+ from .data_files import DataFilesDict, DataFilesPatternsDict, sanitize_patterns
48
+ from .dataset_dict import DatasetDict, IterableDatasetDict
49
+ from .download.download_config import DownloadConfig
50
+ from .download.download_manager import DownloadManager, DownloadMode
51
+ from .download.streaming_download_manager import StreamingDownloadManager, xjoin
52
+ from .exceptions import DatasetGenerationCastError, DatasetGenerationError, FileFormatError, ManualDownloadError
53
+ from .features import Features
54
+ from .filesystems import (
55
+ is_remote_filesystem,
56
+ rename,
57
+ )
58
+ from .fingerprint import Hasher
59
+ from .info import DatasetInfo, DatasetInfosDict, PostProcessedInfo
60
+ from .iterable_dataset import ArrowExamplesIterable, ExamplesIterable, IterableDataset
61
+ from .keyhash import DuplicatedKeysError
62
+ from .naming import INVALID_WINDOWS_CHARACTERS_IN_PATH, camelcase_to_snakecase
63
+ from .splits import Split, SplitDict, SplitGenerator, SplitInfo
64
+ from .streaming import extend_dataset_builder_for_streaming
65
+ from .table import CastError
66
+ from .utils import logging
67
+ from .utils import tqdm as hf_tqdm
68
+ from .utils._filelock import FileLock
69
+ from .utils.file_utils import is_remote_url
70
+ from .utils.info_utils import VerificationMode, get_size_checksum_dict, verify_checksums, verify_splits
71
+ from .utils.py_utils import (
72
+ classproperty,
73
+ convert_file_size_to_int,
74
+ has_sufficient_disk_space,
75
+ iflatmap_unordered,
76
+ map_nested,
77
+ memoize,
78
+ size_str,
79
+ temporary_assignment,
80
+ )
81
+ from .utils.sharding import _number_of_shards_in_gen_kwargs, _split_gen_kwargs
82
+ from .utils.track import tracked_list
83
+
84
+
85
+ if TYPE_CHECKING:
86
+ from .load import DatasetModule
87
+
88
+
89
+ logger = logging.get_logger(__name__)
90
+
91
+
92
+ class InvalidConfigName(ValueError):
93
+ pass
94
+
95
+
96
+ @dataclass
97
+ class BuilderConfig:
98
+ """Base class for `DatasetBuilder` data configuration.
99
+
100
+ `DatasetBuilder` subclasses with data configuration options should subclass
101
+ `BuilderConfig` and add their own properties.
102
+
103
+ Attributes:
104
+ name (`str`, defaults to `default`):
105
+ The name of the configuration.
106
+ version (`Version` or `str`, defaults to `0.0.0`):
107
+ The version of the configuration.
108
+ data_dir (`str`, *optional*):
109
+ Path to the directory containing the source data.
110
+ data_files (`str` or `Sequence` or `Mapping`, *optional*):
111
+ Path(s) to source data file(s).
112
+ description (`str`, *optional*):
113
+ A human description of the configuration.
114
+ """
115
+
116
+ name: str = "default"
117
+ version: Optional[Union[utils.Version, str]] = utils.Version("0.0.0")
118
+ data_dir: Optional[str] = None
119
+ data_files: Optional[Union[DataFilesDict, DataFilesPatternsDict]] = None
120
+ description: Optional[str] = None
121
+
122
+ def __post_init__(self):
123
+ # The config name is used to name the cache directory.
124
+ for invalid_char in INVALID_WINDOWS_CHARACTERS_IN_PATH:
125
+ if invalid_char in self.name:
126
+ raise InvalidConfigName(
127
+ f"Bad characters from black list '{INVALID_WINDOWS_CHARACTERS_IN_PATH}' found in '{self.name}'. "
128
+ f"They could create issues when creating a directory for this config on Windows filesystem."
129
+ )
130
+ if self.data_files is not None and not isinstance(self.data_files, (DataFilesDict, DataFilesPatternsDict)):
131
+ raise ValueError(f"Expected a DataFilesDict in data_files but got {self.data_files}")
132
+
133
+ def __eq__(self, o):
134
+ # we need to override the default dataclass __eq__ since it doesn't check for
135
+ # other attributes that the ones of the signature.
136
+ if set(self.__dict__.keys()) != set(o.__dict__.keys()):
137
+ return False
138
+ return all((k, getattr(self, k)) == (k, getattr(o, k)) for k in self.__dict__.keys())
139
+
140
+ def create_config_id(
141
+ self,
142
+ config_kwargs: dict,
143
+ custom_features: Optional[Features] = None,
144
+ ) -> str:
145
+ """
146
+ The config id is used to build the cache directory.
147
+ By default it is equal to the config name.
148
+ However the name of a config is not sufficient to have a unique identifier for the dataset being generated
149
+ since it doesn't take into account:
150
+ - the config kwargs that can be used to overwrite attributes
151
+ - the custom features used to write the dataset
152
+ - the data_files for json/text/csv/pandas datasets
153
+
154
+ Therefore the config id is just the config name with an optional suffix based on these.
155
+ """
156
+ # Possibly add a suffix to the name to handle custom features/data_files/config_kwargs
157
+ suffix: Optional[str] = None
158
+ config_kwargs_to_add_to_suffix = config_kwargs.copy()
159
+ # name and version are already used to build the cache directory
160
+ config_kwargs_to_add_to_suffix.pop("name", None)
161
+ config_kwargs_to_add_to_suffix.pop("version", None)
162
+ # data dir handling (when specified it points to the manually downloaded data):
163
+ # it was previously ignored before the introduction of config id because we didn't want
164
+ # to change the config name. Now it's fine to take it into account for the config id.
165
+ # config_kwargs_to_add_to_suffix.pop("data_dir", None)
166
+ if "data_dir" in config_kwargs_to_add_to_suffix:
167
+ if config_kwargs_to_add_to_suffix["data_dir"] is None:
168
+ config_kwargs_to_add_to_suffix.pop("data_dir", None)
169
+ else:
170
+ # canonicalize the data dir to avoid two paths to the same location having different
171
+ # hashes
172
+ data_dir = config_kwargs_to_add_to_suffix["data_dir"]
173
+ data_dir = os.path.normpath(data_dir)
174
+ config_kwargs_to_add_to_suffix["data_dir"] = data_dir
175
+ if config_kwargs_to_add_to_suffix:
176
+ # we don't care about the order of the kwargs
177
+ config_kwargs_to_add_to_suffix = {
178
+ k: config_kwargs_to_add_to_suffix[k] for k in sorted(config_kwargs_to_add_to_suffix)
179
+ }
180
+ if all(isinstance(v, (str, bool, int, float)) for v in config_kwargs_to_add_to_suffix.values()):
181
+ suffix = ",".join(
182
+ str(k) + "=" + urllib.parse.quote_plus(str(v)) for k, v in config_kwargs_to_add_to_suffix.items()
183
+ )
184
+ if len(suffix) > 32: # hash if too long
185
+ suffix = Hasher.hash(config_kwargs_to_add_to_suffix)
186
+ else:
187
+ suffix = Hasher.hash(config_kwargs_to_add_to_suffix)
188
+
189
+ if custom_features is not None:
190
+ m = Hasher()
191
+ if suffix:
192
+ m.update(suffix)
193
+ m.update(custom_features)
194
+ suffix = m.hexdigest()
195
+
196
+ if suffix:
197
+ config_id = self.name + "-" + suffix
198
+ if len(config_id) > config.MAX_DATASET_CONFIG_ID_READABLE_LENGTH:
199
+ config_id = self.name + "-" + Hasher.hash(suffix)
200
+ return config_id
201
+ else:
202
+ return self.name
203
+
204
+ def _resolve_data_files(self, base_path: str, download_config: DownloadConfig) -> None:
205
+ if isinstance(self.data_files, DataFilesPatternsDict):
206
+ base_path = xjoin(base_path, self.data_dir) if self.data_dir else base_path
207
+ self.data_files = self.data_files.resolve(base_path, download_config)
208
+
209
+
210
+ class DatasetBuilder:
211
+ """Abstract base class for all datasets.
212
+
213
+ `DatasetBuilder` has 3 key methods:
214
+
215
+ - [`DatasetBuilder.info`]: Documents the dataset, including feature
216
+ names, types, shapes, version, splits, citation, etc.
217
+ - [`DatasetBuilder.download_and_prepare`]: Downloads the source data
218
+ and writes it to disk.
219
+ - [`DatasetBuilder.as_dataset`]: Generates a [`Dataset`].
220
+
221
+ Some `DatasetBuilder`s expose multiple variants of the
222
+ dataset by defining a [`BuilderConfig`] subclass and accepting a
223
+ config object (or name) on construction. Configurable datasets expose a
224
+ pre-defined set of configurations in [`DatasetBuilder.builder_configs`].
225
+
226
+ Args:
227
+ cache_dir (`str`, *optional*):
228
+ Directory to cache data. Defaults to `"~/.cache/huggingface/datasets"`.
229
+ dataset_name (`str`, *optional*):
230
+ Name of the dataset, if different from the builder name. Useful for packaged builders
231
+ like csv, imagefolder, audiofolder, etc. to reflect the difference between datasets
232
+ that use the same packaged builder.
233
+ config_name (`str`, *optional*):
234
+ Name of the dataset configuration.
235
+ It affects the data generated on disk. Different configurations will have their own subdirectories and
236
+ versions.
237
+ If not provided, the default configuration is used (if it exists).
238
+
239
+ <Added version="2.3.0">
240
+
241
+ Parameter `name` was renamed to `config_name`.
242
+
243
+ </Added>
244
+ hash (`str`, *optional*):
245
+ Hash specific to the dataset code. Used to update the caching directory when the
246
+ dataset loading script code is updated (to avoid reusing old data).
247
+ The typical caching directory (defined in `self._relative_data_dir`) is `name/version/hash/`.
248
+ base_path (`str`, *optional*):
249
+ Base path for relative paths that are used to download files.
250
+ This can be a remote URL.
251
+ features ([`Features`], *optional*):
252
+ Features types to use with this dataset.
253
+ It can be used to change the [`Features`] types of a dataset, for example.
254
+ token (`str` or `bool`, *optional*):
255
+ String or boolean to use as Bearer token for remote files on the
256
+ Datasets Hub. If `True`, will get token from `"~/.huggingface"`.
257
+ repo_id (`str`, *optional*):
258
+ ID of the dataset repository.
259
+ Used to distinguish builders with the same name but not coming from the same namespace, for example "rajpurkar/squad"
260
+ and "lhoestq/squad" repo IDs. In the latter, the builder name would be "lhoestq___squad".
261
+ data_files (`str` or `Sequence` or `Mapping`, *optional*):
262
+ Path(s) to source data file(s).
263
+ For builders like "csv" or "json" that need the user to specify data files. They can be either
264
+ local or remote files. For convenience, you can use a `DataFilesDict`.
265
+ data_dir (`str`, *optional*):
266
+ Path to directory containing source data file(s).
267
+ Use only if `data_files` is not passed, in which case it is equivalent to passing
268
+ `os.path.join(data_dir, "**")` as `data_files`.
269
+ For builders that require manual download, it must be the path to the local directory containing the
270
+ manually downloaded data.
271
+ storage_options (`dict`, *optional*):
272
+ Key/value pairs to be passed on to the dataset file-system backend, if any.
273
+ writer_batch_size (`int`, *optional*):
274
+ Batch size used by the ArrowWriter.
275
+ It defines the number of samples that are kept in memory before writing them
276
+ and also the length of the arrow chunks.
277
+ None means that the ArrowWriter will use its default value.
278
+ **config_kwargs (additional keyword arguments): Keyword arguments to be passed to the corresponding builder
279
+ configuration class, set on the class attribute [`DatasetBuilder.BUILDER_CONFIG_CLASS`]. The builder
280
+ configuration class is [`BuilderConfig`] or a subclass of it.
281
+ """
282
+
283
+ # Default version
284
+ VERSION = None # Default version set in BuilderConfig
285
+
286
+ # Class for the builder config.
287
+ BUILDER_CONFIG_CLASS = BuilderConfig
288
+
289
+ # Named configurations that modify the data generated by download_and_prepare.
290
+ BUILDER_CONFIGS = []
291
+
292
+ # Optional default config name to be used when name is None
293
+ DEFAULT_CONFIG_NAME = None
294
+
295
+ # Default batch size used by the ArrowWriter
296
+ # It defines the number of samples that are kept in memory before writing them
297
+ # and also the length of the arrow chunks
298
+ # None means that the ArrowWriter will use its default value
299
+ DEFAULT_WRITER_BATCH_SIZE = None
300
+
301
+ def __init__(
302
+ self,
303
+ cache_dir: Optional[str] = None,
304
+ dataset_name: Optional[str] = None,
305
+ config_name: Optional[str] = None,
306
+ hash: Optional[str] = None,
307
+ base_path: Optional[str] = None,
308
+ info: Optional[DatasetInfo] = None,
309
+ features: Optional[Features] = None,
310
+ token: Optional[Union[bool, str]] = None,
311
+ repo_id: Optional[str] = None,
312
+ data_files: Optional[Union[str, list, dict, DataFilesDict]] = None,
313
+ data_dir: Optional[str] = None,
314
+ storage_options: Optional[dict] = None,
315
+ writer_batch_size: Optional[int] = None,
316
+ **config_kwargs,
317
+ ):
318
+ # DatasetBuilder name
319
+ self.name: str = camelcase_to_snakecase(self.__module__.split(".")[-1])
320
+ self.hash: Optional[str] = hash
321
+ self.base_path = base_path
322
+ self.token = token
323
+ self.repo_id = repo_id
324
+ self.storage_options = storage_options or {}
325
+ self.dataset_name = camelcase_to_snakecase(dataset_name) if dataset_name else self.name
326
+ self._writer_batch_size = writer_batch_size or self.DEFAULT_WRITER_BATCH_SIZE
327
+
328
+ if data_files is not None and not isinstance(data_files, DataFilesDict):
329
+ data_files = DataFilesDict.from_patterns(
330
+ sanitize_patterns(data_files),
331
+ base_path=base_path,
332
+ download_config=DownloadConfig(token=token, storage_options=self.storage_options),
333
+ )
334
+
335
+ # Prepare config: DatasetConfig contains name, version and description but can be extended by each dataset
336
+ if "features" in inspect.signature(self.BUILDER_CONFIG_CLASS.__init__).parameters and features is not None:
337
+ config_kwargs["features"] = features
338
+ if data_files is not None:
339
+ config_kwargs["data_files"] = data_files
340
+ if data_dir is not None:
341
+ config_kwargs["data_dir"] = data_dir
342
+ self.config_kwargs = config_kwargs
343
+ self.config, self.config_id = self._create_builder_config(
344
+ config_name=config_name,
345
+ custom_features=features,
346
+ **config_kwargs,
347
+ )
348
+
349
+ # prepare info: DatasetInfo are a standardized dataclass across all datasets
350
+ # Prefill datasetinfo
351
+ if info is None:
352
+ # TODO FOR PACKAGED MODULES IT IMPORTS DATA FROM src/packaged_modules which doesn't make sense
353
+ info = self.get_exported_dataset_info()
354
+ info.update(self._info())
355
+ info.builder_name = self.name
356
+ info.dataset_name = self.dataset_name
357
+ info.config_name = self.config.name
358
+ info.version = self.config.version
359
+ self.info = info
360
+ # update info with user specified infos
361
+ if features is not None:
362
+ self.info.features = features
363
+
364
+ # Prepare data dirs:
365
+ # cache_dir can be a remote bucket on GCS or S3
366
+ self._cache_dir_root = str(cache_dir or config.HF_DATASETS_CACHE)
367
+ self._cache_dir_root = (
368
+ self._cache_dir_root if is_remote_url(self._cache_dir_root) else os.path.expanduser(self._cache_dir_root)
369
+ )
370
+ self._cache_downloaded_dir = (
371
+ posixpath.join(self._cache_dir_root, config.DOWNLOADED_DATASETS_DIR)
372
+ if cache_dir
373
+ else str(config.DOWNLOADED_DATASETS_PATH)
374
+ )
375
+ self._cache_downloaded_dir = (
376
+ self._cache_downloaded_dir
377
+ if is_remote_url(self._cache_downloaded_dir)
378
+ else os.path.expanduser(self._cache_downloaded_dir)
379
+ )
380
+
381
+ # In case there exists a legacy cache directory
382
+ self._legacy_relative_data_dir = None
383
+
384
+ self._cache_dir = self._build_cache_dir()
385
+ if not is_remote_url(self._cache_dir_root):
386
+ os.makedirs(self._cache_dir_root, exist_ok=True)
387
+ lock_path = os.path.join(
388
+ self._cache_dir_root, Path(self._cache_dir).as_posix().replace("/", "_") + ".lock"
389
+ )
390
+ with FileLock(lock_path):
391
+ if os.path.exists(self._cache_dir): # check if data exist
392
+ if len(os.listdir(self._cache_dir)) > 0:
393
+ if os.path.exists(os.path.join(self._cache_dir, config.DATASET_INFO_FILENAME)):
394
+ logger.info("Overwrite dataset info from restored data version if exists.")
395
+ self.info = DatasetInfo.from_directory(self._cache_dir)
396
+ else: # dir exists but no data, remove the empty dir as data aren't available anymore
397
+ logger.warning(
398
+ f"Old caching folder {self._cache_dir} for dataset {self.dataset_name} exists but no data were found. Removing it. "
399
+ )
400
+ os.rmdir(self._cache_dir)
401
+
402
+ # Store in the cache by default unless the user specifies a custom output_dir to download_and_prepare
403
+ self._output_dir = self._cache_dir
404
+ self._fs: fsspec.AbstractFileSystem = fsspec.filesystem("file")
405
+
406
+ # Set download manager
407
+ self.dl_manager = None
408
+
409
+ # Set to True by "datasets-cli test" to generate file checksums for (deprecated) dataset_infos.json independently of verification_mode value.
410
+ self._record_infos = False
411
+
412
+ # Set in `.download_and_prepare` once the format of the generated dataset is known
413
+ self._file_format = None
414
+
415
+ # Enable streaming (e.g. it patches "open" to work with remote files)
416
+ extend_dataset_builder_for_streaming(self)
417
+
418
+ def __getstate__(self):
419
+ return self.__dict__
420
+
421
+ def __setstate__(self, d):
422
+ self.__dict__ = d
423
+ # Re-enable streaming, since patched functions are not kept when pickling
424
+ extend_dataset_builder_for_streaming(self)
425
+
426
+ # Must be set for datasets that use 'data_dir' functionality - the ones
427
+ # that require users to do additional steps to download the data
428
+ # (this is usually due to some external regulations / rules).
429
+ # This field should contain a string with user instructions, including
430
+ # the list of files that should be present. It will be
431
+ # displayed in the dataset documentation.
432
+ @property
433
+ def manual_download_instructions(self) -> Optional[str]:
434
+ return None
435
+
436
+ def _check_legacy_cache(self) -> Optional[str]:
437
+ """Check for the old cache directory template {cache_dir}/{namespace}___{builder_name} from 2.13"""
438
+ if (
439
+ self.__module__.startswith("datasets.")
440
+ and not is_remote_url(self._cache_dir_root)
441
+ and self.config.name == "default"
442
+ ):
443
+ from .packaged_modules import _PACKAGED_DATASETS_MODULES
444
+
445
+ namespace = self.repo_id.split("/")[0] if self.repo_id and self.repo_id.count("/") > 0 else None
446
+ config_name = self.repo_id.replace("/", "--") if self.repo_id is not None else self.dataset_name
447
+ config_id = config_name + self.config_id[len(self.config.name) :]
448
+ hash = _PACKAGED_DATASETS_MODULES.get(self.name, "missing")[1]
449
+ legacy_relative_data_dir = posixpath.join(
450
+ self.dataset_name if namespace is None else f"{namespace}___{self.dataset_name}",
451
+ config_id,
452
+ "0.0.0",
453
+ hash,
454
+ )
455
+ legacy_cache_dir = posixpath.join(self._cache_dir_root, legacy_relative_data_dir)
456
+ if os.path.isdir(legacy_cache_dir):
457
+ return legacy_relative_data_dir
458
+
459
+ def _check_legacy_cache2(self, dataset_module: "DatasetModule") -> Optional[str]:
460
+ """Check for the old cache directory template {cache_dir}/{namespace}___{dataset_name}/{config_name}-xxx from 2.14 and 2.15"""
461
+ if (
462
+ self.__module__.startswith("datasets.")
463
+ and not is_remote_url(self._cache_dir_root)
464
+ and not (set(self.config_kwargs) - {"data_files", "data_dir"})
465
+ ):
466
+ from .packaged_modules import _PACKAGED_DATASETS_MODULES_2_15_HASHES
467
+ from .utils._dill import Pickler
468
+
469
+ def update_hash_with_config_parameters(hash: str, config_parameters: dict) -> str:
470
+ """
471
+ Used to update hash of packaged modules which is used for creating unique cache directories to reflect
472
+ different config parameters which are passed in metadata from readme.
473
+ """
474
+ params_to_exclude = {"config_name", "version", "description"}
475
+ params_to_add_to_hash = {
476
+ param: value
477
+ for param, value in sorted(config_parameters.items())
478
+ if param not in params_to_exclude
479
+ }
480
+ m = Hasher()
481
+ m.update(hash)
482
+ m.update(params_to_add_to_hash)
483
+ return m.hexdigest()
484
+
485
+ namespace = self.repo_id.split("/")[0] if self.repo_id and self.repo_id.count("/") > 0 else None
486
+ with patch.object(Pickler, "_legacy_no_dict_keys_sorting", True):
487
+ config_id = self.config.name + "-" + Hasher.hash({"data_files": self.config.data_files})
488
+ hash = _PACKAGED_DATASETS_MODULES_2_15_HASHES.get(self.name, "missing")
489
+ if (
490
+ dataset_module.builder_configs_parameters.metadata_configs
491
+ and self.config.name in dataset_module.builder_configs_parameters.metadata_configs
492
+ ):
493
+ hash = update_hash_with_config_parameters(
494
+ hash, dataset_module.builder_configs_parameters.metadata_configs[self.config.name]
495
+ )
496
+ legacy_relative_data_dir = posixpath.join(
497
+ self.dataset_name if namespace is None else f"{namespace}___{self.dataset_name}",
498
+ config_id,
499
+ "0.0.0",
500
+ hash,
501
+ )
502
+ legacy_cache_dir = posixpath.join(self._cache_dir_root, legacy_relative_data_dir)
503
+ if os.path.isdir(legacy_cache_dir):
504
+ return legacy_relative_data_dir
505
+
506
+ @classmethod
507
+ def get_all_exported_dataset_infos(cls) -> DatasetInfosDict:
508
+ """Empty dict if doesn't exist
509
+
510
+ Example:
511
+
512
+ ```py
513
+ >>> from datasets import load_dataset_builder
514
+ >>> ds_builder = load_dataset_builder('vivos')
515
+ >>> ds_builder.get_all_exported_dataset_infos()
516
+ {'default': DatasetInfo(description='', citation='', homepage='', license='', features={'speaker_id': Value(dtype='string', id=None), 'path': Value(dtype='string', id=None), 'audio': Audio(sampling_rate=16000, mono=True, decode=True, id=None), 'sentence': Value(dtype='string', id=None)}, post_processed=None, supervised_keys=None, builder_name=None, dataset_name=None, config_name='default', version=None, splits={'train': SplitInfo(name='train', num_bytes=1722002133, num_examples=11660, shard_lengths=None, dataset_name=None), 'test': SplitInfo(name='test', num_bytes=86120227, num_examples=760, shard_lengths=None, dataset_name=None)}, download_checksums=None, download_size=1475540500, post_processing_size=None, dataset_size=1808122360, size_in_bytes=None)}
517
+ ```
518
+ """
519
+ return DatasetInfosDict.from_directory(cls.get_imported_module_dir())
520
+
521
+ def get_exported_dataset_info(self) -> DatasetInfo:
522
+ """Empty `DatasetInfo` if doesn't exist
523
+
524
+ Example:
525
+
526
+ ```py
527
+ >>> from datasets import load_dataset_builder
528
+ >>> ds_builder = load_dataset_builder('cornell-movie-review-data/rotten_tomatoes')
529
+ >>> ds_builder.get_exported_dataset_info()
530
+ DatasetInfo(description='', citation='', homepage='', license='', features={'speaker_id': Value(dtype='string', id=None), 'path': Value(dtype='string', id=None), 'audio': Audio(sampling_rate=16000, mono=True, decode=True, id=None), 'sentence': Value(dtype='string', id=None)}, post_processed=None, supervised_keys=None, builder_name=None, dataset_name=None, config_name='default', version=None, splits={'train': SplitInfo(name='train', num_bytes=1722002133, num_examples=11660, shard_lengths=None, dataset_name=None), 'test': SplitInfo(name='test', num_bytes=86120227, num_examples=760, shard_lengths=None, dataset_name=None)}, download_checksums=None, download_size=1475540500, post_processing_size=None, dataset_size=1808122360, size_in_bytes=None)
531
+ ```
532
+ """
533
+ return self.get_all_exported_dataset_infos().get(self.config.name, DatasetInfo())
534
+
535
+ def _create_builder_config(
536
+ self, config_name=None, custom_features=None, **config_kwargs
537
+ ) -> tuple[BuilderConfig, str]:
538
+ """Create and validate BuilderConfig object as well as a unique config id for this config.
539
+ Raises ValueError if there are multiple builder configs and config_name and DEFAULT_CONFIG_NAME are None.
540
+ config_kwargs override the defaults kwargs in config
541
+ """
542
+ builder_config = None
543
+
544
+ # try default config
545
+ if config_name is None and self.BUILDER_CONFIGS:
546
+ if self.DEFAULT_CONFIG_NAME is not None:
547
+ builder_config = self.builder_configs.get(self.DEFAULT_CONFIG_NAME)
548
+ logger.info(f"No config specified, defaulting to: {self.dataset_name}/{builder_config.name}")
549
+ else:
550
+ if len(self.BUILDER_CONFIGS) > 1:
551
+ if not config_kwargs:
552
+ example_of_usage = (
553
+ f"load_dataset('{self.repo_id or self.dataset_name}', '{self.BUILDER_CONFIGS[0].name}')"
554
+ )
555
+ raise ValueError(
556
+ "Config name is missing."
557
+ f"\nPlease pick one among the available configs: {list(self.builder_configs.keys())}"
558
+ + f"\nExample of usage:\n\t`{example_of_usage}`"
559
+ )
560
+ else:
561
+ builder_config = self.BUILDER_CONFIGS[0]
562
+ logger.info(
563
+ f"No config specified, defaulting to the single config: {self.dataset_name}/{builder_config.name}"
564
+ )
565
+
566
+ # try to get config by name
567
+ if isinstance(config_name, str):
568
+ builder_config = self.builder_configs.get(config_name)
569
+ if builder_config is None and self.BUILDER_CONFIGS:
570
+ raise ValueError(
571
+ f"BuilderConfig '{config_name}' not found. Available: {list(self.builder_configs.keys())}"
572
+ )
573
+
574
+ # if not using an existing config, then create a new config on the fly
575
+ if not builder_config:
576
+ if config_name is not None:
577
+ config_kwargs["name"] = config_name
578
+ elif self.DEFAULT_CONFIG_NAME and not config_kwargs:
579
+ # Use DEFAULT_CONFIG_NAME only if no config_kwargs are passed
580
+ config_kwargs["name"] = self.DEFAULT_CONFIG_NAME
581
+ if "version" not in config_kwargs and hasattr(self, "VERSION") and self.VERSION:
582
+ config_kwargs["version"] = self.VERSION
583
+ builder_config = self.BUILDER_CONFIG_CLASS(**config_kwargs)
584
+
585
+ # otherwise use the config_kwargs to overwrite the attributes
586
+ else:
587
+ builder_config = copy.deepcopy(builder_config) if config_kwargs else builder_config
588
+ for key, value in config_kwargs.items():
589
+ if value is not None:
590
+ if not hasattr(builder_config, key):
591
+ raise ValueError(f"BuilderConfig {builder_config} doesn't have a '{key}' key.")
592
+ setattr(builder_config, key, value)
593
+
594
+ if not builder_config.name:
595
+ raise ValueError(f"BuilderConfig must have a name, got {builder_config.name}")
596
+
597
+ # resolve data files if needed
598
+ builder_config._resolve_data_files(
599
+ base_path=self.base_path,
600
+ download_config=DownloadConfig(token=self.token, storage_options=self.storage_options),
601
+ )
602
+
603
+ # compute the config id that is going to be used for caching
604
+ config_id = builder_config.create_config_id(
605
+ config_kwargs,
606
+ custom_features=custom_features,
607
+ )
608
+ is_custom = (config_id not in self.builder_configs) and config_id != "default"
609
+ if is_custom:
610
+ logger.info(f"Using custom data configuration {config_id}")
611
+ else:
612
+ if (
613
+ builder_config.name in self.builder_configs
614
+ and builder_config != self.builder_configs[builder_config.name]
615
+ ):
616
+ raise ValueError(
617
+ "Cannot name a custom BuilderConfig the same as an available "
618
+ f"BuilderConfig. Change the name. Available BuilderConfigs: {list(self.builder_configs.keys())}"
619
+ )
620
+ if not builder_config.version:
621
+ raise ValueError(f"BuilderConfig {builder_config.name} must have a version")
622
+
623
+ return builder_config, config_id
624
+
625
+ @classproperty
626
+ @classmethod
627
+ @memoize()
628
+ def builder_configs(cls) -> dict[str, BuilderConfig]:
629
+ """Dictionary of pre-defined configurations for this builder class."""
630
+ configs = {config.name: config for config in cls.BUILDER_CONFIGS}
631
+ if len(configs) != len(cls.BUILDER_CONFIGS):
632
+ names = [config.name for config in cls.BUILDER_CONFIGS]
633
+ raise ValueError(f"Names in BUILDER_CONFIGS must not be duplicated. Got {names}")
634
+ return configs
635
+
636
+ @property
637
+ def cache_dir(self):
638
+ return self._cache_dir
639
+
640
+ def _use_legacy_cache_dir_if_possible(self, dataset_module: "DatasetModule"):
641
+ # Check for the legacy cache directory template (datasets<3.0.0)
642
+ self._legacy_relative_data_dir = (
643
+ self._check_legacy_cache2(dataset_module) or self._check_legacy_cache() or None
644
+ )
645
+ self._cache_dir = self._build_cache_dir()
646
+ self._output_dir = self._cache_dir
647
+
648
+ def _relative_data_dir(self, with_version=True, with_hash=True) -> str:
649
+ """Relative path of this dataset in cache_dir:
650
+ Will be:
651
+ self.dataset_name/self.config.version/self.hash/
652
+ or if a repo_id with a namespace has been specified:
653
+ self.namespace___self.dataset_name/self.config.version/self.hash/
654
+ If any of these element is missing or if ``with_version=False`` the corresponding subfolders are dropped.
655
+ """
656
+ if self._legacy_relative_data_dir is not None and with_version and with_hash:
657
+ return self._legacy_relative_data_dir
658
+
659
+ namespace = self.repo_id.split("/")[0] if self.repo_id and self.repo_id.count("/") > 0 else None
660
+ builder_data_dir = self.dataset_name if namespace is None else f"{namespace}___{self.dataset_name}"
661
+ builder_data_dir = posixpath.join(builder_data_dir, self.config_id)
662
+ if with_version:
663
+ builder_data_dir = posixpath.join(builder_data_dir, str(self.config.version))
664
+ if with_hash and self.hash and isinstance(self.hash, str):
665
+ builder_data_dir = posixpath.join(builder_data_dir, self.hash)
666
+ return builder_data_dir
667
+
668
+ def _build_cache_dir(self):
669
+ """Return the data directory for the current version."""
670
+ builder_data_dir = posixpath.join(self._cache_dir_root, self._relative_data_dir(with_version=False))
671
+ version_data_dir = posixpath.join(self._cache_dir_root, self._relative_data_dir(with_version=True))
672
+
673
+ def _other_versions_on_disk():
674
+ """Returns previous versions on disk."""
675
+ if not os.path.exists(builder_data_dir):
676
+ return []
677
+
678
+ version_dirnames = []
679
+ for dir_name in os.listdir(builder_data_dir):
680
+ try:
681
+ version_dirnames.append((utils.Version(dir_name), dir_name))
682
+ except ValueError: # Invalid version (ex: incomplete data dir)
683
+ pass
684
+ version_dirnames.sort(reverse=True)
685
+ return version_dirnames
686
+
687
+ # Check and warn if other versions exist
688
+ if not is_remote_url(builder_data_dir):
689
+ version_dirs = _other_versions_on_disk()
690
+ if version_dirs:
691
+ other_version = version_dirs[0][0]
692
+ if other_version != self.config.version:
693
+ warn_msg = (
694
+ f"Found a different version {str(other_version)} of dataset {self.dataset_name} in "
695
+ f"cache_dir {self._cache_dir_root}. Using currently defined version "
696
+ f"{str(self.config.version)}."
697
+ )
698
+ logger.warning(warn_msg)
699
+
700
+ return version_data_dir
701
+
702
+ @abc.abstractmethod
703
+ def _info(self) -> DatasetInfo:
704
+ """Construct the DatasetInfo object. See `DatasetInfo` for details.
705
+
706
+ Warning: This function is only called once and the result is cached for all
707
+ following .info() calls.
708
+
709
+ Returns:
710
+ info: (DatasetInfo) The dataset information
711
+ """
712
+ raise NotImplementedError
713
+
714
+ @classmethod
715
+ def get_imported_module_dir(cls):
716
+ """Return the path of the module of this class or subclass."""
717
+ return os.path.dirname(inspect.getfile(inspect.getmodule(cls)))
718
+
719
+ def _rename(self, src: str, dst: str):
720
+ rename(self._fs, src, dst)
721
+
722
+ def download_and_prepare(
723
+ self,
724
+ output_dir: Optional[str] = None,
725
+ download_config: Optional[DownloadConfig] = None,
726
+ download_mode: Optional[Union[DownloadMode, str]] = None,
727
+ verification_mode: Optional[Union[VerificationMode, str]] = None,
728
+ dl_manager: Optional[DownloadManager] = None,
729
+ base_path: Optional[str] = None,
730
+ file_format: str = "arrow",
731
+ max_shard_size: Optional[Union[int, str]] = None,
732
+ num_proc: Optional[int] = None,
733
+ storage_options: Optional[dict] = None,
734
+ **download_and_prepare_kwargs,
735
+ ):
736
+ """Downloads and prepares dataset for reading.
737
+
738
+ Args:
739
+ output_dir (`str`, *optional*):
740
+ Output directory for the dataset.
741
+ Default to this builder's `cache_dir`, which is inside `~/.cache/huggingface/datasets` by default.
742
+
743
+ <Added version="2.5.0"/>
744
+ download_config (`DownloadConfig`, *optional*):
745
+ Specific download configuration parameters.
746
+ download_mode ([`DownloadMode`] or `str`, *optional*):
747
+ Select the download/generate mode, default to `REUSE_DATASET_IF_EXISTS`.
748
+ verification_mode ([`VerificationMode`] or `str`, defaults to `BASIC_CHECKS`):
749
+ Verification mode determining the checks to run on the downloaded/processed dataset information (checksums/size/splits/...).
750
+
751
+ <Added version="2.9.1"/>
752
+ dl_manager (`DownloadManager`, *optional*):
753
+ Specific `DownloadManger` to use.
754
+ base_path (`str`, *optional*):
755
+ Base path for relative paths that are used to download files. This can be a remote url.
756
+ If not specified, the value of the `base_path` attribute (`self.base_path`) will be used instead.
757
+ file_format (`str`, *optional*):
758
+ Format of the data files in which the dataset will be written.
759
+ Supported formats: "arrow", "parquet". Default to "arrow" format.
760
+ If the format is "parquet", then image and audio data are embedded into the Parquet files instead of pointing to local files.
761
+
762
+ <Added version="2.5.0"/>
763
+ max_shard_size (`Union[str, int]`, *optional*):
764
+ Maximum number of bytes written per shard, default is "500MB".
765
+ The size is based on uncompressed data size, so in practice your shard files may be smaller than
766
+ `max_shard_size` thanks to Parquet compression for example.
767
+
768
+ <Added version="2.5.0"/>
769
+ num_proc (`int`, *optional*, defaults to `None`):
770
+ Number of processes when downloading and generating the dataset locally.
771
+ Multiprocessing is disabled by default.
772
+
773
+ <Added version="2.7.0"/>
774
+ storage_options (`dict`, *optional*):
775
+ Key/value pairs to be passed on to the caching file-system backend, if any.
776
+
777
+ <Added version="2.5.0"/>
778
+ **download_and_prepare_kwargs (additional keyword arguments): Keyword arguments.
779
+
780
+ Example:
781
+
782
+ Download and prepare the dataset as Arrow files that can be loaded as a Dataset using `builder.as_dataset()`:
783
+
784
+ ```py
785
+ >>> from datasets import load_dataset_builder
786
+ >>> builder = load_dataset_builder("cornell-movie-review-data/rotten_tomatoes")
787
+ >>> builder.download_and_prepare()
788
+ ```
789
+
790
+ Download and prepare the dataset as sharded Parquet files locally:
791
+
792
+ ```py
793
+ >>> from datasets import load_dataset_builder
794
+ >>> builder = load_dataset_builder("cornell-movie-review-data/rotten_tomatoes")
795
+ >>> builder.download_and_prepare("./output_dir", file_format="parquet")
796
+ ```
797
+
798
+ Download and prepare the dataset as sharded Parquet files in a cloud storage:
799
+
800
+ ```py
801
+ >>> from datasets import load_dataset_builder
802
+ >>> storage_options = {"key": aws_access_key_id, "secret": aws_secret_access_key}
803
+ >>> builder = load_dataset_builder("cornell-movie-review-data/rotten_tomatoes")
804
+ >>> builder.download_and_prepare("s3://my-bucket/my_rotten_tomatoes", storage_options=storage_options, file_format="parquet")
805
+ ```
806
+ """
807
+ output_dir = output_dir if output_dir is not None else self._cache_dir
808
+ # output_dir can be a remote bucket on GCS or S3
809
+ fs, output_dir = url_to_fs(output_dir, **(storage_options or {}))
810
+ self._fs = fs
811
+ self._output_dir = output_dir if not is_remote_filesystem(self._fs) else self._fs.unstrip_protocol(output_dir)
812
+
813
+ download_mode = DownloadMode(download_mode or DownloadMode.REUSE_DATASET_IF_EXISTS)
814
+ verification_mode = VerificationMode(verification_mode or VerificationMode.BASIC_CHECKS)
815
+ base_path = base_path if base_path is not None else self.base_path
816
+
817
+ if file_format is not None and file_format not in ["arrow", "parquet"]:
818
+ raise ValueError(f"Unsupported file_format: {file_format}. Expected 'arrow' or 'parquet'")
819
+ self._file_format = file_format
820
+
821
+ if self._fs._strip_protocol(self._output_dir) == "":
822
+ # We don't support the root directory, because it has no dirname,
823
+ # and we need a dirname to use a <dirname>.incomplete directory
824
+ # when the dataset is being written
825
+ raise RuntimeError(
826
+ f"Unable to download and prepare the dataset at the root {self._output_dir}. "
827
+ f"Please specify a subdirectory, e.g. '{self._output_dir + self.dataset_name}'"
828
+ )
829
+
830
+ if dl_manager is None:
831
+ if download_config is None:
832
+ download_config = DownloadConfig(
833
+ cache_dir=self._cache_downloaded_dir,
834
+ force_download=download_mode == DownloadMode.FORCE_REDOWNLOAD,
835
+ force_extract=download_mode == DownloadMode.FORCE_REDOWNLOAD,
836
+ use_etag=False,
837
+ num_proc=num_proc,
838
+ token=self.token,
839
+ storage_options=self.storage_options,
840
+ ) # We don't use etag for data files to speed up the process
841
+
842
+ dl_manager = DownloadManager(
843
+ dataset_name=self.dataset_name,
844
+ download_config=download_config,
845
+ data_dir=self.config.data_dir,
846
+ base_path=base_path,
847
+ record_checksums=(self._record_infos or verification_mode == VerificationMode.ALL_CHECKS),
848
+ )
849
+
850
+ is_local = not is_remote_filesystem(self._fs)
851
+ self.dl_manager = dl_manager
852
+
853
+ # Prevent parallel local disk operations
854
+ if is_local:
855
+ # Create parent directory of the output_dir to put the lock file in there
856
+ Path(self._output_dir).parent.mkdir(parents=True, exist_ok=True)
857
+ lock_path = self._output_dir + "_builder.lock"
858
+
859
+ # File locking only with local paths; no file locking on GCS or S3
860
+ with FileLock(lock_path) if is_local else contextlib.nullcontext():
861
+ # Check if the data already exists
862
+ data_exists = self._fs.exists(posixpath.join(self._output_dir, config.DATASET_INFO_FILENAME))
863
+ if data_exists and download_mode == DownloadMode.REUSE_DATASET_IF_EXISTS:
864
+ logger.info(f"Found cached dataset {self.dataset_name} ({self._output_dir})")
865
+ # We need to update the info in case some splits were added in the meantime
866
+ # for example when calling load_dataset from multiple workers.
867
+ self.info = self._load_info()
868
+ self.download_post_processing_resources(dl_manager)
869
+ return
870
+
871
+ logger.info(f"Generating dataset {self.dataset_name} ({self._output_dir})")
872
+ if is_local: # if cache dir is local, check for available space
873
+ if not has_sufficient_disk_space(
874
+ self.info.size_in_bytes or 0, directory=Path(self._output_dir).parent
875
+ ):
876
+ raise OSError(
877
+ f"Not enough disk space. Needed: {size_str(self.info.size_in_bytes or 0)} (download: {size_str(self.info.download_size or 0)}, generated: {size_str(self.info.dataset_size or 0)}, post-processed: {size_str(self.info.post_processing_size or 0)})"
878
+ )
879
+
880
+ @contextlib.contextmanager
881
+ def incomplete_dir(dirname):
882
+ """Create temporary dir for dirname and rename on exit."""
883
+ if not is_local:
884
+ self._fs.makedirs(dirname, exist_ok=True)
885
+ yield dirname
886
+ else:
887
+ tmp_dir = dirname + ".incomplete"
888
+ os.makedirs(tmp_dir, exist_ok=True)
889
+ try:
890
+ yield tmp_dir
891
+ if os.path.isdir(dirname):
892
+ shutil.rmtree(dirname)
893
+ # LocalFileSystem.mv does copy + rm, it is more efficient to simply rename a local directory
894
+ shutil.move(tmp_dir, dirname)
895
+ finally:
896
+ if os.path.exists(tmp_dir):
897
+ shutil.rmtree(tmp_dir)
898
+
899
+ # Print is intentional: we want this to always go to stdout so user has
900
+ # information needed to cancel download/preparation if needed.
901
+ # This comes right before the progress bar.
902
+ if self.info.size_in_bytes:
903
+ logger.info(
904
+ f"Downloading and preparing dataset {self.dataset_name}/{self.config.name} "
905
+ f"(download: {size_str(self.info.download_size)}, generated: {size_str(self.info.dataset_size)}, "
906
+ f"post-processed: {size_str(self.info.post_processing_size)}, "
907
+ f"total: {size_str(self.info.size_in_bytes)}) to {self._output_dir}..."
908
+ )
909
+ else:
910
+ _dest = self._fs._strip_protocol(self._output_dir) if is_local else self._output_dir
911
+ logger.info(f"Downloading and preparing dataset {self.dataset_name}/{self.config.name} to {_dest}...")
912
+
913
+ self._check_manual_download(dl_manager)
914
+
915
+ # Create a tmp dir and rename to self._output_dir on successful exit.
916
+ with incomplete_dir(self._output_dir) as tmp_output_dir:
917
+ # Temporarily assign _output_dir to tmp_data_dir to avoid having to forward
918
+ # it to every sub function.
919
+ with temporary_assignment(self, "_output_dir", tmp_output_dir):
920
+ prepare_split_kwargs = {"file_format": file_format}
921
+ if max_shard_size is not None:
922
+ prepare_split_kwargs["max_shard_size"] = max_shard_size
923
+ if num_proc is not None:
924
+ prepare_split_kwargs["num_proc"] = num_proc
925
+ self._download_and_prepare(
926
+ dl_manager=dl_manager,
927
+ verification_mode=verification_mode,
928
+ **prepare_split_kwargs,
929
+ **download_and_prepare_kwargs,
930
+ )
931
+ # Sync info
932
+ self.info.dataset_size = sum(split.num_bytes for split in self.info.splits.values())
933
+ self.info.download_checksums = dl_manager.get_recorded_sizes_checksums()
934
+ if self.info.download_size is not None:
935
+ self.info.size_in_bytes = self.info.dataset_size + self.info.download_size
936
+ # Save info
937
+ self._save_info()
938
+
939
+ # Download post processing resources
940
+ self.download_post_processing_resources(dl_manager)
941
+
942
+ logger.info(
943
+ f"Dataset {self.dataset_name} downloaded and prepared to {self._output_dir}. "
944
+ f"Subsequent calls will reuse this data."
945
+ )
946
+
947
+ def _check_manual_download(self, dl_manager):
948
+ if self.manual_download_instructions is not None and dl_manager.manual_dir is None:
949
+ raise ManualDownloadError(
950
+ textwrap.dedent(
951
+ f"""\
952
+ The dataset {self.dataset_name} with config {self.config.name} requires manual data.
953
+ Please follow the manual download instructions:
954
+ {self.manual_download_instructions}
955
+ Manual data can be loaded with:
956
+ datasets.load_dataset("{self.repo_id or self.dataset_name}", data_dir="<path/to/manual/data>")"""
957
+ )
958
+ )
959
+
960
+ def _download_and_prepare(self, dl_manager, verification_mode, **prepare_split_kwargs):
961
+ """Downloads and prepares dataset for reading.
962
+
963
+ This is the internal implementation to overwrite called when user calls
964
+ `download_and_prepare`. It should download all required data and generate
965
+ the pre-processed datasets files.
966
+
967
+ Args:
968
+ dl_manager ([`DownloadManager`]):
969
+ `DownloadManager` used to download and cache data.
970
+ verification_mode ([`VerificationMode`]):
971
+ if `ALL_CHECKS`, perform all the verifications including checksums.
972
+ if `BASIC_CHECKS`, do not perform checksums, only perform split tests.
973
+ if `NO_CHECKS`, do not perform any verification.
974
+ prepare_split_kwargs: Additional options, such as `file_format`, `max_shard_size`
975
+ """
976
+ # Generating data for all splits
977
+ split_dict = SplitDict(dataset_name=self.dataset_name)
978
+ split_generators_kwargs = self._make_split_generators_kwargs(prepare_split_kwargs)
979
+ split_generators = self._split_generators(dl_manager, **split_generators_kwargs)
980
+
981
+ # Checksums verification
982
+ if verification_mode == VerificationMode.ALL_CHECKS and dl_manager.record_checksums:
983
+ verify_checksums(
984
+ self.info.download_checksums, dl_manager.get_recorded_sizes_checksums(), "dataset source files"
985
+ )
986
+
987
+ # Build splits
988
+ for split_generator in split_generators:
989
+ if str(split_generator.split_info.name).lower() == "all":
990
+ raise ValueError(
991
+ "`all` is a special split keyword corresponding to the "
992
+ "union of all splits, so cannot be used as key in "
993
+ "._split_generator()."
994
+ )
995
+
996
+ logger.info(f"Generating {split_generator.split_info.name} split")
997
+ split_dict.add(split_generator.split_info)
998
+
999
+ try:
1000
+ # Prepare split will record examples associated to the split
1001
+ self._prepare_split(split_generator, **prepare_split_kwargs)
1002
+ except OSError as e:
1003
+ raise OSError(
1004
+ "Cannot find data file. "
1005
+ + (self.manual_download_instructions or "")
1006
+ + "\nOriginal error:\n"
1007
+ + str(e)
1008
+ ) from None
1009
+ # If check_duplicates is set to True , then except DuplicatedKeysError
1010
+ except DuplicatedKeysError as e:
1011
+ raise DuplicatedKeysError(
1012
+ e.key,
1013
+ e.duplicate_key_indices,
1014
+ fix_msg=f"To avoid duplicate keys, please fix the dataset script {self.name}.py",
1015
+ ) from None
1016
+ dl_manager.manage_extracted_files()
1017
+
1018
+ if verification_mode == VerificationMode.BASIC_CHECKS or verification_mode == VerificationMode.ALL_CHECKS:
1019
+ verify_splits(self.info.splits, split_dict)
1020
+
1021
+ # Update the info object with the splits.
1022
+ self.info.splits = split_dict
1023
+ self.info.download_size = dl_manager.downloaded_size
1024
+
1025
+ def download_post_processing_resources(self, dl_manager):
1026
+ for split in self.info.splits or []:
1027
+ for resource_name, resource_file_name in self._post_processing_resources(split).items():
1028
+ if not not is_remote_filesystem(self._fs):
1029
+ raise NotImplementedError(f"Post processing is not supported on filesystem {self._fs}")
1030
+ if os.sep in resource_file_name:
1031
+ raise ValueError(f"Resources shouldn't be in a sub-directory: {resource_file_name}")
1032
+ resource_path = os.path.join(self._output_dir, resource_file_name)
1033
+ if not os.path.exists(resource_path):
1034
+ downloaded_resource_path = self._download_post_processing_resources(
1035
+ split, resource_name, dl_manager
1036
+ )
1037
+ if downloaded_resource_path:
1038
+ logger.info(f"Downloaded post-processing resource {resource_name} as {resource_file_name}")
1039
+ shutil.move(downloaded_resource_path, resource_path)
1040
+
1041
+ def _load_info(self) -> DatasetInfo:
1042
+ return DatasetInfo.from_directory(self._output_dir, storage_options=self._fs.storage_options)
1043
+
1044
+ def _save_info(self):
1045
+ file_lock = (
1046
+ FileLock(self._output_dir + "_info.lock")
1047
+ if not is_remote_filesystem(self._fs)
1048
+ else contextlib.nullcontext()
1049
+ )
1050
+ with file_lock:
1051
+ self.info.write_to_directory(self._output_dir, storage_options=self._fs.storage_options)
1052
+
1053
+ def _save_infos(self):
1054
+ file_lock = (
1055
+ FileLock(self._output_dir + "_infos.lock")
1056
+ if not is_remote_filesystem(self._fs)
1057
+ else contextlib.nullcontext()
1058
+ )
1059
+ with file_lock:
1060
+ DatasetInfosDict(**{self.config.name: self.info}).write_to_directory(self.get_imported_module_dir())
1061
+
1062
+ def _make_split_generators_kwargs(self, prepare_split_kwargs):
1063
+ """Get kwargs for `self._split_generators()` from `prepare_split_kwargs`."""
1064
+ del prepare_split_kwargs
1065
+ return {}
1066
+
1067
+ def as_dataset(
1068
+ self,
1069
+ split: Optional[Split] = None,
1070
+ run_post_process=True,
1071
+ verification_mode: Optional[Union[VerificationMode, str]] = None,
1072
+ in_memory=False,
1073
+ ) -> Union[Dataset, DatasetDict]:
1074
+ """Return a Dataset for the specified split.
1075
+
1076
+ Args:
1077
+ split (`datasets.Split`):
1078
+ Which subset of the data to return.
1079
+ run_post_process (`bool`, defaults to `True`):
1080
+ Whether to run post-processing dataset transforms and/or add
1081
+ indexes.
1082
+ verification_mode ([`VerificationMode`] or `str`, defaults to `BASIC_CHECKS`):
1083
+ Verification mode determining the checks to run on the
1084
+ downloaded/processed dataset information (checksums/size/splits/...).
1085
+
1086
+ <Added version="2.9.1"/>
1087
+ in_memory (`bool`, defaults to `False`):
1088
+ Whether to copy the data in-memory.
1089
+
1090
+ Returns:
1091
+ datasets.Dataset
1092
+
1093
+ Example:
1094
+
1095
+ ```py
1096
+ >>> from datasets import load_dataset_builder
1097
+ >>> builder = load_dataset_builder('cornell-movie-review-data/rotten_tomatoes')
1098
+ >>> builder.download_and_prepare()
1099
+ >>> ds = builder.as_dataset(split='train')
1100
+ >>> ds
1101
+ Dataset({
1102
+ features: ['text', 'label'],
1103
+ num_rows: 8530
1104
+ })
1105
+ ```
1106
+ """
1107
+ if self._file_format is not None and self._file_format != "arrow":
1108
+ raise FileFormatError('Loading a dataset not written in the "arrow" format is not supported.')
1109
+ if is_remote_filesystem(self._fs):
1110
+ raise NotImplementedError(f"Loading a dataset cached in a {type(self._fs).__name__} is not supported.")
1111
+ if not os.path.exists(self._output_dir):
1112
+ raise FileNotFoundError(
1113
+ f"Dataset {self.dataset_name}: could not find data in {self._output_dir}. Please make sure to call "
1114
+ "builder.download_and_prepare(), or use "
1115
+ "datasets.load_dataset() before trying to access the Dataset object."
1116
+ )
1117
+
1118
+ logger.debug(f"Constructing Dataset for split {split or ', '.join(self.info.splits)}, from {self._output_dir}")
1119
+
1120
+ # By default, return all splits
1121
+ if split is None:
1122
+ split = {s: s for s in self.info.splits}
1123
+
1124
+ verification_mode = VerificationMode(verification_mode or VerificationMode.BASIC_CHECKS)
1125
+
1126
+ # Create a dataset for each of the given splits
1127
+ datasets = map_nested(
1128
+ partial(
1129
+ self._build_single_dataset,
1130
+ run_post_process=run_post_process,
1131
+ verification_mode=verification_mode,
1132
+ in_memory=in_memory,
1133
+ ),
1134
+ split,
1135
+ map_tuple=True,
1136
+ disable_tqdm=True,
1137
+ )
1138
+ if isinstance(datasets, dict):
1139
+ datasets = DatasetDict(datasets)
1140
+ return datasets
1141
+
1142
+ def _build_single_dataset(
1143
+ self,
1144
+ split: Union[str, ReadInstruction, Split],
1145
+ run_post_process: bool,
1146
+ verification_mode: VerificationMode,
1147
+ in_memory: bool = False,
1148
+ ):
1149
+ """as_dataset for a single split."""
1150
+ if not isinstance(split, ReadInstruction):
1151
+ split = str(split)
1152
+ if split == "all":
1153
+ split = "+".join(self.info.splits.keys())
1154
+ split = Split(split)
1155
+
1156
+ # Build base dataset
1157
+ ds = self._as_dataset(
1158
+ split=split,
1159
+ in_memory=in_memory,
1160
+ )
1161
+ if run_post_process:
1162
+ for resource_file_name in self._post_processing_resources(split).values():
1163
+ if os.sep in resource_file_name:
1164
+ raise ValueError(f"Resources shouldn't be in a sub-directory: {resource_file_name}")
1165
+ resources_paths = {
1166
+ resource_name: os.path.join(self._output_dir, resource_file_name)
1167
+ for resource_name, resource_file_name in self._post_processing_resources(split).items()
1168
+ }
1169
+ post_processed = self._post_process(ds, resources_paths)
1170
+ if post_processed is not None:
1171
+ ds = post_processed
1172
+ recorded_checksums = {}
1173
+ record_checksums = False
1174
+ for resource_name, resource_path in resources_paths.items():
1175
+ size_checksum = get_size_checksum_dict(resource_path)
1176
+ recorded_checksums[resource_name] = size_checksum
1177
+ if verification_mode == VerificationMode.ALL_CHECKS and record_checksums:
1178
+ if self.info.post_processed is None or self.info.post_processed.resources_checksums is None:
1179
+ expected_checksums = None
1180
+ else:
1181
+ expected_checksums = self.info.post_processed.resources_checksums.get(split)
1182
+ verify_checksums(expected_checksums, recorded_checksums, "post processing resources")
1183
+ if self.info.post_processed is None:
1184
+ self.info.post_processed = PostProcessedInfo()
1185
+ if self.info.post_processed.resources_checksums is None:
1186
+ self.info.post_processed.resources_checksums = {}
1187
+ self.info.post_processed.resources_checksums[str(split)] = recorded_checksums
1188
+ self.info.post_processing_size = sum(
1189
+ checksums_dict["num_bytes"]
1190
+ for split_checksums_dicts in self.info.post_processed.resources_checksums.values()
1191
+ for checksums_dict in split_checksums_dicts.values()
1192
+ )
1193
+ if self.info.dataset_size is not None and self.info.download_size is not None:
1194
+ self.info.size_in_bytes = (
1195
+ self.info.dataset_size + self.info.download_size + self.info.post_processing_size
1196
+ )
1197
+ self._save_info()
1198
+ ds._info.post_processed = self.info.post_processed
1199
+ ds._info.post_processing_size = self.info.post_processing_size
1200
+ ds._info.size_in_bytes = self.info.size_in_bytes
1201
+ if self.info.post_processed.features is not None:
1202
+ if self.info.post_processed.features.type != ds.features.type:
1203
+ raise ValueError(
1204
+ f"Post-processed features info don't match the dataset:\nGot\n{self.info.post_processed.features}\nbut expected something like\n{ds.features}"
1205
+ )
1206
+ else:
1207
+ ds.info.features = self.info.post_processed.features
1208
+
1209
+ return ds
1210
+
1211
+ def _as_dataset(self, split: Union[ReadInstruction, Split] = Split.TRAIN, in_memory: bool = False) -> Dataset:
1212
+ """Constructs a `Dataset`.
1213
+
1214
+ This is the internal implementation to overwrite called when user calls
1215
+ `as_dataset`. It should read the pre-processed datasets files and generate
1216
+ the `Dataset` object.
1217
+
1218
+ Args:
1219
+ split (`datasets.Split`):
1220
+ which subset of the data to read.
1221
+ in_memory (`bool`, defaults to `False`):
1222
+ Whether to copy the data in-memory.
1223
+
1224
+ Returns:
1225
+ `Dataset`
1226
+ """
1227
+ cache_dir = self._fs._strip_protocol(self._output_dir)
1228
+ dataset_name = self.dataset_name
1229
+ if self._check_legacy_cache():
1230
+ dataset_name = self.name
1231
+ dataset_kwargs = ArrowReader(cache_dir, self.info).read(
1232
+ name=dataset_name,
1233
+ instructions=split,
1234
+ split_infos=self.info.splits.values(),
1235
+ in_memory=in_memory,
1236
+ )
1237
+ fingerprint = self._get_dataset_fingerprint(split)
1238
+ return Dataset(fingerprint=fingerprint, **dataset_kwargs)
1239
+
1240
+ def _get_dataset_fingerprint(self, split: Union[ReadInstruction, Split]) -> str:
1241
+ """The dataset fingerprint is the hash of the relative directory dataset_name/config_name/version/hash, as well as the split specs."""
1242
+ hasher = Hasher()
1243
+ hasher.update(Path(self._relative_data_dir()).as_posix())
1244
+ hasher.update(str(split)) # for example: train, train+test, train[:10%], test[:33%](pct1_dropremainder)
1245
+ fingerprint = hasher.hexdigest()
1246
+ return fingerprint
1247
+
1248
+ def as_streaming_dataset(
1249
+ self,
1250
+ split: Optional[str] = None,
1251
+ base_path: Optional[str] = None,
1252
+ ) -> Union[dict[str, IterableDataset], IterableDataset]:
1253
+ if is_remote_filesystem(self._fs):
1254
+ raise NotImplementedError(
1255
+ f"Loading a streaming dataset cached in a {type(self._fs).__name__} is not supported yet."
1256
+ )
1257
+
1258
+ dl_manager = StreamingDownloadManager(
1259
+ base_path=base_path or self.base_path,
1260
+ download_config=DownloadConfig(token=self.token, storage_options=self.storage_options),
1261
+ dataset_name=self.dataset_name,
1262
+ data_dir=self.config.data_dir,
1263
+ )
1264
+ self._check_manual_download(dl_manager)
1265
+ splits_generators = {sg.name: sg for sg in self._split_generators(dl_manager)}
1266
+ # By default, return all splits
1267
+ if split is None:
1268
+ splits_generator = splits_generators
1269
+ elif split in splits_generators:
1270
+ splits_generator = splits_generators[split]
1271
+ else:
1272
+ raise ValueError(f"Bad split: {split}. Available splits: {list(splits_generators)}")
1273
+
1274
+ # Create a dataset for each of the given splits
1275
+ datasets = map_nested(
1276
+ self._as_streaming_dataset_single,
1277
+ splits_generator,
1278
+ map_tuple=True,
1279
+ )
1280
+ if isinstance(datasets, dict):
1281
+ datasets = IterableDatasetDict(datasets)
1282
+ return datasets
1283
+
1284
+ def _as_streaming_dataset_single(
1285
+ self,
1286
+ splits_generator,
1287
+ ) -> IterableDataset:
1288
+ ex_iterable = self._get_examples_iterable_for_split(splits_generator)
1289
+ # add auth to be able to access and decode audio/image files from private repositories.
1290
+ token_per_repo_id = {self.repo_id: self.token} if self.repo_id else {}
1291
+ return IterableDataset(
1292
+ ex_iterable, info=self.info, split=splits_generator.name, token_per_repo_id=token_per_repo_id
1293
+ )
1294
+
1295
+ def _post_process(self, dataset: Dataset, resources_paths: Mapping[str, str]) -> Optional[Dataset]:
1296
+ """Run dataset transforms or add indexes"""
1297
+ return None
1298
+
1299
+ def _post_processing_resources(self, split: str) -> dict[str, str]:
1300
+ """Mapping resource_name -> resource_file_name"""
1301
+ return {}
1302
+
1303
+ def _download_post_processing_resources(
1304
+ self, split: str, resource_name: str, dl_manager: DownloadManager
1305
+ ) -> Optional[str]:
1306
+ """Download the resource using the download manager and return the downloaded path."""
1307
+ return None
1308
+
1309
+ @abc.abstractmethod
1310
+ def _split_generators(self, dl_manager: Union[DownloadManager, StreamingDownloadManager]):
1311
+ """Specify feature dictionary generators and dataset splits.
1312
+
1313
+ This function returns a list of `SplitGenerator`s defining how to generate
1314
+ data and what splits to use.
1315
+
1316
+ Example:
1317
+
1318
+ return [
1319
+ datasets.SplitGenerator(
1320
+ name=datasets.Split.TRAIN,
1321
+ gen_kwargs={'file': 'train_data.zip'},
1322
+ ),
1323
+ datasets.SplitGenerator(
1324
+ name=datasets.Split.TEST,
1325
+ gen_kwargs={'file': 'test_data.zip'},
1326
+ ),
1327
+ ]
1328
+
1329
+ The above code will first call `_generate_examples(file='train_data.zip')`
1330
+ to write the train data, then `_generate_examples(file='test_data.zip')` to
1331
+ write the test data.
1332
+
1333
+ Datasets are typically split into different subsets to be used at various
1334
+ stages of training and evaluation.
1335
+
1336
+ Note that for datasets without a `VALIDATION` split, you can use a
1337
+ fraction of the `TRAIN` data for evaluation as you iterate on your model
1338
+ so as not to overfit to the `TEST` data.
1339
+
1340
+ For downloads and extractions, use the given `download_manager`.
1341
+ Note that the `DownloadManager` caches downloads, so it is fine to have each
1342
+ generator attempt to download the source data.
1343
+
1344
+ A good practice is to download all data in this function, and then
1345
+ distribute the relevant parts to each split with the `gen_kwargs` argument
1346
+
1347
+ Args:
1348
+ dl_manager (`Union[DownloadManager, StreamingDownloadManager]`):
1349
+ Download manager to download the data
1350
+
1351
+ Returns:
1352
+ `list<SplitGenerator>`.
1353
+ """
1354
+ raise NotImplementedError()
1355
+
1356
+ @abc.abstractmethod
1357
+ def _prepare_split(
1358
+ self,
1359
+ split_generator: SplitGenerator,
1360
+ file_format: str = "arrow",
1361
+ max_shard_size: Optional[Union[str, int]] = None,
1362
+ num_proc: Optional[int] = None,
1363
+ **kwargs,
1364
+ ):
1365
+ """Generate the examples and record them on disk.
1366
+
1367
+ Args:
1368
+ split_generator (`SplitGenerator`):
1369
+ Split generator to process
1370
+ file_format (`str`, *optional*):
1371
+ format of the data files in which the dataset will be written.
1372
+ Supported formats: "arrow", "parquet". Default to "arrow" format.
1373
+ max_shard_size (`Union[str, int]`, *optional*):
1374
+ Maximum number of bytes written per shard, default is "500MB".
1375
+ The size is based on uncompressed data size, so in practice your shard files may be smaller than
1376
+ `max_shard_size` thanks to Parquet compression for example.
1377
+ num_proc (`int`, *optional*, defaults to `None`):
1378
+ Number of processes when downloading and generating the dataset locally.
1379
+ Multiprocessing is disabled by default.
1380
+
1381
+ <Added version="2.7.0"/>
1382
+ **kwargs: Additional kwargs forwarded from _download_and_prepare
1383
+ """
1384
+ raise NotImplementedError()
1385
+
1386
+ def _get_examples_iterable_for_split(self, split_generator: SplitGenerator) -> ExamplesIterable:
1387
+ """Generate the examples on the fly.
1388
+
1389
+ Args:
1390
+ split_generator (`SplitGenerator`):
1391
+ Split generator to process
1392
+ """
1393
+ raise NotImplementedError()
1394
+
1395
+
1396
+ class GeneratorBasedBuilder(DatasetBuilder):
1397
+ """Base class for datasets with data generation based on dict generators.
1398
+
1399
+ `GeneratorBasedBuilder` is a convenience class that abstracts away much
1400
+ of the data writing and reading of `DatasetBuilder`. It expects subclasses to
1401
+ implement generators of feature dictionaries across the dataset splits
1402
+ (`_split_generators`). See the method docstrings for details.
1403
+ """
1404
+
1405
+ @abc.abstractmethod
1406
+ def _generate_examples(self, **kwargs):
1407
+ """Default function generating examples for each `SplitGenerator`.
1408
+
1409
+ This function preprocess the examples from the raw data to the preprocessed
1410
+ dataset files.
1411
+ This function is called once for each `SplitGenerator` defined in
1412
+ `_split_generators`. The examples yielded here will be written on
1413
+ disk.
1414
+
1415
+ Args:
1416
+ **kwargs (additional keyword arguments):
1417
+ Arguments forwarded from the SplitGenerator.gen_kwargs
1418
+
1419
+ Yields:
1420
+ key: `str` or `int`, a unique deterministic example identification key.
1421
+ * Unique: An error will be raised if two examples are yield with the
1422
+ same key.
1423
+ * Deterministic: When generating the dataset twice, the same example
1424
+ should have the same key.
1425
+ Good keys can be the image id, or line number if examples are extracted
1426
+ from a text file.
1427
+ The key will be hashed and sorted to shuffle examples deterministically,
1428
+ such as generating the dataset multiple times keep examples in the
1429
+ same order.
1430
+ example: `dict<str feature_name, feature_value>`, a feature dictionary
1431
+ ready to be encoded and written to disk. The example will be
1432
+ encoded with `self.info.features.encode_example({...})`.
1433
+ """
1434
+ raise NotImplementedError()
1435
+
1436
+ def _prepare_split(
1437
+ self,
1438
+ split_generator: SplitGenerator,
1439
+ check_duplicate_keys: bool,
1440
+ file_format="arrow",
1441
+ num_proc: Optional[int] = None,
1442
+ max_shard_size: Optional[Union[int, str]] = None,
1443
+ ):
1444
+ max_shard_size = convert_file_size_to_int(max_shard_size or config.MAX_SHARD_SIZE)
1445
+
1446
+ if self.info.splits is not None:
1447
+ split_info = self.info.splits[split_generator.name]
1448
+ else:
1449
+ split_info = split_generator.split_info
1450
+
1451
+ SUFFIX = "-JJJJJ-SSSSS-of-NNNNN"
1452
+ fname = f"{self.dataset_name}-{split_generator.name}{SUFFIX}.{file_format}"
1453
+ fpath = posixpath.join(self._output_dir, fname)
1454
+
1455
+ if num_proc and num_proc > 1:
1456
+ num_input_shards = _number_of_shards_in_gen_kwargs(split_generator.gen_kwargs)
1457
+ if num_input_shards <= 1:
1458
+ logger.warning(
1459
+ f"Setting num_proc from {num_proc} back to 1 for the {split_info.name} split to disable multiprocessing as it only contains one shard."
1460
+ )
1461
+ num_proc = 1
1462
+ elif num_input_shards < num_proc:
1463
+ logger.warning(
1464
+ f"Setting num_proc from {num_proc} to {num_input_shards} for the {split_info.name} split as it only contains {num_input_shards} shards."
1465
+ )
1466
+ num_proc = num_input_shards
1467
+
1468
+ pbar = hf_tqdm(
1469
+ unit=" examples",
1470
+ total=split_info.num_examples,
1471
+ desc=f"Generating {split_info.name} split",
1472
+ )
1473
+
1474
+ _prepare_split_args = {
1475
+ "fpath": fpath,
1476
+ "file_format": file_format,
1477
+ "max_shard_size": max_shard_size,
1478
+ "split_info": split_info,
1479
+ "check_duplicate_keys": check_duplicate_keys,
1480
+ }
1481
+
1482
+ if num_proc is None or num_proc == 1:
1483
+ result = None
1484
+ gen_kwargs = split_generator.gen_kwargs
1485
+ job_id = 0
1486
+ with pbar:
1487
+ for job_id, done, content in self._prepare_split_single(
1488
+ gen_kwargs=gen_kwargs, job_id=job_id, **_prepare_split_args
1489
+ ):
1490
+ if done:
1491
+ result = content
1492
+ else:
1493
+ pbar.update(content)
1494
+ # wrapping everything into lists for consistency with the multiprocessed code path
1495
+ assert result is not None, "Failed to retrieve results from prepare_split"
1496
+ examples_per_job, bytes_per_job, features_per_job, shards_per_job, shard_lengths_per_job = (
1497
+ [item] for item in result
1498
+ )
1499
+ else:
1500
+ kwargs_per_job = [
1501
+ {"gen_kwargs": gen_kwargs, "job_id": job_id, **_prepare_split_args}
1502
+ for job_id, gen_kwargs in enumerate(
1503
+ _split_gen_kwargs(split_generator.gen_kwargs, max_num_jobs=num_proc)
1504
+ )
1505
+ ]
1506
+ num_jobs = len(kwargs_per_job)
1507
+
1508
+ examples_per_job = [None] * num_jobs
1509
+ bytes_per_job = [None] * num_jobs
1510
+ features_per_job = [None] * num_jobs
1511
+ shards_per_job = [None] * num_jobs
1512
+ shard_lengths_per_job = [None] * num_jobs
1513
+
1514
+ with Pool(num_proc) as pool:
1515
+ with pbar:
1516
+ for job_id, done, content in iflatmap_unordered(
1517
+ pool, self._prepare_split_single, kwargs_iterable=kwargs_per_job
1518
+ ):
1519
+ if done:
1520
+ # the content is the result of the job
1521
+ (
1522
+ examples_per_job[job_id],
1523
+ bytes_per_job[job_id],
1524
+ features_per_job[job_id],
1525
+ shards_per_job[job_id],
1526
+ shard_lengths_per_job[job_id],
1527
+ ) = content
1528
+ else:
1529
+ # the content is the number of examples progress update
1530
+ pbar.update(content)
1531
+
1532
+ assert None not in examples_per_job, (
1533
+ f"Failed to retrieve results from prepare_split: result list {examples_per_job} still contains None - at least one worker failed to return its results"
1534
+ )
1535
+
1536
+ total_shards = sum(shards_per_job)
1537
+ total_num_examples = sum(examples_per_job)
1538
+ total_num_bytes = sum(bytes_per_job)
1539
+ features = features_per_job[0]
1540
+
1541
+ split_generator.split_info.num_examples = total_num_examples
1542
+ split_generator.split_info.num_bytes = total_num_bytes
1543
+
1544
+ # should rename everything at the end
1545
+ logger.debug(f"Renaming {total_shards} shards.")
1546
+ if total_shards > 1:
1547
+ # use the -SSSSS-of-NNNNN pattern
1548
+
1549
+ def _rename_shard(shard_and_job: tuple[int]):
1550
+ shard_id, job_id = shard_and_job
1551
+ global_shard_id = sum(shards_per_job[:job_id]) + shard_id
1552
+ self._rename(
1553
+ fpath.replace("SSSSS", f"{shard_id:05d}").replace("JJJJJ", f"{job_id:05d}"),
1554
+ fpath.replace("JJJJJ-SSSSS", f"{global_shard_id:05d}").replace("NNNNN", f"{total_shards:05d}"),
1555
+ )
1556
+
1557
+ shards_and_jobs = [
1558
+ (shard_id, job_id)
1559
+ for job_id, num_shards in enumerate(shards_per_job)
1560
+ for shard_id in range(num_shards)
1561
+ ]
1562
+ thread_map(_rename_shard, shards_and_jobs, disable=True, max_workers=64)
1563
+
1564
+ split_generator.split_info.shard_lengths = [
1565
+ shard_length for shard_lengths in shard_lengths_per_job for shard_length in shard_lengths
1566
+ ]
1567
+ else:
1568
+ # don't use any pattern
1569
+ shard_id, job_id = 0, 0
1570
+ self._rename(
1571
+ fpath.replace("SSSSS", f"{shard_id:05d}").replace("JJJJJ", f"{job_id:05d}"),
1572
+ fpath.replace(SUFFIX, ""),
1573
+ )
1574
+
1575
+ if self.info.features is None:
1576
+ self.info.features = features
1577
+
1578
+ def _prepare_split_single(
1579
+ self,
1580
+ gen_kwargs: dict,
1581
+ fpath: str,
1582
+ file_format: str,
1583
+ max_shard_size: int,
1584
+ split_info: SplitInfo,
1585
+ check_duplicate_keys: bool,
1586
+ job_id: int,
1587
+ ) -> Iterable[tuple[int, bool, Union[int, tuple]]]:
1588
+ generator = self._generate_examples(**gen_kwargs)
1589
+ writer_class = ParquetWriter if file_format == "parquet" else ArrowWriter
1590
+ embed_local_files = file_format == "parquet"
1591
+ shard_lengths = []
1592
+ total_num_examples, total_num_bytes = 0, 0
1593
+
1594
+ shard_id = 0
1595
+ num_examples_progress_update = 0
1596
+ try:
1597
+ writer = writer_class(
1598
+ features=self.info.features,
1599
+ path=fpath.replace("SSSSS", f"{shard_id:05d}").replace("JJJJJ", f"{job_id:05d}"),
1600
+ writer_batch_size=self._writer_batch_size,
1601
+ hash_salt=split_info.name,
1602
+ check_duplicates=check_duplicate_keys,
1603
+ storage_options=self._fs.storage_options,
1604
+ embed_local_files=embed_local_files,
1605
+ )
1606
+ try:
1607
+ _time = time.time()
1608
+ for key, record in generator:
1609
+ if max_shard_size is not None and writer._num_bytes > max_shard_size:
1610
+ num_examples, num_bytes = writer.finalize()
1611
+ writer.close()
1612
+ shard_lengths.append(num_examples)
1613
+ total_num_examples += num_examples
1614
+ total_num_bytes += num_bytes
1615
+ shard_id += 1
1616
+ writer = writer_class(
1617
+ features=writer._features,
1618
+ path=fpath.replace("SSSSS", f"{shard_id:05d}").replace("JJJJJ", f"{job_id:05d}"),
1619
+ writer_batch_size=self._writer_batch_size,
1620
+ hash_salt=split_info.name,
1621
+ check_duplicates=check_duplicate_keys,
1622
+ storage_options=self._fs.storage_options,
1623
+ embed_local_files=embed_local_files,
1624
+ )
1625
+ example = self.info.features.encode_example(record) if self.info.features is not None else record
1626
+ writer.write(example, key)
1627
+ num_examples_progress_update += 1
1628
+ if time.time() > _time + config.PBAR_REFRESH_TIME_INTERVAL:
1629
+ _time = time.time()
1630
+ yield job_id, False, num_examples_progress_update
1631
+ num_examples_progress_update = 0
1632
+ finally:
1633
+ yield job_id, False, num_examples_progress_update
1634
+ num_shards = shard_id + 1
1635
+ num_examples, num_bytes = writer.finalize()
1636
+ writer.close()
1637
+ shard_lengths.append(num_examples)
1638
+ total_num_examples += num_examples
1639
+ total_num_bytes += num_bytes
1640
+ except Exception as e:
1641
+ # Ignore the writer's error for no examples written to the file if this error was caused by the error in _generate_examples before the first example was yielded
1642
+ if isinstance(e, SchemaInferenceError) and e.__context__ is not None:
1643
+ e = e.__context__
1644
+ raise DatasetGenerationError("An error occurred while generating the dataset") from e
1645
+
1646
+ yield job_id, True, (total_num_examples, total_num_bytes, writer._features, num_shards, shard_lengths)
1647
+
1648
+ def _download_and_prepare(self, dl_manager, verification_mode, **prepare_splits_kwargs):
1649
+ super()._download_and_prepare(
1650
+ dl_manager,
1651
+ verification_mode,
1652
+ check_duplicate_keys=verification_mode == VerificationMode.BASIC_CHECKS
1653
+ or verification_mode == VerificationMode.ALL_CHECKS,
1654
+ **prepare_splits_kwargs,
1655
+ )
1656
+
1657
+ def _get_examples_iterable_for_split(self, split_generator: SplitGenerator) -> ExamplesIterable:
1658
+ return ExamplesIterable(self._generate_examples, split_generator.gen_kwargs)
1659
+
1660
+
1661
+ class ArrowBasedBuilder(DatasetBuilder):
1662
+ """Base class for datasets with data generation based on Arrow loading functions (CSV/JSON/Parquet)."""
1663
+
1664
+ @abc.abstractmethod
1665
+ def _generate_tables(self, **kwargs):
1666
+ """Default function generating examples for each `SplitGenerator`.
1667
+
1668
+ This function preprocess the examples from the raw data to the preprocessed
1669
+ dataset files.
1670
+ This function is called once for each `SplitGenerator` defined in
1671
+ `_split_generators`. The examples yielded here will be written on
1672
+ disk.
1673
+
1674
+ Args:
1675
+ **kwargs (additional keyword arguments):
1676
+ Arguments forwarded from the SplitGenerator.gen_kwargs
1677
+
1678
+ Yields:
1679
+ key: `str` or `int`, a unique deterministic example identification key.
1680
+ * Unique: An error will be raised if two examples are yield with the
1681
+ same key.
1682
+ * Deterministic: When generating the dataset twice, the same example
1683
+ should have the same key.
1684
+ Good keys can be the image id, or line number if examples are extracted
1685
+ from a text file.
1686
+ The key will be hashed and sorted to shuffle examples deterministically,
1687
+ such as generating the dataset multiple times keep examples in the
1688
+ same order.
1689
+ example: `pyarrow.Table`, a feature table
1690
+ ready to be encoded and written to disk.
1691
+ """
1692
+ raise NotImplementedError()
1693
+
1694
+ def _prepare_split(
1695
+ self,
1696
+ split_generator: SplitGenerator,
1697
+ file_format: str = "arrow",
1698
+ num_proc: Optional[int] = None,
1699
+ max_shard_size: Optional[Union[str, int]] = None,
1700
+ ):
1701
+ max_shard_size = convert_file_size_to_int(max_shard_size or config.MAX_SHARD_SIZE)
1702
+
1703
+ try:
1704
+ split_info = self.info.splits[split_generator.name]
1705
+ except Exception:
1706
+ split_info = split_generator.split_info
1707
+
1708
+ SUFFIX = "-JJJJJ-SSSSS-of-NNNNN"
1709
+ fname = f"{self.dataset_name}-{split_generator.name}{SUFFIX}.{file_format}"
1710
+ fpath = posixpath.join(self._output_dir, fname)
1711
+
1712
+ if num_proc and num_proc > 1:
1713
+ num_input_shards = _number_of_shards_in_gen_kwargs(split_generator.gen_kwargs)
1714
+ if num_input_shards <= 1:
1715
+ logger.warning(
1716
+ f"Setting num_proc from {num_proc} back to 1 for the {split_info.name} split to disable multiprocessing as it only contains one shard."
1717
+ )
1718
+ num_proc = 1
1719
+ elif num_input_shards < num_proc:
1720
+ logger.warning(
1721
+ f"Setting num_proc from {num_proc} to {num_input_shards} for the {split_info.name} split as it only contains {num_input_shards} shards."
1722
+ )
1723
+ num_proc = num_input_shards
1724
+
1725
+ pbar = hf_tqdm(
1726
+ unit=" examples",
1727
+ total=split_info.num_examples,
1728
+ desc=f"Generating {split_info.name} split",
1729
+ )
1730
+
1731
+ _prepare_split_args = {
1732
+ "fpath": fpath,
1733
+ "file_format": file_format,
1734
+ "max_shard_size": max_shard_size,
1735
+ }
1736
+
1737
+ if num_proc is None or num_proc == 1:
1738
+ result = None
1739
+ gen_kwargs = split_generator.gen_kwargs
1740
+ job_id = 0
1741
+ with pbar:
1742
+ for job_id, done, content in self._prepare_split_single(
1743
+ gen_kwargs=gen_kwargs, job_id=job_id, **_prepare_split_args
1744
+ ):
1745
+ if done:
1746
+ result = content
1747
+ else:
1748
+ pbar.update(content)
1749
+ # wrapping everything into lists for consistency with the multiprocessed code path
1750
+ assert result is not None, "Failed to retrieve results from prepare_split"
1751
+ examples_per_job, bytes_per_job, features_per_job, shards_per_job, shard_lengths_per_job = (
1752
+ [item] for item in result
1753
+ )
1754
+ else:
1755
+ kwargs_per_job = [
1756
+ {"gen_kwargs": gen_kwargs, "job_id": job_id, **_prepare_split_args}
1757
+ for job_id, gen_kwargs in enumerate(
1758
+ _split_gen_kwargs(split_generator.gen_kwargs, max_num_jobs=num_proc)
1759
+ )
1760
+ ]
1761
+ num_jobs = len(kwargs_per_job)
1762
+
1763
+ examples_per_job = [None] * num_jobs
1764
+ bytes_per_job = [None] * num_jobs
1765
+ features_per_job = [None] * num_jobs
1766
+ shards_per_job = [None] * num_jobs
1767
+ shard_lengths_per_job = [None] * num_jobs
1768
+
1769
+ with Pool(num_proc) as pool:
1770
+ with pbar:
1771
+ for job_id, done, content in iflatmap_unordered(
1772
+ pool, self._prepare_split_single, kwargs_iterable=kwargs_per_job
1773
+ ):
1774
+ if done:
1775
+ # the content is the result of the job
1776
+ (
1777
+ examples_per_job[job_id],
1778
+ bytes_per_job[job_id],
1779
+ features_per_job[job_id],
1780
+ shards_per_job[job_id],
1781
+ shard_lengths_per_job[job_id],
1782
+ ) = content
1783
+ else:
1784
+ # the content is the number of examples progress update
1785
+ pbar.update(content)
1786
+
1787
+ assert None not in examples_per_job, (
1788
+ f"Failed to retrieve results from prepare_split: result list {examples_per_job} still contains None - at least one worker failed to return its results"
1789
+ )
1790
+
1791
+ total_shards = sum(shards_per_job)
1792
+ total_num_examples = sum(examples_per_job)
1793
+ total_num_bytes = sum(bytes_per_job)
1794
+ features = features_per_job[0]
1795
+
1796
+ split_generator.split_info.num_examples = total_num_examples
1797
+ split_generator.split_info.num_bytes = total_num_bytes
1798
+
1799
+ # should rename everything at the end
1800
+ logger.debug(f"Renaming {total_shards} shards.")
1801
+ if total_shards > 1:
1802
+ # use the -SSSSS-of-NNNNN pattern
1803
+
1804
+ def _rename_shard(shard_id_and_job: tuple[int]):
1805
+ shard_id, job_id = shard_id_and_job
1806
+ global_shard_id = sum(shards_per_job[:job_id]) + shard_id
1807
+ self._rename(
1808
+ fpath.replace("SSSSS", f"{shard_id:05d}").replace("JJJJJ", f"{job_id:05d}"),
1809
+ fpath.replace("JJJJJ-SSSSS", f"{global_shard_id:05d}").replace("NNNNN", f"{total_shards:05d}"),
1810
+ )
1811
+
1812
+ shard_ids_and_jobs = [
1813
+ (shard_id, job_id)
1814
+ for job_id, num_shards in enumerate(shards_per_job)
1815
+ for shard_id in range(num_shards)
1816
+ ]
1817
+ thread_map(_rename_shard, shard_ids_and_jobs, disable=True, max_workers=64)
1818
+
1819
+ split_generator.split_info.shard_lengths = [
1820
+ shard_length for shard_lengths in shard_lengths_per_job for shard_length in shard_lengths
1821
+ ]
1822
+ else:
1823
+ # don't use any pattern
1824
+ shard_id, job_id = 0, 0
1825
+ self._rename(
1826
+ fpath.replace("SSSSS", f"{shard_id:05d}").replace("JJJJJ", f"{job_id:05d}"),
1827
+ fpath.replace(SUFFIX, ""),
1828
+ )
1829
+
1830
+ if self.info.features is None:
1831
+ self.info.features = features
1832
+
1833
+ def _prepare_split_single(
1834
+ self, gen_kwargs: dict, fpath: str, file_format: str, max_shard_size: int, job_id: int
1835
+ ) -> Iterable[tuple[int, bool, Union[int, tuple]]]:
1836
+ gen_kwargs = {k: tracked_list(v) if isinstance(v, list) else v for k, v in gen_kwargs.items()}
1837
+ generator = self._generate_tables(**gen_kwargs)
1838
+ writer_class = ParquetWriter if file_format == "parquet" else ArrowWriter
1839
+ embed_local_files = file_format == "parquet"
1840
+ shard_lengths = []
1841
+ total_num_examples, total_num_bytes = 0, 0
1842
+
1843
+ shard_id = 0
1844
+ num_examples_progress_update = 0
1845
+ try:
1846
+ writer = writer_class(
1847
+ features=self.info.features,
1848
+ path=fpath.replace("SSSSS", f"{shard_id:05d}").replace("JJJJJ", f"{job_id:05d}"),
1849
+ writer_batch_size=self._writer_batch_size,
1850
+ storage_options=self._fs.storage_options,
1851
+ embed_local_files=embed_local_files,
1852
+ )
1853
+ try:
1854
+ _time = time.time()
1855
+ for _, table in generator:
1856
+ if max_shard_size is not None and writer._num_bytes > max_shard_size:
1857
+ num_examples, num_bytes = writer.finalize()
1858
+ writer.close()
1859
+ shard_lengths.append(num_examples)
1860
+ total_num_examples += num_examples
1861
+ total_num_bytes += num_bytes
1862
+ shard_id += 1
1863
+ writer = writer_class(
1864
+ features=writer._features,
1865
+ path=fpath.replace("SSSSS", f"{shard_id:05d}").replace("JJJJJ", f"{job_id:05d}"),
1866
+ writer_batch_size=self._writer_batch_size,
1867
+ storage_options=self._fs.storage_options,
1868
+ embed_local_files=embed_local_files,
1869
+ )
1870
+ try:
1871
+ writer.write_table(table)
1872
+ except CastError as cast_error:
1873
+ raise DatasetGenerationCastError.from_cast_error(
1874
+ cast_error=cast_error,
1875
+ builder_name=self.info.builder_name,
1876
+ gen_kwargs=gen_kwargs,
1877
+ token=self.token,
1878
+ )
1879
+ num_examples_progress_update += len(table)
1880
+ if time.time() > _time + config.PBAR_REFRESH_TIME_INTERVAL:
1881
+ _time = time.time()
1882
+ yield job_id, False, num_examples_progress_update
1883
+ num_examples_progress_update = 0
1884
+ finally:
1885
+ yield job_id, False, num_examples_progress_update
1886
+ num_shards = shard_id + 1
1887
+ num_examples, num_bytes = writer.finalize()
1888
+ writer.close()
1889
+ shard_lengths.append(num_examples)
1890
+ total_num_examples += num_examples
1891
+ total_num_bytes += num_bytes
1892
+ except Exception as e:
1893
+ # Ignore the writer's error for no examples written to the file if this error was caused by the error in _generate_examples before the first example was yielded
1894
+ if isinstance(e, SchemaInferenceError) and e.__context__ is not None:
1895
+ e = e.__context__
1896
+ if isinstance(e, DatasetGenerationError):
1897
+ raise
1898
+ raise DatasetGenerationError("An error occurred while generating the dataset") from e
1899
+
1900
+ yield job_id, True, (total_num_examples, total_num_bytes, writer._features, num_shards, shard_lengths)
1901
+
1902
+ def _get_examples_iterable_for_split(self, split_generator: SplitGenerator) -> ExamplesIterable:
1903
+ return ArrowExamplesIterable(self._generate_tables, kwargs=split_generator.gen_kwargs)
lib/python3.12/site-packages/datasets/combine.py ADDED
@@ -0,0 +1,215 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Optional, TypeVar
2
+
3
+ from .arrow_dataset import Dataset, _concatenate_map_style_datasets, _interleave_map_style_datasets
4
+ from .dataset_dict import DatasetDict, IterableDatasetDict
5
+ from .info import DatasetInfo
6
+ from .iterable_dataset import IterableDataset, _concatenate_iterable_datasets, _interleave_iterable_datasets
7
+ from .splits import NamedSplit
8
+ from .utils import logging
9
+ from .utils.py_utils import Literal
10
+
11
+
12
+ logger = logging.get_logger(__name__)
13
+
14
+
15
+ DatasetType = TypeVar("DatasetType", Dataset, IterableDataset)
16
+
17
+
18
+ def interleave_datasets(
19
+ datasets: list[DatasetType],
20
+ probabilities: Optional[list[float]] = None,
21
+ seed: Optional[int] = None,
22
+ info: Optional[DatasetInfo] = None,
23
+ split: Optional[NamedSplit] = None,
24
+ stopping_strategy: Literal["first_exhausted", "all_exhausted"] = "first_exhausted",
25
+ ) -> DatasetType:
26
+ """
27
+ Interleave several datasets (sources) into a single dataset.
28
+ The new dataset is constructed by alternating between the sources to get the examples.
29
+
30
+ You can use this function on a list of [`Dataset`] objects, or on a list of [`IterableDataset`] objects.
31
+
32
+ - If `probabilities` is `None` (default) the new dataset is constructed by cycling between each source to get the examples.
33
+ - If `probabilities` is not `None`, the new dataset is constructed by getting examples from a random source at a time according to the provided probabilities.
34
+
35
+ The resulting dataset ends when one of the source datasets runs out of examples except when `oversampling` is `True`,
36
+ in which case, the resulting dataset ends when all datasets have ran out of examples at least one time.
37
+
38
+ Note for iterable datasets:
39
+
40
+ In a distributed setup or in PyTorch DataLoader workers, the stopping strategy is applied per process.
41
+ Therefore the "first_exhausted" strategy on an sharded iterable dataset can generate less samples in total (up to 1 missing sample per subdataset per worker).
42
+
43
+ Args:
44
+ datasets (`List[Dataset]` or `List[IterableDataset]`):
45
+ List of datasets to interleave.
46
+ probabilities (`List[float]`, *optional*, defaults to `None`):
47
+ If specified, the new dataset is constructed by sampling
48
+ examples from one source at a time according to these probabilities.
49
+ seed (`int`, *optional*, defaults to `None`):
50
+ The random seed used to choose a source for each example.
51
+ info ([`DatasetInfo`], *optional*):
52
+ Dataset information, like description, citation, etc.
53
+ <Added version="2.4.0"/>
54
+ split ([`NamedSplit`], *optional*):
55
+ Name of the dataset split.
56
+ <Added version="2.4.0"/>
57
+ stopping_strategy (`str`, defaults to `first_exhausted`):
58
+ Two strategies are proposed right now, `first_exhausted` and `all_exhausted`.
59
+ By default, `first_exhausted` is an undersampling strategy, i.e the dataset construction is stopped as soon as one dataset has ran out of samples.
60
+ If the strategy is `all_exhausted`, we use an oversampling strategy, i.e the dataset construction is stopped as soon as every samples of every dataset has been added at least once.
61
+ Note that if the strategy is `all_exhausted`, the interleaved dataset size can get enormous:
62
+ - with no probabilities, the resulting dataset will have `max_length_datasets*nb_dataset` samples.
63
+ - with given probabilities, the resulting dataset will have more samples if some datasets have really low probability of visiting.
64
+ Returns:
65
+ [`Dataset`] or [`IterableDataset`]: Return type depends on the input `datasets`
66
+ parameter. `Dataset` if the input is a list of `Dataset`, `IterableDataset` if the input is a list of
67
+ `IterableDataset`.
68
+
69
+ Example:
70
+
71
+ For regular datasets (map-style):
72
+
73
+ ```python
74
+ >>> from datasets import Dataset, interleave_datasets
75
+ >>> d1 = Dataset.from_dict({"a": [0, 1, 2]})
76
+ >>> d2 = Dataset.from_dict({"a": [10, 11, 12]})
77
+ >>> d3 = Dataset.from_dict({"a": [20, 21, 22]})
78
+ >>> dataset = interleave_datasets([d1, d2, d3], probabilities=[0.7, 0.2, 0.1], seed=42, stopping_strategy="all_exhausted")
79
+ >>> dataset["a"]
80
+ [10, 0, 11, 1, 2, 20, 12, 10, 0, 1, 2, 21, 0, 11, 1, 2, 0, 1, 12, 2, 10, 0, 22]
81
+ >>> dataset = interleave_datasets([d1, d2, d3], probabilities=[0.7, 0.2, 0.1], seed=42)
82
+ >>> dataset["a"]
83
+ [10, 0, 11, 1, 2]
84
+ >>> dataset = interleave_datasets([d1, d2, d3])
85
+ >>> dataset["a"]
86
+ [0, 10, 20, 1, 11, 21, 2, 12, 22]
87
+ >>> dataset = interleave_datasets([d1, d2, d3], stopping_strategy="all_exhausted")
88
+ >>> dataset["a"]
89
+ [0, 10, 20, 1, 11, 21, 2, 12, 22]
90
+ >>> d1 = Dataset.from_dict({"a": [0, 1, 2]})
91
+ >>> d2 = Dataset.from_dict({"a": [10, 11, 12, 13]})
92
+ >>> d3 = Dataset.from_dict({"a": [20, 21, 22, 23, 24]})
93
+ >>> dataset = interleave_datasets([d1, d2, d3])
94
+ >>> dataset["a"]
95
+ [0, 10, 20, 1, 11, 21, 2, 12, 22]
96
+ >>> dataset = interleave_datasets([d1, d2, d3], stopping_strategy="all_exhausted")
97
+ >>> dataset["a"]
98
+ [0, 10, 20, 1, 11, 21, 2, 12, 22, 0, 13, 23, 1, 10, 24]
99
+ >>> dataset = interleave_datasets([d1, d2, d3], probabilities=[0.7, 0.2, 0.1], seed=42)
100
+ >>> dataset["a"]
101
+ [10, 0, 11, 1, 2]
102
+ >>> dataset = interleave_datasets([d1, d2, d3], probabilities=[0.7, 0.2, 0.1], seed=42, stopping_strategy="all_exhausted")
103
+ >>> dataset["a"]
104
+ [10, 0, 11, 1, 2, 20, 12, 13, ..., 0, 1, 2, 0, 24]
105
+ For datasets in streaming mode (iterable):
106
+
107
+ >>> from datasets import interleave_datasets
108
+ >>> d1 = load_dataset('allenai/c4', 'es', split='train', streaming=True)
109
+ >>> d2 = load_dataset('allenai/c4', 'fr', split='train', streaming=True)
110
+ >>> dataset = interleave_datasets([d1, d2])
111
+ >>> iterator = iter(dataset)
112
+ >>> next(iterator)
113
+ {'text': 'Comprar Zapatillas para niña en chancla con goma por...'}
114
+ >>> next(iterator)
115
+ {'text': 'Le sacre de philippe ier, 23 mai 1059 - Compte Rendu...'
116
+ ```
117
+ """
118
+ from .arrow_dataset import Dataset
119
+ from .iterable_dataset import IterableDataset
120
+
121
+ if not datasets:
122
+ raise ValueError("Unable to interleave an empty list of datasets.")
123
+ for i, dataset in enumerate(datasets):
124
+ if not isinstance(dataset, (Dataset, IterableDataset)):
125
+ if isinstance(dataset, (DatasetDict, IterableDatasetDict)):
126
+ if not dataset:
127
+ raise ValueError(
128
+ f"Expected a list of Dataset objects or a list of IterableDataset objects, but element at position {i} "
129
+ "is an empty dataset dictionary."
130
+ )
131
+ raise ValueError(
132
+ f"Dataset at position {i} has at least one split: {list(dataset)}\n"
133
+ f"Please pick one to interleave with the other datasets, for example: dataset['{next(iter(dataset))}']"
134
+ )
135
+ raise ValueError(
136
+ f"Expected a list of Dataset objects or a list of IterableDataset objects, but element at position {i} is a {type(dataset).__name__}."
137
+ )
138
+ if i == 0:
139
+ dataset_type, other_type = (
140
+ (Dataset, IterableDataset) if isinstance(dataset, Dataset) else (IterableDataset, Dataset)
141
+ )
142
+ elif not isinstance(dataset, dataset_type):
143
+ raise ValueError(
144
+ f"Unable to interleave a {dataset_type.__name__} (at position 0) with a {other_type.__name__} (at position {i}). Expected a list of Dataset objects or a list of IterableDataset objects."
145
+ )
146
+ if stopping_strategy not in ["first_exhausted", "all_exhausted"]:
147
+ raise ValueError(f"{stopping_strategy} is not supported. Please enter a valid stopping_strategy.")
148
+ if dataset_type is Dataset:
149
+ return _interleave_map_style_datasets(
150
+ datasets, probabilities, seed, info=info, split=split, stopping_strategy=stopping_strategy
151
+ )
152
+ else:
153
+ return _interleave_iterable_datasets(
154
+ datasets, probabilities, seed, info=info, split=split, stopping_strategy=stopping_strategy
155
+ )
156
+
157
+
158
+ def concatenate_datasets(
159
+ dsets: list[DatasetType],
160
+ info: Optional[DatasetInfo] = None,
161
+ split: Optional[NamedSplit] = None,
162
+ axis: int = 0,
163
+ ) -> DatasetType:
164
+ """
165
+ Converts a list of [`Dataset`] with the same schema into a single [`Dataset`].
166
+
167
+ Args:
168
+ dsets (`List[datasets.Dataset]`):
169
+ List of Datasets to concatenate.
170
+ info (`DatasetInfo`, *optional*):
171
+ Dataset information, like description, citation, etc.
172
+ split (`NamedSplit`, *optional*):
173
+ Name of the dataset split.
174
+ axis (`{0, 1}`, defaults to `0`):
175
+ Axis to concatenate over, where `0` means over rows (vertically) and `1` means over columns
176
+ (horizontally).
177
+
178
+ <Added version="1.6.0"/>
179
+
180
+ Example:
181
+
182
+ ```py
183
+ >>> ds3 = concatenate_datasets([ds1, ds2])
184
+ ```
185
+ """
186
+
187
+ if not dsets:
188
+ raise ValueError("Unable to concatenate an empty list of datasets.")
189
+ for i, dataset in enumerate(dsets):
190
+ if not isinstance(dataset, (Dataset, IterableDataset)):
191
+ if isinstance(dataset, (DatasetDict, IterableDatasetDict)):
192
+ if not dataset:
193
+ raise ValueError(
194
+ f"Expected a list of Dataset objects or a list of IterableDataset objects, but element at position {i} "
195
+ "is an empty dataset dictionary."
196
+ )
197
+ raise ValueError(
198
+ f"Dataset at position {i} has at least one split: {list(dataset)}\n"
199
+ f"Please pick one to interleave with the other datasets, for example: dataset['{next(iter(dataset))}']"
200
+ )
201
+ raise ValueError(
202
+ f"Expected a list of Dataset objects or a list of IterableDataset objects, but element at position {i} is a {type(dataset).__name__}."
203
+ )
204
+ if i == 0:
205
+ dataset_type, other_type = (
206
+ (Dataset, IterableDataset) if isinstance(dataset, Dataset) else (IterableDataset, Dataset)
207
+ )
208
+ elif not isinstance(dataset, dataset_type):
209
+ raise ValueError(
210
+ f"Unable to interleave a {dataset_type.__name__} (at position 0) with a {other_type.__name__} (at position {i}). Expected a list of Dataset objects or a list of IterableDataset objects."
211
+ )
212
+ if dataset_type is Dataset:
213
+ return _concatenate_map_style_datasets(dsets, info=info, split=split, axis=axis)
214
+ else:
215
+ return _concatenate_iterable_datasets(dsets, info=info, split=split, axis=axis)
lib/python3.12/site-packages/datasets/commands/__init__.py ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from abc import ABC, abstractmethod
2
+ from argparse import ArgumentParser
3
+
4
+
5
+ class BaseDatasetsCLICommand(ABC):
6
+ @staticmethod
7
+ @abstractmethod
8
+ def register_subcommand(parser: ArgumentParser):
9
+ raise NotImplementedError()
10
+
11
+ @abstractmethod
12
+ def run(self):
13
+ raise NotImplementedError()
lib/python3.12/site-packages/datasets/commands/__pycache__/__init__.cpython-312.pyc ADDED
Binary file (953 Bytes). View file
 
lib/python3.12/site-packages/datasets/commands/__pycache__/convert.cpython-312.pyc ADDED
Binary file (9.6 kB). View file
 
lib/python3.12/site-packages/datasets/commands/__pycache__/convert_to_parquet.cpython-312.pyc ADDED
Binary file (2.61 kB). View file
 
lib/python3.12/site-packages/datasets/commands/__pycache__/datasets_cli.cpython-312.pyc ADDED
Binary file (2.23 kB). View file
 
lib/python3.12/site-packages/datasets/commands/__pycache__/delete_from_hub.cpython-312.pyc ADDED
Binary file (2.46 kB). View file
 
lib/python3.12/site-packages/datasets/commands/__pycache__/env.cpython-312.pyc ADDED
Binary file (2.39 kB). View file
 
lib/python3.12/site-packages/datasets/commands/__pycache__/test.cpython-312.pyc ADDED
Binary file (10.4 kB). View file
 
lib/python3.12/site-packages/datasets/commands/convert.py ADDED
@@ -0,0 +1,195 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import re
3
+ import shutil
4
+ from argparse import ArgumentParser, Namespace
5
+
6
+ from datasets.commands import BaseDatasetsCLICommand
7
+ from datasets.utils.logging import get_logger
8
+
9
+
10
+ HIGHLIGHT_MESSAGE_PRE = """<<<<<<< This should probably be modified because it mentions: """
11
+
12
+ HIGHLIGHT_MESSAGE_POST = """=======
13
+ >>>>>>>
14
+ """
15
+
16
+ TO_HIGHLIGHT = [
17
+ "TextEncoderConfig",
18
+ "ByteTextEncoder",
19
+ "SubwordTextEncoder",
20
+ "encoder_config",
21
+ "maybe_build_from_corpus",
22
+ "manual_dir",
23
+ ]
24
+
25
+ TO_CONVERT = [
26
+ # (pattern, replacement)
27
+ # Order is important here for some replacements
28
+ (r"tfds\.core", r"datasets"),
29
+ (r"tf\.io\.gfile\.GFile", r"open"),
30
+ (r"tf\.([\w\d]+)", r"datasets.Value('\1')"),
31
+ (r"tfds\.features\.Text\(\)", r"datasets.Value('string')"),
32
+ (r"tfds\.features\.Text\(", r"datasets.Value('string'),"),
33
+ (r"features\s*=\s*tfds.features.FeaturesDict\(", r"features=datasets.Features("),
34
+ (r"tfds\.features\.FeaturesDict\(", r"dict("),
35
+ (r"The TensorFlow Datasets Authors", r"The TensorFlow Datasets Authors and the HuggingFace Datasets Authors"),
36
+ (r"tfds\.", r"datasets."),
37
+ (r"dl_manager\.manual_dir", r"self.config.data_dir"),
38
+ (r"self\.builder_config", r"self.config"),
39
+ ]
40
+
41
+
42
+ def convert_command_factory(args: Namespace):
43
+ """
44
+ Factory function used to convert a model TF 1.0 checkpoint in a PyTorch checkpoint.
45
+
46
+ Returns: ConvertCommand
47
+ """
48
+ return ConvertCommand(args.tfds_path, args.datasets_directory)
49
+
50
+
51
+ class ConvertCommand(BaseDatasetsCLICommand):
52
+ @staticmethod
53
+ def register_subcommand(parser: ArgumentParser):
54
+ """
55
+ Register this command to argparse so it's available for the datasets-cli
56
+
57
+ Args:
58
+ parser: Root parser to register command-specific arguments
59
+ """
60
+ train_parser = parser.add_parser(
61
+ "convert",
62
+ help="Convert a TensorFlow Datasets dataset to a HuggingFace Datasets dataset.",
63
+ )
64
+ train_parser.add_argument(
65
+ "--tfds_path",
66
+ type=str,
67
+ required=True,
68
+ help="Path to a TensorFlow Datasets folder to convert or a single tfds file to convert.",
69
+ )
70
+ train_parser.add_argument(
71
+ "--datasets_directory", type=str, required=True, help="Path to the HuggingFace Datasets folder."
72
+ )
73
+ train_parser.set_defaults(func=convert_command_factory)
74
+
75
+ def __init__(self, tfds_path: str, datasets_directory: str, *args):
76
+ self._logger = get_logger("datasets-cli/converting")
77
+
78
+ self._tfds_path = tfds_path
79
+ self._datasets_directory = datasets_directory
80
+
81
+ def run(self):
82
+ if os.path.isdir(self._tfds_path):
83
+ abs_tfds_path = os.path.abspath(self._tfds_path)
84
+ elif os.path.isfile(self._tfds_path):
85
+ abs_tfds_path = os.path.dirname(self._tfds_path)
86
+ else:
87
+ raise ValueError("--tfds_path is neither a directory nor a file. Please check path.")
88
+
89
+ abs_datasets_path = os.path.abspath(self._datasets_directory)
90
+
91
+ self._logger.info(f"Converting datasets from {abs_tfds_path} to {abs_datasets_path}")
92
+
93
+ utils_files = []
94
+ with_manual_update = []
95
+ imports_to_builder_map = {}
96
+
97
+ if os.path.isdir(self._tfds_path):
98
+ file_names = os.listdir(abs_tfds_path)
99
+ else:
100
+ file_names = [os.path.basename(self._tfds_path)]
101
+
102
+ for f_name in file_names:
103
+ self._logger.info(f"Looking at file {f_name}")
104
+ input_file = os.path.join(abs_tfds_path, f_name)
105
+ output_file = os.path.join(abs_datasets_path, f_name)
106
+
107
+ if not os.path.isfile(input_file) or "__init__" in f_name or "_test" in f_name or ".py" not in f_name:
108
+ self._logger.info("Skipping file")
109
+ continue
110
+
111
+ with open(input_file, encoding="utf-8") as f:
112
+ lines = f.readlines()
113
+
114
+ out_lines = []
115
+ is_builder = False
116
+ needs_manual_update = False
117
+ tfds_imports = []
118
+ for line in lines:
119
+ out_line = line
120
+
121
+ # Convert imports
122
+ if "import tensorflow.compat.v2 as tf" in out_line:
123
+ continue
124
+ elif "@tfds.core" in out_line:
125
+ continue
126
+ elif "builder=self" in out_line:
127
+ continue
128
+ elif "import tensorflow_datasets.public_api as tfds" in out_line:
129
+ out_line = "import datasets\n"
130
+ elif "import tensorflow" in out_line:
131
+ # order is important here
132
+ out_line = ""
133
+ continue
134
+ elif "from absl import logging" in out_line:
135
+ out_line = "from datasets import logging\n"
136
+ elif "getLogger" in out_line:
137
+ out_line = out_line.replace("getLogger", "get_logger")
138
+ elif any(expression in out_line for expression in TO_HIGHLIGHT):
139
+ needs_manual_update = True
140
+ to_remove = list(filter(lambda e: e in out_line, TO_HIGHLIGHT))
141
+ out_lines.append(HIGHLIGHT_MESSAGE_PRE + str(to_remove) + "\n")
142
+ out_lines.append(out_line)
143
+ out_lines.append(HIGHLIGHT_MESSAGE_POST)
144
+ continue
145
+ else:
146
+ for pattern, replacement in TO_CONVERT:
147
+ out_line = re.sub(pattern, replacement, out_line)
148
+
149
+ # Take care of saving utilities (to later move them together with main script)
150
+ if "tensorflow_datasets" in out_line:
151
+ match = re.match(r"from\stensorflow_datasets.*import\s([^\.\r\n]+)", out_line)
152
+ tfds_imports.extend(imp.strip() for imp in match.group(1).split(","))
153
+ out_line = "from . import " + match.group(1)
154
+
155
+ # Check we have not forget anything
156
+ if "tf." in out_line or "tfds." in out_line or "tensorflow_datasets" in out_line:
157
+ raise ValueError(f"Error converting {out_line.strip()}")
158
+
159
+ if "GeneratorBasedBuilder" in out_line:
160
+ is_builder = True
161
+ out_lines.append(out_line)
162
+
163
+ if is_builder or "wmt" in f_name:
164
+ # We create a new directory for each dataset
165
+ dir_name = f_name.replace(".py", "")
166
+ output_dir = os.path.join(abs_datasets_path, dir_name)
167
+ output_file = os.path.join(output_dir, f_name)
168
+ os.makedirs(output_dir, exist_ok=True)
169
+ self._logger.info(f"Adding directory {output_dir}")
170
+ imports_to_builder_map.update(dict.fromkeys(tfds_imports, output_dir))
171
+ else:
172
+ # Utilities will be moved at the end
173
+ utils_files.append(output_file)
174
+
175
+ if needs_manual_update:
176
+ with_manual_update.append(output_file)
177
+
178
+ with open(output_file, "w", encoding="utf-8") as f:
179
+ f.writelines(out_lines)
180
+ self._logger.info(f"Converted in {output_file}")
181
+
182
+ for utils_file in utils_files:
183
+ try:
184
+ f_name = os.path.basename(utils_file)
185
+ dest_folder = imports_to_builder_map[f_name.replace(".py", "")]
186
+ self._logger.info(f"Moving {dest_folder} to {utils_file}")
187
+ shutil.copy(utils_file, dest_folder)
188
+ except KeyError:
189
+ self._logger.error(f"Cannot find destination folder for {utils_file}. Please copy manually.")
190
+
191
+ if with_manual_update:
192
+ for file_path in with_manual_update:
193
+ self._logger.warning(
194
+ f"You need to manually update file {file_path} to remove configurations using 'TextEncoderConfig'."
195
+ )
lib/python3.12/site-packages/datasets/commands/convert_to_parquet.py ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from argparse import ArgumentParser
2
+ from typing import Optional
3
+
4
+ from datasets.commands import BaseDatasetsCLICommand
5
+ from datasets.hub import convert_to_parquet
6
+
7
+
8
+ def _command_factory(args):
9
+ return ConvertToParquetCommand(
10
+ args.dataset_id,
11
+ args.token,
12
+ args.revision,
13
+ args.trust_remote_code,
14
+ )
15
+
16
+
17
+ class ConvertToParquetCommand(BaseDatasetsCLICommand):
18
+ @staticmethod
19
+ def register_subcommand(parser):
20
+ parser: ArgumentParser = parser.add_parser("convert_to_parquet", help="Convert dataset to Parquet")
21
+ parser.add_argument(
22
+ "dataset_id", help="source dataset ID, e.g. USERNAME/DATASET_NAME or ORGANIZATION/DATASET_NAME"
23
+ )
24
+ parser.add_argument("--token", help="access token to the Hugging Face Hub (defaults to logged-in user's one)")
25
+ parser.add_argument("--revision", help="source revision")
26
+ parser.add_argument(
27
+ "--trust_remote_code", action="store_true", help="whether to trust the code execution of the load script"
28
+ )
29
+ parser.set_defaults(func=_command_factory)
30
+
31
+ def __init__(
32
+ self,
33
+ dataset_id: str,
34
+ token: Optional[str],
35
+ revision: Optional[str],
36
+ trust_remote_code: bool,
37
+ ):
38
+ self._dataset_id = dataset_id
39
+ self._token = token
40
+ self._revision = revision
41
+ self._trust_remote_code = trust_remote_code
42
+
43
+ def run(self) -> None:
44
+ _ = convert_to_parquet(
45
+ self._dataset_id, revision=self._revision, token=self._token, trust_remote_code=self._trust_remote_code
46
+ )
lib/python3.12/site-packages/datasets/commands/datasets_cli.py ADDED
@@ -0,0 +1,43 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python
2
+ from argparse import ArgumentParser
3
+
4
+ from datasets.commands.convert import ConvertCommand
5
+ from datasets.commands.convert_to_parquet import ConvertToParquetCommand
6
+ from datasets.commands.delete_from_hub import DeleteFromHubCommand
7
+ from datasets.commands.env import EnvironmentCommand
8
+ from datasets.commands.test import TestCommand
9
+ from datasets.utils.logging import set_verbosity_info
10
+
11
+
12
+ def parse_unknown_args(unknown_args):
13
+ return {key.lstrip("-"): value for key, value in zip(unknown_args[::2], unknown_args[1::2])}
14
+
15
+
16
+ def main():
17
+ parser = ArgumentParser(
18
+ "HuggingFace Datasets CLI tool", usage="datasets-cli <command> [<args>]", allow_abbrev=False
19
+ )
20
+ commands_parser = parser.add_subparsers(help="datasets-cli command helpers")
21
+ set_verbosity_info()
22
+
23
+ # Register commands
24
+ ConvertCommand.register_subcommand(commands_parser)
25
+ EnvironmentCommand.register_subcommand(commands_parser)
26
+ TestCommand.register_subcommand(commands_parser)
27
+ ConvertToParquetCommand.register_subcommand(commands_parser)
28
+ DeleteFromHubCommand.register_subcommand(commands_parser)
29
+
30
+ # Parse args
31
+ args, unknown_args = parser.parse_known_args()
32
+ if not hasattr(args, "func"):
33
+ parser.print_help()
34
+ exit(1)
35
+ kwargs = parse_unknown_args(unknown_args)
36
+
37
+ # Run
38
+ service = args.func(args, **kwargs)
39
+ service.run()
40
+
41
+
42
+ if __name__ == "__main__":
43
+ main()
lib/python3.12/site-packages/datasets/commands/delete_from_hub.py ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from argparse import ArgumentParser
2
+ from typing import Optional
3
+
4
+ from datasets.commands import BaseDatasetsCLICommand
5
+ from datasets.hub import delete_from_hub
6
+
7
+
8
+ def _command_factory(args):
9
+ return DeleteFromHubCommand(
10
+ args.dataset_id,
11
+ args.config_name,
12
+ args.token,
13
+ args.revision,
14
+ )
15
+
16
+
17
+ class DeleteFromHubCommand(BaseDatasetsCLICommand):
18
+ @staticmethod
19
+ def register_subcommand(parser):
20
+ parser: ArgumentParser = parser.add_parser("delete_from_hub", help="Delete dataset config from the Hub")
21
+ parser.add_argument(
22
+ "dataset_id", help="source dataset ID, e.g. USERNAME/DATASET_NAME or ORGANIZATION/DATASET_NAME"
23
+ )
24
+ parser.add_argument("config_name", help="config name to delete")
25
+ parser.add_argument("--token", help="access token to the Hugging Face Hub")
26
+ parser.add_argument("--revision", help="source revision")
27
+ parser.set_defaults(func=_command_factory)
28
+
29
+ def __init__(
30
+ self,
31
+ dataset_id: str,
32
+ config_name: str,
33
+ token: Optional[str],
34
+ revision: Optional[str],
35
+ ):
36
+ self._dataset_id = dataset_id
37
+ self._config_name = config_name
38
+ self._token = token
39
+ self._revision = revision
40
+
41
+ def run(self) -> None:
42
+ _ = delete_from_hub(self._dataset_id, self._config_name, revision=self._revision, token=self._token)
lib/python3.12/site-packages/datasets/commands/env.py ADDED
@@ -0,0 +1,41 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import platform
2
+ from argparse import ArgumentParser
3
+
4
+ import fsspec
5
+ import huggingface_hub
6
+ import pandas
7
+ import pyarrow
8
+
9
+ from datasets import __version__ as version
10
+ from datasets.commands import BaseDatasetsCLICommand
11
+
12
+
13
+ def info_command_factory(_):
14
+ return EnvironmentCommand()
15
+
16
+
17
+ class EnvironmentCommand(BaseDatasetsCLICommand):
18
+ @staticmethod
19
+ def register_subcommand(parser: ArgumentParser):
20
+ download_parser = parser.add_parser("env", help="Print relevant system environment info.")
21
+ download_parser.set_defaults(func=info_command_factory)
22
+
23
+ def run(self):
24
+ info = {
25
+ "`datasets` version": version,
26
+ "Platform": platform.platform(),
27
+ "Python version": platform.python_version(),
28
+ "`huggingface_hub` version": huggingface_hub.__version__,
29
+ "PyArrow version": pyarrow.__version__,
30
+ "Pandas version": pandas.__version__,
31
+ "`fsspec` version": fsspec.__version__,
32
+ }
33
+
34
+ print("\nCopy-and-paste the text below in your GitHub issue.\n")
35
+ print(self.format_dict(info))
36
+
37
+ return info
38
+
39
+ @staticmethod
40
+ def format_dict(d):
41
+ return "\n".join([f"- {prop}: {val}" for prop, val in d.items()]) + "\n"
lib/python3.12/site-packages/datasets/commands/test.py ADDED
@@ -0,0 +1,207 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import logging
2
+ import os
3
+ from argparse import ArgumentParser
4
+ from collections.abc import Generator
5
+ from pathlib import Path
6
+ from shutil import copyfile, rmtree
7
+ from typing import Optional
8
+
9
+ import datasets.config
10
+ from datasets.builder import DatasetBuilder
11
+ from datasets.commands import BaseDatasetsCLICommand
12
+ from datasets.download.download_manager import DownloadMode
13
+ from datasets.load import dataset_module_factory, import_main_class
14
+ from datasets.utils.info_utils import VerificationMode
15
+ from datasets.utils.logging import ERROR, get_logger
16
+
17
+
18
+ logger = get_logger(__name__)
19
+
20
+
21
+ def _test_command_factory(args):
22
+ return TestCommand(
23
+ args.dataset,
24
+ args.name,
25
+ args.cache_dir,
26
+ args.data_dir,
27
+ args.all_configs,
28
+ args.save_info or args.save_infos,
29
+ args.ignore_verifications,
30
+ args.force_redownload,
31
+ args.clear_cache,
32
+ args.num_proc,
33
+ args.trust_remote_code,
34
+ )
35
+
36
+
37
+ class TestCommand(BaseDatasetsCLICommand):
38
+ __test__ = False # to tell pytest it's not a test class
39
+
40
+ @staticmethod
41
+ def register_subcommand(parser: ArgumentParser):
42
+ test_parser = parser.add_parser("test", help="Test dataset implementation.")
43
+ test_parser.add_argument("--name", type=str, default=None, help="Dataset processing name")
44
+ test_parser.add_argument(
45
+ "--cache_dir",
46
+ type=str,
47
+ default=None,
48
+ help="Cache directory where the datasets are stored.",
49
+ )
50
+ test_parser.add_argument(
51
+ "--data_dir",
52
+ type=str,
53
+ default=None,
54
+ help="Can be used to specify a manual directory to get the files from.",
55
+ )
56
+ test_parser.add_argument("--all_configs", action="store_true", help="Test all dataset configurations")
57
+ test_parser.add_argument(
58
+ "--save_info", action="store_true", help="Save the dataset infos in the dataset card (README.md)"
59
+ )
60
+ test_parser.add_argument(
61
+ "--ignore_verifications",
62
+ action="store_true",
63
+ help="Run the test without checksums and splits checks.",
64
+ )
65
+ test_parser.add_argument("--force_redownload", action="store_true", help="Force dataset redownload")
66
+ test_parser.add_argument(
67
+ "--clear_cache",
68
+ action="store_true",
69
+ help="Remove downloaded files and cached datasets after each config test",
70
+ )
71
+ test_parser.add_argument("--num_proc", type=int, default=None, help="Number of processes")
72
+ test_parser.add_argument(
73
+ "--trust_remote_code", action="store_true", help="whether to trust the code execution of the load script"
74
+ )
75
+ # aliases
76
+ test_parser.add_argument("--save_infos", action="store_true", help="alias to save_info")
77
+ test_parser.add_argument("dataset", type=str, help="Name of the dataset to download")
78
+ test_parser.set_defaults(func=_test_command_factory)
79
+
80
+ def __init__(
81
+ self,
82
+ dataset: str,
83
+ name: str,
84
+ cache_dir: str,
85
+ data_dir: str,
86
+ all_configs: bool,
87
+ save_infos: bool,
88
+ ignore_verifications: bool,
89
+ force_redownload: bool,
90
+ clear_cache: bool,
91
+ num_proc: int,
92
+ trust_remote_code: Optional[bool],
93
+ ):
94
+ self._dataset = dataset
95
+ self._name = name
96
+ self._cache_dir = cache_dir
97
+ self._data_dir = data_dir
98
+ self._all_configs = all_configs
99
+ self._save_infos = save_infos
100
+ self._ignore_verifications = ignore_verifications
101
+ self._force_redownload = force_redownload
102
+ self._clear_cache = clear_cache
103
+ self._num_proc = num_proc
104
+ self._trust_remote_code = trust_remote_code
105
+ if clear_cache and not cache_dir:
106
+ print(
107
+ "When --clear_cache is used, specifying a cache directory is mandatory.\n"
108
+ "The 'download' folder of the cache directory and the dataset builder cache will be deleted after each configuration test.\n"
109
+ "Please provide a --cache_dir that will be used to test the dataset script."
110
+ )
111
+ exit(1)
112
+ if save_infos:
113
+ self._ignore_verifications = True
114
+
115
+ def run(self):
116
+ logging.getLogger("filelock").setLevel(ERROR)
117
+ if self._name is not None and self._all_configs:
118
+ print("Both parameters `config` and `all_configs` can't be used at once.")
119
+ exit(1)
120
+ path, config_name = self._dataset, self._name
121
+ module = dataset_module_factory(path, trust_remote_code=self._trust_remote_code)
122
+ builder_cls = import_main_class(module.module_path)
123
+ n_builders = len(builder_cls.BUILDER_CONFIGS) if self._all_configs and builder_cls.BUILDER_CONFIGS else 1
124
+
125
+ def get_builders() -> Generator[DatasetBuilder, None, None]:
126
+ if self._all_configs and builder_cls.BUILDER_CONFIGS:
127
+ for i, config in enumerate(builder_cls.BUILDER_CONFIGS):
128
+ if "config_name" in module.builder_kwargs:
129
+ yield builder_cls(
130
+ cache_dir=self._cache_dir,
131
+ data_dir=self._data_dir,
132
+ **module.builder_kwargs,
133
+ )
134
+ else:
135
+ yield builder_cls(
136
+ config_name=config.name,
137
+ cache_dir=self._cache_dir,
138
+ data_dir=self._data_dir,
139
+ **module.builder_kwargs,
140
+ )
141
+ else:
142
+ if "config_name" in module.builder_kwargs:
143
+ yield builder_cls(cache_dir=self._cache_dir, data_dir=self._data_dir, **module.builder_kwargs)
144
+ else:
145
+ yield builder_cls(
146
+ config_name=config_name,
147
+ cache_dir=self._cache_dir,
148
+ data_dir=self._data_dir,
149
+ **module.builder_kwargs,
150
+ )
151
+
152
+ for j, builder in enumerate(get_builders()):
153
+ print(f"Testing builder '{builder.config.name}' ({j + 1}/{n_builders})")
154
+ builder._record_infos = os.path.exists(
155
+ os.path.join(builder.get_imported_module_dir(), datasets.config.DATASETDICT_INFOS_FILENAME)
156
+ ) # record checksums only if we need to update a (deprecated) dataset_infos.json
157
+ builder.download_and_prepare(
158
+ download_mode=DownloadMode.REUSE_CACHE_IF_EXISTS
159
+ if not self._force_redownload
160
+ else DownloadMode.FORCE_REDOWNLOAD,
161
+ verification_mode=VerificationMode.NO_CHECKS
162
+ if self._ignore_verifications
163
+ else VerificationMode.ALL_CHECKS,
164
+ num_proc=self._num_proc,
165
+ )
166
+ builder.as_dataset()
167
+ if self._save_infos:
168
+ builder._save_infos()
169
+
170
+ # If save_infos=True, the dataset card (README.md) is created next to the loaded module file.
171
+ # The dataset_infos are saved in the YAML part of the README.md
172
+
173
+ # Let's move it to the original directory of the dataset script, to allow the user to
174
+ # upload them on S3 at the same time afterwards.
175
+ if self._save_infos:
176
+ dataset_readme_path = os.path.join(
177
+ builder_cls.get_imported_module_dir(), datasets.config.REPOCARD_FILENAME
178
+ )
179
+ name = Path(path).name + ".py"
180
+ combined_path = os.path.join(path, name)
181
+ if os.path.isfile(path):
182
+ dataset_dir = os.path.dirname(path)
183
+ elif os.path.isfile(combined_path):
184
+ dataset_dir = path
185
+ elif os.path.isdir(path): # for local directories containing only data files
186
+ dataset_dir = path
187
+ else: # in case of a remote dataset
188
+ dataset_dir = None
189
+ print(f"Dataset card saved at {dataset_readme_path}")
190
+
191
+ # Move dataset_info back to the user
192
+ if dataset_dir is not None:
193
+ user_dataset_readme_path = os.path.join(dataset_dir, datasets.config.REPOCARD_FILENAME)
194
+ copyfile(dataset_readme_path, user_dataset_readme_path)
195
+ print(f"Dataset card saved at {user_dataset_readme_path}")
196
+
197
+ # If clear_cache=True, the download folder and the dataset builder cache directory are deleted
198
+ if self._clear_cache:
199
+ if os.path.isdir(builder._cache_dir):
200
+ logger.warning(f"Clearing cache at {builder._cache_dir}")
201
+ rmtree(builder._cache_dir)
202
+ download_dir = os.path.join(self._cache_dir, datasets.config.DOWNLOADED_DATASETS_DIR)
203
+ if os.path.isdir(download_dir):
204
+ logger.warning(f"Clearing cache at {download_dir}")
205
+ rmtree(download_dir)
206
+
207
+ print("Test successful.")
lib/python3.12/site-packages/datasets/config.py ADDED
@@ -0,0 +1,269 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import importlib
2
+ import importlib.metadata
3
+ import logging
4
+ import os
5
+ import platform
6
+ from pathlib import Path
7
+ from typing import Optional
8
+
9
+ from huggingface_hub import constants
10
+ from packaging import version
11
+
12
+
13
+ logger = logging.getLogger(__name__.split(".", 1)[0]) # to avoid circular import from .utils.logging
14
+
15
+ # Datasets
16
+ S3_DATASETS_BUCKET_PREFIX = "https://s3.amazonaws.com/datasets.huggingface.co/datasets/datasets"
17
+ CLOUDFRONT_DATASETS_DISTRIB_PREFIX = "https://cdn-datasets.huggingface.co/datasets/datasets"
18
+ REPO_DATASETS_URL = "https://raw.githubusercontent.com/huggingface/datasets/{revision}/datasets/{path}/{name}"
19
+
20
+ # Hub
21
+ HF_ENDPOINT = os.environ.get("HF_ENDPOINT", "https://huggingface.co")
22
+ HUB_DATASETS_URL = HF_ENDPOINT + "/datasets/{repo_id}/resolve/{revision}/{path}"
23
+ HUB_DATASETS_HFFS_URL = "hf://datasets/{repo_id}@{revision}/{path}"
24
+ HUB_DEFAULT_VERSION = "main"
25
+
26
+ PY_VERSION = version.parse(platform.python_version())
27
+
28
+ # General environment variables accepted values for booleans
29
+ ENV_VARS_TRUE_VALUES = {"1", "ON", "YES", "TRUE"}
30
+ ENV_VARS_FALSE_VALUES = {"0", "OFF", "NO", "FALSE"}
31
+ ENV_VARS_TRUE_AND_AUTO_VALUES = ENV_VARS_TRUE_VALUES.union({"AUTO"})
32
+ ENV_VARS_FALSE_AND_AUTO_VALUES = ENV_VARS_FALSE_VALUES.union({"AUTO"})
33
+
34
+
35
+ # Imports
36
+ DILL_VERSION = version.parse(importlib.metadata.version("dill"))
37
+ FSSPEC_VERSION = version.parse(importlib.metadata.version("fsspec"))
38
+ PANDAS_VERSION = version.parse(importlib.metadata.version("pandas"))
39
+ PYARROW_VERSION = version.parse(importlib.metadata.version("pyarrow"))
40
+ HF_HUB_VERSION = version.parse(importlib.metadata.version("huggingface_hub"))
41
+
42
+ USE_TF = os.environ.get("USE_TF", "AUTO").upper()
43
+ USE_TORCH = os.environ.get("USE_TORCH", "AUTO").upper()
44
+ USE_JAX = os.environ.get("USE_JAX", "AUTO").upper()
45
+
46
+ TORCH_VERSION = "N/A"
47
+ TORCH_AVAILABLE = False
48
+
49
+ if USE_TORCH in ENV_VARS_TRUE_AND_AUTO_VALUES and USE_TF not in ENV_VARS_TRUE_VALUES:
50
+ TORCH_AVAILABLE = importlib.util.find_spec("torch") is not None
51
+ if TORCH_AVAILABLE:
52
+ try:
53
+ TORCH_VERSION = version.parse(importlib.metadata.version("torch"))
54
+ logger.info(f"PyTorch version {TORCH_VERSION} available.")
55
+ except importlib.metadata.PackageNotFoundError:
56
+ pass
57
+ else:
58
+ logger.info("Disabling PyTorch because USE_TF is set")
59
+
60
+ POLARS_VERSION = "N/A"
61
+ POLARS_AVAILABLE = importlib.util.find_spec("polars") is not None
62
+
63
+ if POLARS_AVAILABLE:
64
+ try:
65
+ POLARS_VERSION = version.parse(importlib.metadata.version("polars"))
66
+ logger.info(f"Polars version {POLARS_VERSION} available.")
67
+ except importlib.metadata.PackageNotFoundError:
68
+ pass
69
+
70
+
71
+ DUCKDB_VERSION = "N/A"
72
+ DUCKDB_AVAILABLE = importlib.util.find_spec("duckdb") is not None
73
+
74
+ if DUCKDB_AVAILABLE:
75
+ try:
76
+ DUCKDB_VERSION = version.parse(importlib.metadata.version("duckdb"))
77
+ logger.info(f"Duckdb version {DUCKDB_VERSION} available.")
78
+ except importlib.metadata.PackageNotFoundError:
79
+ pass
80
+
81
+ TF_VERSION = "N/A"
82
+ TF_AVAILABLE = False
83
+
84
+ if USE_TF in ENV_VARS_TRUE_AND_AUTO_VALUES and USE_TORCH not in ENV_VARS_TRUE_VALUES:
85
+ TF_AVAILABLE = importlib.util.find_spec("tensorflow") is not None
86
+ if TF_AVAILABLE:
87
+ # For the metadata, we have to look for both tensorflow and tensorflow-cpu
88
+ for package in [
89
+ "tensorflow",
90
+ "tensorflow-cpu",
91
+ "tensorflow-gpu",
92
+ "tf-nightly",
93
+ "tf-nightly-cpu",
94
+ "tf-nightly-gpu",
95
+ "intel-tensorflow",
96
+ "tensorflow-rocm",
97
+ "tensorflow-macos",
98
+ ]:
99
+ try:
100
+ TF_VERSION = version.parse(importlib.metadata.version(package))
101
+ except importlib.metadata.PackageNotFoundError:
102
+ continue
103
+ else:
104
+ break
105
+ else:
106
+ TF_AVAILABLE = False
107
+ if TF_AVAILABLE:
108
+ if TF_VERSION.major < 2:
109
+ logger.info(f"TensorFlow found but with version {TF_VERSION}. `datasets` requires version 2 minimum.")
110
+ TF_AVAILABLE = False
111
+ else:
112
+ logger.info(f"TensorFlow version {TF_VERSION} available.")
113
+ else:
114
+ logger.info("Disabling Tensorflow because USE_TORCH is set")
115
+
116
+
117
+ JAX_VERSION = "N/A"
118
+ JAX_AVAILABLE = False
119
+
120
+ if USE_JAX in ENV_VARS_TRUE_AND_AUTO_VALUES:
121
+ JAX_AVAILABLE = importlib.util.find_spec("jax") is not None and importlib.util.find_spec("jaxlib") is not None
122
+ if JAX_AVAILABLE:
123
+ try:
124
+ JAX_VERSION = version.parse(importlib.metadata.version("jax"))
125
+ logger.info(f"JAX version {JAX_VERSION} available.")
126
+ except importlib.metadata.PackageNotFoundError:
127
+ pass
128
+ else:
129
+ logger.info("Disabling JAX because USE_JAX is set to False")
130
+
131
+
132
+ # Optional tools for data loading
133
+ SQLALCHEMY_AVAILABLE = importlib.util.find_spec("sqlalchemy") is not None
134
+
135
+ # Optional tools for feature decoding
136
+ PIL_AVAILABLE = importlib.util.find_spec("PIL") is not None
137
+ IS_OPUS_SUPPORTED = importlib.util.find_spec("soundfile") is not None and version.parse(
138
+ importlib.import_module("soundfile").__libsndfile_version__
139
+ ) >= version.parse("1.0.31")
140
+ IS_MP3_SUPPORTED = importlib.util.find_spec("soundfile") is not None and version.parse(
141
+ importlib.import_module("soundfile").__libsndfile_version__
142
+ ) >= version.parse("1.1.0")
143
+ TORCHVISION_AVAILABLE = importlib.util.find_spec("torchvision") is not None
144
+ PDFPLUMBER_AVAILABLE = importlib.util.find_spec("pdfplumber") is not None
145
+
146
+ # Optional compression tools
147
+ RARFILE_AVAILABLE = importlib.util.find_spec("rarfile") is not None
148
+ ZSTANDARD_AVAILABLE = importlib.util.find_spec("zstandard") is not None
149
+ LZ4_AVAILABLE = importlib.util.find_spec("lz4") is not None
150
+ PY7ZR_AVAILABLE = importlib.util.find_spec("py7zr") is not None
151
+
152
+ # Cache location
153
+ DEFAULT_XDG_CACHE_HOME = "~/.cache"
154
+ XDG_CACHE_HOME = os.getenv("XDG_CACHE_HOME", DEFAULT_XDG_CACHE_HOME)
155
+ DEFAULT_HF_CACHE_HOME = os.path.join(XDG_CACHE_HOME, "huggingface")
156
+ HF_CACHE_HOME = os.path.expanduser(os.getenv("HF_HOME", DEFAULT_HF_CACHE_HOME))
157
+
158
+ DEFAULT_HF_DATASETS_CACHE = os.path.join(HF_CACHE_HOME, "datasets")
159
+ HF_DATASETS_CACHE = Path(os.getenv("HF_DATASETS_CACHE", DEFAULT_HF_DATASETS_CACHE))
160
+
161
+ DEFAULT_HF_MODULES_CACHE = os.path.join(HF_CACHE_HOME, "modules")
162
+ HF_MODULES_CACHE = Path(os.getenv("HF_MODULES_CACHE", DEFAULT_HF_MODULES_CACHE))
163
+
164
+ DOWNLOADED_DATASETS_DIR = "downloads"
165
+ DEFAULT_DOWNLOADED_DATASETS_PATH = os.path.join(HF_DATASETS_CACHE, DOWNLOADED_DATASETS_DIR)
166
+ DOWNLOADED_DATASETS_PATH = Path(os.getenv("HF_DATASETS_DOWNLOADED_DATASETS_PATH", DEFAULT_DOWNLOADED_DATASETS_PATH))
167
+
168
+ EXTRACTED_DATASETS_DIR = "extracted"
169
+ DEFAULT_EXTRACTED_DATASETS_PATH = os.path.join(DEFAULT_DOWNLOADED_DATASETS_PATH, EXTRACTED_DATASETS_DIR)
170
+ EXTRACTED_DATASETS_PATH = Path(os.getenv("HF_DATASETS_EXTRACTED_DATASETS_PATH", DEFAULT_EXTRACTED_DATASETS_PATH))
171
+
172
+ # Download count for the website
173
+ HF_UPDATE_DOWNLOAD_COUNTS = (
174
+ os.environ.get("HF_UPDATE_DOWNLOAD_COUNTS", "AUTO").upper() in ENV_VARS_TRUE_AND_AUTO_VALUES
175
+ )
176
+
177
+ # For downloads and to check remote files metadata
178
+ HF_DATASETS_MULTITHREADING_MAX_WORKERS = 16
179
+
180
+ # Remote dataset scripts support
181
+ __HF_DATASETS_TRUST_REMOTE_CODE = os.environ.get("HF_DATASETS_TRUST_REMOTE_CODE", "ask")
182
+ HF_DATASETS_TRUST_REMOTE_CODE: Optional[bool] = (
183
+ True
184
+ if __HF_DATASETS_TRUST_REMOTE_CODE.upper() in ENV_VARS_TRUE_VALUES
185
+ else False
186
+ if __HF_DATASETS_TRUST_REMOTE_CODE.upper() in ENV_VARS_FALSE_VALUES
187
+ else None
188
+ )
189
+ TIME_OUT_REMOTE_CODE = 15
190
+
191
+ # Dataset viewer API
192
+ USE_PARQUET_EXPORT = True
193
+
194
+ # Batch size constants. For more info, see:
195
+ # https://github.com/apache/arrow/blob/master/docs/source/cpp/arrays.rst#size-limitations-and-recommendations)
196
+ DEFAULT_MAX_BATCH_SIZE = 1000
197
+
198
+ # Size of the preloaded record batch in `Dataset.__iter__`
199
+ ARROW_READER_BATCH_SIZE_IN_DATASET_ITER = 10
200
+
201
+ # Max shard size in bytes (e.g. to shard parquet datasets in push_to_hub or download_and_prepare)
202
+ MAX_SHARD_SIZE = "500MB"
203
+
204
+ # Parquet configuration
205
+ PARQUET_ROW_GROUP_SIZE_FOR_AUDIO_DATASETS = 100
206
+ PARQUET_ROW_GROUP_SIZE_FOR_IMAGE_DATASETS = 100
207
+ PARQUET_ROW_GROUP_SIZE_FOR_BINARY_DATASETS = 100
208
+ PARQUET_ROW_GROUP_SIZE_FOR_VIDEO_DATASETS = 10
209
+
210
+ # Offline mode
211
+ _offline = os.environ.get("HF_DATASETS_OFFLINE")
212
+ HF_HUB_OFFLINE = constants.HF_HUB_OFFLINE if _offline is None else _offline.upper() in ENV_VARS_TRUE_VALUES
213
+ HF_DATASETS_OFFLINE = HF_HUB_OFFLINE # kept for backward-compatibility
214
+
215
+ # Here, `True` will disable progress bars globally without possibility of enabling it
216
+ # programmatically. `False` will enable them without possibility of disabling them.
217
+ # If environment variable is not set (None), then the user is free to enable/disable
218
+ # them programmatically.
219
+ # TL;DR: env variable has priority over code
220
+ __HF_DATASETS_DISABLE_PROGRESS_BARS = os.environ.get("HF_DATASETS_DISABLE_PROGRESS_BARS")
221
+ HF_DATASETS_DISABLE_PROGRESS_BARS: Optional[bool] = (
222
+ __HF_DATASETS_DISABLE_PROGRESS_BARS.upper() in ENV_VARS_TRUE_VALUES
223
+ if __HF_DATASETS_DISABLE_PROGRESS_BARS is not None
224
+ else None
225
+ )
226
+
227
+ # In-memory
228
+ DEFAULT_IN_MEMORY_MAX_SIZE = 0 # Disabled
229
+ IN_MEMORY_MAX_SIZE = float(os.environ.get("HF_DATASETS_IN_MEMORY_MAX_SIZE", DEFAULT_IN_MEMORY_MAX_SIZE))
230
+
231
+ # File names
232
+ DATASET_ARROW_FILENAME = "dataset.arrow"
233
+ DATASET_INDICES_FILENAME = "indices.arrow"
234
+ DATASET_STATE_JSON_FILENAME = "state.json"
235
+ DATASET_INFO_FILENAME = "dataset_info.json"
236
+ DATASETDICT_INFOS_FILENAME = "dataset_infos.json"
237
+ LICENSE_FILENAME = "LICENSE"
238
+ DATASETDICT_JSON_FILENAME = "dataset_dict.json"
239
+ METADATA_CONFIGS_FIELD = "configs"
240
+ REPOCARD_FILENAME = "README.md"
241
+ REPOYAML_FILENAME = ".huggingface.yaml"
242
+
243
+ MODULE_NAME_FOR_DYNAMIC_MODULES = "datasets_modules"
244
+
245
+ MAX_DATASET_CONFIG_ID_READABLE_LENGTH = 255
246
+
247
+ # Temporary cache directory prefix
248
+ TEMP_CACHE_DIR_PREFIX = "hf_datasets-"
249
+
250
+ # Streaming
251
+ STREAMING_READ_MAX_RETRIES = 20
252
+ STREAMING_READ_RETRY_INTERVAL = 5
253
+
254
+ # Datasets without script
255
+ DATA_FILES_MAX_NUMBER_FOR_MODULE_INFERENCE = 200
256
+ GLOBBED_DATA_FILES_MAX_NUMBER_FOR_MODULE_INFERENCE = 10
257
+ ARCHIVED_DATA_FILES_MAX_NUMBER_FOR_MODULE_INFERENCE = 200
258
+
259
+ # Async map functions
260
+ MAX_NUM_RUNNING_ASYNC_MAP_FUNCTIONS_IN_PARALLEL = 1000
261
+
262
+ # Progress bars
263
+ PBAR_REFRESH_TIME_INTERVAL = 0.05 # 20 progress updates per sec
264
+
265
+ # Maximum number of uploaded files per commit
266
+ UPLOADS_MAX_NUMBER_PER_COMMIT = 50
267
+
268
+ # Backward compatibility
269
+ MAX_TABLE_NBYTES_FOR_PICKLING = 4 << 30
lib/python3.12/site-packages/datasets/data_files.py ADDED
@@ -0,0 +1,795 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import re
3
+ from functools import partial
4
+ from glob import has_magic
5
+ from pathlib import Path, PurePath
6
+ from typing import Callable, Optional, Union
7
+
8
+ import huggingface_hub
9
+ from fsspec.core import url_to_fs
10
+ from huggingface_hub import HfFileSystem
11
+ from packaging import version
12
+ from tqdm.contrib.concurrent import thread_map
13
+
14
+ from . import config
15
+ from .download import DownloadConfig
16
+ from .naming import _split_re
17
+ from .splits import Split
18
+ from .utils import logging
19
+ from .utils import tqdm as hf_tqdm
20
+ from .utils.file_utils import _prepare_path_and_storage_options, is_local_path, is_relative_path, xbasename, xjoin
21
+ from .utils.py_utils import glob_pattern_to_regex, string_to_dict
22
+
23
+
24
+ SingleOriginMetadata = Union[tuple[str, str], tuple[str], tuple[()]]
25
+
26
+
27
+ SANITIZED_DEFAULT_SPLIT = str(Split.TRAIN)
28
+
29
+
30
+ logger = logging.get_logger(__name__)
31
+
32
+
33
+ class Url(str):
34
+ pass
35
+
36
+
37
+ class EmptyDatasetError(FileNotFoundError):
38
+ pass
39
+
40
+
41
+ SPLIT_PATTERN_SHARDED = "data/{split}-[0-9][0-9][0-9][0-9][0-9]-of-[0-9][0-9][0-9][0-9][0-9]*.*"
42
+
43
+ SPLIT_KEYWORDS = {
44
+ Split.TRAIN: ["train", "training"],
45
+ Split.VALIDATION: ["validation", "valid", "dev", "val"],
46
+ Split.TEST: ["test", "testing", "eval", "evaluation"],
47
+ }
48
+ NON_WORDS_CHARS = "-._ 0-9"
49
+ if config.FSSPEC_VERSION < version.parse("2023.9.0"):
50
+ KEYWORDS_IN_FILENAME_BASE_PATTERNS = ["**[{sep}/]{keyword}[{sep}]*", "{keyword}[{sep}]*"]
51
+ KEYWORDS_IN_DIR_NAME_BASE_PATTERNS = [
52
+ "{keyword}/**",
53
+ "{keyword}[{sep}]*/**",
54
+ "**[{sep}/]{keyword}/**",
55
+ "**[{sep}/]{keyword}[{sep}]*/**",
56
+ ]
57
+ elif config.FSSPEC_VERSION < version.parse("2023.12.0"):
58
+ KEYWORDS_IN_FILENAME_BASE_PATTERNS = ["**/*[{sep}/]{keyword}[{sep}]*", "{keyword}[{sep}]*"]
59
+ KEYWORDS_IN_DIR_NAME_BASE_PATTERNS = [
60
+ "{keyword}/**/*",
61
+ "{keyword}[{sep}]*/**/*",
62
+ "**/*[{sep}/]{keyword}/**/*",
63
+ "**/*[{sep}/]{keyword}[{sep}]*/**/*",
64
+ ]
65
+ else:
66
+ KEYWORDS_IN_FILENAME_BASE_PATTERNS = ["**/{keyword}[{sep}]*", "**/*[{sep}]{keyword}[{sep}]*"]
67
+ KEYWORDS_IN_DIR_NAME_BASE_PATTERNS = [
68
+ "**/{keyword}/**",
69
+ "**/{keyword}[{sep}]*/**",
70
+ "**/*[{sep}]{keyword}/**",
71
+ "**/*[{sep}]{keyword}[{sep}]*/**",
72
+ ]
73
+
74
+ DEFAULT_SPLITS = [Split.TRAIN, Split.VALIDATION, Split.TEST]
75
+ DEFAULT_PATTERNS_SPLIT_IN_FILENAME = {
76
+ split: [
77
+ pattern.format(keyword=keyword, sep=NON_WORDS_CHARS)
78
+ for keyword in SPLIT_KEYWORDS[split]
79
+ for pattern in KEYWORDS_IN_FILENAME_BASE_PATTERNS
80
+ ]
81
+ for split in DEFAULT_SPLITS
82
+ }
83
+ DEFAULT_PATTERNS_SPLIT_IN_DIR_NAME = {
84
+ split: [
85
+ pattern.format(keyword=keyword, sep=NON_WORDS_CHARS)
86
+ for keyword in SPLIT_KEYWORDS[split]
87
+ for pattern in KEYWORDS_IN_DIR_NAME_BASE_PATTERNS
88
+ ]
89
+ for split in DEFAULT_SPLITS
90
+ }
91
+
92
+
93
+ DEFAULT_PATTERNS_ALL = {
94
+ Split.TRAIN: ["**"],
95
+ }
96
+
97
+ ALL_SPLIT_PATTERNS = [SPLIT_PATTERN_SHARDED]
98
+ ALL_DEFAULT_PATTERNS = [
99
+ DEFAULT_PATTERNS_SPLIT_IN_DIR_NAME,
100
+ DEFAULT_PATTERNS_SPLIT_IN_FILENAME,
101
+ DEFAULT_PATTERNS_ALL,
102
+ ]
103
+ WILDCARD_CHARACTERS = "*[]"
104
+ FILES_TO_IGNORE = [
105
+ "README.md",
106
+ "config.json",
107
+ "dataset_info.json",
108
+ "dataset_infos.json",
109
+ "dummy_data.zip",
110
+ "dataset_dict.json",
111
+ ]
112
+
113
+
114
+ def contains_wildcards(pattern: str) -> bool:
115
+ return any(wildcard_character in pattern for wildcard_character in WILDCARD_CHARACTERS)
116
+
117
+
118
+ def sanitize_patterns(patterns: Union[dict, list, str]) -> dict[str, Union[list[str], "DataFilesList"]]:
119
+ """
120
+ Take the data_files patterns from the user, and format them into a dictionary.
121
+ Each key is the name of the split, and each value is a list of data files patterns (paths or urls).
122
+ The default split is "train".
123
+
124
+ Returns:
125
+ patterns: dictionary of split_name -> list of patterns
126
+ """
127
+ if isinstance(patterns, dict):
128
+ return {str(key): value if isinstance(value, list) else [value] for key, value in patterns.items()}
129
+ elif isinstance(patterns, str):
130
+ return {SANITIZED_DEFAULT_SPLIT: [patterns]}
131
+ elif isinstance(patterns, list):
132
+ if any(isinstance(pattern, dict) for pattern in patterns):
133
+ for pattern in patterns:
134
+ if not (
135
+ isinstance(pattern, dict)
136
+ and len(pattern) == 2
137
+ and "split" in pattern
138
+ and isinstance(pattern.get("path"), (str, list))
139
+ ):
140
+ raise ValueError(
141
+ f"Expected each split to have a 'path' key which can be a string or a list of strings, but got {pattern}"
142
+ )
143
+ splits = [pattern["split"] for pattern in patterns]
144
+ if len(set(splits)) != len(splits):
145
+ raise ValueError(f"Some splits are duplicated in data_files: {splits}")
146
+ return {
147
+ str(pattern["split"]): pattern["path"] if isinstance(pattern["path"], list) else [pattern["path"]]
148
+ for pattern in patterns
149
+ }
150
+ else:
151
+ return {SANITIZED_DEFAULT_SPLIT: patterns}
152
+ else:
153
+ return sanitize_patterns(list(patterns))
154
+
155
+
156
+ def _is_inside_unrequested_special_dir(matched_rel_path: str, pattern: str) -> bool:
157
+ """
158
+ When a path matches a pattern, we additionally check if it's inside a special directory
159
+ we ignore by default (if it starts with a double underscore).
160
+
161
+ Users can still explicitly request a filepath inside such a directory if "__pycache__" is
162
+ mentioned explicitly in the requested pattern.
163
+
164
+ Some examples:
165
+
166
+ base directory:
167
+
168
+ ./
169
+ └── __pycache__
170
+ └── b.txt
171
+
172
+ >>> _is_inside_unrequested_special_dir("__pycache__/b.txt", "**")
173
+ True
174
+ >>> _is_inside_unrequested_special_dir("__pycache__/b.txt", "*/b.txt")
175
+ True
176
+ >>> _is_inside_unrequested_special_dir("__pycache__/b.txt", "__pycache__/*")
177
+ False
178
+ >>> _is_inside_unrequested_special_dir("__pycache__/b.txt", "__*/*")
179
+ False
180
+ """
181
+ # We just need to check if every special directories from the path is present explicitly in the pattern.
182
+ # Since we assume that the path matches the pattern, it's equivalent to counting that both
183
+ # the parent path and the parent pattern have the same number of special directories.
184
+ data_dirs_to_ignore_in_path = [part for part in PurePath(matched_rel_path).parent.parts if part.startswith("__")]
185
+ data_dirs_to_ignore_in_pattern = [part for part in PurePath(pattern).parent.parts if part.startswith("__")]
186
+ return len(data_dirs_to_ignore_in_path) != len(data_dirs_to_ignore_in_pattern)
187
+
188
+
189
+ def _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(matched_rel_path: str, pattern: str) -> bool:
190
+ """
191
+ When a path matches a pattern, we additionally check if it's a hidden file or if it's inside
192
+ a hidden directory we ignore by default, i.e. if the file name or a parent directory name starts with a dot.
193
+
194
+ Users can still explicitly request a filepath that is hidden or is inside a hidden directory
195
+ if the hidden part is mentioned explicitly in the requested pattern.
196
+
197
+ Some examples:
198
+
199
+ base directory:
200
+
201
+ ./
202
+ └── .hidden_file.txt
203
+
204
+ >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_file.txt", "**")
205
+ True
206
+ >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_file.txt", ".*")
207
+ False
208
+
209
+ base directory:
210
+
211
+ ./
212
+ └── .hidden_dir
213
+ └── a.txt
214
+
215
+ >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/a.txt", "**")
216
+ True
217
+ >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/a.txt", ".*/*")
218
+ False
219
+ >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/a.txt", ".hidden_dir/*")
220
+ False
221
+
222
+ base directory:
223
+
224
+ ./
225
+ └── .hidden_dir
226
+ └── .hidden_file.txt
227
+
228
+ >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/.hidden_file.txt", "**")
229
+ True
230
+ >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/.hidden_file.txt", ".*/*")
231
+ True
232
+ >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/.hidden_file.txt", ".*/.*")
233
+ False
234
+ >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/.hidden_file.txt", ".hidden_dir/*")
235
+ True
236
+ >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/.hidden_file.txt", ".hidden_dir/.*")
237
+ False
238
+ """
239
+ # We just need to check if every hidden part from the path is present explicitly in the pattern.
240
+ # Since we assume that the path matches the pattern, it's equivalent to counting that both
241
+ # the path and the pattern have the same number of hidden parts.
242
+ hidden_directories_in_path = [
243
+ part for part in PurePath(matched_rel_path).parts if part.startswith(".") and not set(part) == {"."}
244
+ ]
245
+ hidden_directories_in_pattern = [
246
+ part for part in PurePath(pattern).parts if part.startswith(".") and not set(part) == {"."}
247
+ ]
248
+ return len(hidden_directories_in_path) != len(hidden_directories_in_pattern)
249
+
250
+
251
+ def _get_data_files_patterns(pattern_resolver: Callable[[str], list[str]]) -> dict[str, list[str]]:
252
+ """
253
+ Get the default pattern from a directory or repository by testing all the supported patterns.
254
+ The first patterns to return a non-empty list of data files is returned.
255
+
256
+ In order, it first tests if SPLIT_PATTERN_SHARDED works, otherwise it tests the patterns in ALL_DEFAULT_PATTERNS.
257
+ """
258
+ # first check the split patterns like data/{split}-00000-of-00001.parquet
259
+ for split_pattern in ALL_SPLIT_PATTERNS:
260
+ pattern = split_pattern.replace("{split}", "*")
261
+ try:
262
+ data_files = pattern_resolver(pattern)
263
+ except FileNotFoundError:
264
+ continue
265
+ if len(data_files) > 0:
266
+ splits: set[str] = set()
267
+ for p in data_files:
268
+ p_parts = string_to_dict(xbasename(p), glob_pattern_to_regex(xbasename(split_pattern)))
269
+ assert p_parts is not None
270
+ splits.add(p_parts["split"])
271
+
272
+ if any(not re.match(_split_re, split) for split in splits):
273
+ raise ValueError(f"Split name should match '{_split_re}'' but got '{splits}'.")
274
+ sorted_splits = [str(split) for split in DEFAULT_SPLITS if split in splits] + sorted(
275
+ splits - {str(split) for split in DEFAULT_SPLITS}
276
+ )
277
+ return {split: [split_pattern.format(split=split)] for split in sorted_splits}
278
+ # then check the default patterns based on train/valid/test splits
279
+ for patterns_dict in ALL_DEFAULT_PATTERNS:
280
+ non_empty_splits = []
281
+ for split, patterns in patterns_dict.items():
282
+ for pattern in patterns:
283
+ try:
284
+ data_files = pattern_resolver(pattern)
285
+ except FileNotFoundError:
286
+ continue
287
+ if len(data_files) > 0:
288
+ non_empty_splits.append(split)
289
+ break
290
+ if non_empty_splits:
291
+ return {split: patterns_dict[split] for split in non_empty_splits}
292
+ raise FileNotFoundError(f"Couldn't resolve pattern {pattern} with resolver {pattern_resolver}")
293
+
294
+
295
+ def resolve_pattern(
296
+ pattern: str,
297
+ base_path: str,
298
+ allowed_extensions: Optional[list[str]] = None,
299
+ download_config: Optional[DownloadConfig] = None,
300
+ ) -> list[str]:
301
+ """
302
+ Resolve the paths and URLs of the data files from the pattern passed by the user.
303
+
304
+ You can use patterns to resolve multiple local files. Here are a few examples:
305
+ - *.csv to match all the CSV files at the first level
306
+ - **.csv to match all the CSV files at any level
307
+ - data/* to match all the files inside "data"
308
+ - data/** to match all the files inside "data" and its subdirectories
309
+
310
+ The patterns are resolved using the fsspec glob. In fsspec>=2023.12.0 this is equivalent to
311
+ Python's glob.glob, Path.glob, Path.match and fnmatch where ** is unsupported with a prefix/suffix
312
+ other than a forward slash /.
313
+
314
+ More generally:
315
+ - '*' matches any character except a forward-slash (to match just the file or directory name)
316
+ - '**' matches any character including a forward-slash /
317
+
318
+ Hidden files and directories (i.e. whose names start with a dot) are ignored, unless they are explicitly requested.
319
+ The same applies to special directories that start with a double underscore like "__pycache__".
320
+ You can still include one if the pattern explicitly mentions it:
321
+ - to include a hidden file: "*/.hidden.txt" or "*/.*"
322
+ - to include a hidden directory: ".hidden/*" or ".*/*"
323
+ - to include a special directory: "__special__/*" or "__*/*"
324
+
325
+ Example::
326
+
327
+ >>> from datasets.data_files import resolve_pattern
328
+ >>> base_path = "."
329
+ >>> resolve_pattern("docs/**/*.py", base_path)
330
+ [/Users/mariosasko/Desktop/projects/datasets/docs/source/_config.py']
331
+
332
+ Args:
333
+ pattern (str): Unix pattern or paths or URLs of the data files to resolve.
334
+ The paths can be absolute or relative to base_path.
335
+ Remote filesystems using fsspec are supported, e.g. with the hf:// protocol.
336
+ base_path (str): Base path to use when resolving relative paths.
337
+ allowed_extensions (Optional[list], optional): White-list of file extensions to use. Defaults to None (all extensions).
338
+ For example: allowed_extensions=[".csv", ".json", ".txt", ".parquet"]
339
+ download_config ([`DownloadConfig`], *optional*): Specific download configuration parameters.
340
+ Returns:
341
+ List[str]: List of paths or URLs to the local or remote files that match the patterns.
342
+ """
343
+ if is_relative_path(pattern):
344
+ pattern = xjoin(base_path, pattern)
345
+ elif is_local_path(pattern):
346
+ base_path = os.path.splitdrive(pattern)[0] + os.sep
347
+ else:
348
+ base_path = ""
349
+ pattern, storage_options = _prepare_path_and_storage_options(pattern, download_config=download_config)
350
+ fs, fs_pattern = url_to_fs(pattern, **storage_options)
351
+ files_to_ignore = set(FILES_TO_IGNORE) - {xbasename(pattern)}
352
+ protocol = fs.protocol if isinstance(fs.protocol, str) else fs.protocol[0]
353
+ protocol_prefix = protocol + "://" if protocol != "file" else ""
354
+ glob_kwargs = {}
355
+ if protocol == "hf" and config.HF_HUB_VERSION >= version.parse("0.20.0"):
356
+ # 10 times faster glob with detail=True (ignores costly info like lastCommit)
357
+ glob_kwargs["expand_info"] = False
358
+ matched_paths = [
359
+ filepath if filepath.startswith(protocol_prefix) else protocol_prefix + filepath
360
+ for filepath, info in fs.glob(pattern, detail=True, **glob_kwargs).items()
361
+ if (info["type"] == "file" or (info.get("islink") and os.path.isfile(os.path.realpath(filepath))))
362
+ and (xbasename(filepath) not in files_to_ignore)
363
+ and not _is_inside_unrequested_special_dir(filepath, fs_pattern)
364
+ and not _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(filepath, fs_pattern)
365
+ ] # ignore .ipynb and __pycache__, but keep /../
366
+ if allowed_extensions is not None:
367
+ out = [
368
+ filepath
369
+ for filepath in matched_paths
370
+ if any("." + suffix in allowed_extensions for suffix in xbasename(filepath).split(".")[1:])
371
+ ]
372
+ if len(out) < len(matched_paths):
373
+ invalid_matched_files = list(set(matched_paths) - set(out))
374
+ logger.info(
375
+ f"Some files matched the pattern '{pattern}' but don't have valid data file extensions: {invalid_matched_files}"
376
+ )
377
+ else:
378
+ out = matched_paths
379
+ if not out:
380
+ error_msg = f"Unable to find '{pattern}'"
381
+ if allowed_extensions is not None:
382
+ error_msg += f" with any supported extension {list(allowed_extensions)}"
383
+ raise FileNotFoundError(error_msg)
384
+ return out
385
+
386
+
387
+ def get_data_patterns(base_path: str, download_config: Optional[DownloadConfig] = None) -> dict[str, list[str]]:
388
+ """
389
+ Get the default pattern from a directory testing all the supported patterns.
390
+ The first patterns to return a non-empty list of data files is returned.
391
+
392
+ Some examples of supported patterns:
393
+
394
+ Input:
395
+
396
+ my_dataset_repository/
397
+ ├── README.md
398
+ └── dataset.csv
399
+
400
+ Output:
401
+
402
+ {'train': ['**']}
403
+
404
+ Input:
405
+
406
+ my_dataset_repository/
407
+ ├── README.md
408
+ ├── train.csv
409
+ └── test.csv
410
+
411
+ my_dataset_repository/
412
+ ├── README.md
413
+ └── data/
414
+ ├── train.csv
415
+ └── test.csv
416
+
417
+ my_dataset_repository/
418
+ ├── README.md
419
+ ├── train_0.csv
420
+ ├── train_1.csv
421
+ ├── train_2.csv
422
+ ├── train_3.csv
423
+ ├── test_0.csv
424
+ └── test_1.csv
425
+
426
+ Output:
427
+
428
+ {'train': ['**/train[-._ 0-9]*', '**/*[-._ 0-9]train[-._ 0-9]*', '**/training[-._ 0-9]*', '**/*[-._ 0-9]training[-._ 0-9]*'],
429
+ 'test': ['**/test[-._ 0-9]*', '**/*[-._ 0-9]test[-._ 0-9]*', '**/testing[-._ 0-9]*', '**/*[-._ 0-9]testing[-._ 0-9]*', ...]}
430
+
431
+ Input:
432
+
433
+ my_dataset_repository/
434
+ ├── README.md
435
+ └── data/
436
+ ├── train/
437
+ │ ├── shard_0.csv
438
+ │ ├── shard_1.csv
439
+ │ ├── shard_2.csv
440
+ │ └── shard_3.csv
441
+ └── test/
442
+ ├── shard_0.csv
443
+ └── shard_1.csv
444
+
445
+ Output:
446
+
447
+ {'train': ['**/train/**', '**/train[-._ 0-9]*/**', '**/*[-._ 0-9]train/**', '**/*[-._ 0-9]train[-._ 0-9]*/**', ...],
448
+ 'test': ['**/test/**', '**/test[-._ 0-9]*/**', '**/*[-._ 0-9]test/**', '**/*[-._ 0-9]test[-._ 0-9]*/**', ...]}
449
+
450
+ Input:
451
+
452
+ my_dataset_repository/
453
+ ├── README.md
454
+ └── data/
455
+ ├── train-00000-of-00003.csv
456
+ ├── train-00001-of-00003.csv
457
+ ├── train-00002-of-00003.csv
458
+ ├── test-00000-of-00001.csv
459
+ ├── random-00000-of-00003.csv
460
+ ├── random-00001-of-00003.csv
461
+ └── random-00002-of-00003.csv
462
+
463
+ Output:
464
+
465
+ {'train': ['data/train-[0-9][0-9][0-9][0-9][0-9]-of-[0-9][0-9][0-9][0-9][0-9]*.*'],
466
+ 'test': ['data/test-[0-9][0-9][0-9][0-9][0-9]-of-[0-9][0-9][0-9][0-9][0-9]*.*'],
467
+ 'random': ['data/random-[0-9][0-9][0-9][0-9][0-9]-of-[0-9][0-9][0-9][0-9][0-9]*.*']}
468
+
469
+ In order, it first tests if SPLIT_PATTERN_SHARDED works, otherwise it tests the patterns in ALL_DEFAULT_PATTERNS.
470
+ """
471
+ resolver = partial(resolve_pattern, base_path=base_path, download_config=download_config)
472
+ try:
473
+ return _get_data_files_patterns(resolver)
474
+ except FileNotFoundError:
475
+ raise EmptyDatasetError(f"The directory at {base_path} doesn't contain any data files") from None
476
+
477
+
478
+ def _get_single_origin_metadata(
479
+ data_file: str,
480
+ download_config: Optional[DownloadConfig] = None,
481
+ ) -> SingleOriginMetadata:
482
+ data_file, storage_options = _prepare_path_and_storage_options(data_file, download_config=download_config)
483
+ fs, *_ = url_to_fs(data_file, **storage_options)
484
+ if isinstance(fs, HfFileSystem):
485
+ resolved_path = fs.resolve_path(data_file)
486
+ return resolved_path.repo_id, resolved_path.revision
487
+ elif data_file.startswith(config.HF_ENDPOINT):
488
+ hffs = HfFileSystem(endpoint=config.HF_ENDPOINT, token=download_config.token)
489
+ data_file = "hf://" + data_file[len(config.HF_ENDPOINT) + 1 :].replace("/resolve/", "@", 1)
490
+ resolved_path = hffs.resolve_path(data_file)
491
+ return resolved_path.repo_id, resolved_path.revision
492
+ info = fs.info(data_file)
493
+ # s3fs uses "ETag", gcsfs uses "etag", and for local we simply check mtime
494
+ for key in ["ETag", "etag", "mtime"]:
495
+ if key in info:
496
+ return (str(info[key]),)
497
+ return ()
498
+
499
+
500
+ def _get_origin_metadata(
501
+ data_files: list[str],
502
+ download_config: Optional[DownloadConfig] = None,
503
+ max_workers: Optional[int] = None,
504
+ ) -> list[SingleOriginMetadata]:
505
+ max_workers = max_workers if max_workers is not None else config.HF_DATASETS_MULTITHREADING_MAX_WORKERS
506
+ return thread_map(
507
+ partial(_get_single_origin_metadata, download_config=download_config),
508
+ data_files,
509
+ max_workers=max_workers,
510
+ tqdm_class=hf_tqdm,
511
+ desc="Resolving data files",
512
+ # set `disable=None` rather than `disable=False` by default to disable progress bar when no TTY attached
513
+ disable=len(data_files) <= 16 or None,
514
+ )
515
+
516
+
517
+ class DataFilesList(list[str]):
518
+ """
519
+ List of data files (absolute local paths or URLs).
520
+ It has two construction methods given the user's data files patterns:
521
+ - ``from_hf_repo``: resolve patterns inside a dataset repository
522
+ - ``from_local_or_remote``: resolve patterns from a local path
523
+
524
+ Moreover, DataFilesList has an additional attribute ``origin_metadata``.
525
+ It can store:
526
+ - the last modified time of local files
527
+ - ETag of remote files
528
+ - commit sha of a dataset repository
529
+
530
+ Thanks to this additional attribute, it is possible to hash the list
531
+ and get a different hash if and only if at least one file changed.
532
+ This is useful for caching Dataset objects that are obtained from a list of data files.
533
+ """
534
+
535
+ def __init__(self, data_files: list[str], origin_metadata: list[SingleOriginMetadata]) -> None:
536
+ super().__init__(data_files)
537
+ self.origin_metadata = origin_metadata
538
+
539
+ def __add__(self, other: "DataFilesList") -> "DataFilesList":
540
+ return DataFilesList([*self, *other], self.origin_metadata + other.origin_metadata)
541
+
542
+ @classmethod
543
+ def from_hf_repo(
544
+ cls,
545
+ patterns: list[str],
546
+ dataset_info: huggingface_hub.hf_api.DatasetInfo,
547
+ base_path: Optional[str] = None,
548
+ allowed_extensions: Optional[list[str]] = None,
549
+ download_config: Optional[DownloadConfig] = None,
550
+ ) -> "DataFilesList":
551
+ base_path = f"hf://datasets/{dataset_info.id}@{dataset_info.sha}/{base_path or ''}".rstrip("/")
552
+ return cls.from_patterns(
553
+ patterns, base_path=base_path, allowed_extensions=allowed_extensions, download_config=download_config
554
+ )
555
+
556
+ @classmethod
557
+ def from_local_or_remote(
558
+ cls,
559
+ patterns: list[str],
560
+ base_path: Optional[str] = None,
561
+ allowed_extensions: Optional[list[str]] = None,
562
+ download_config: Optional[DownloadConfig] = None,
563
+ ) -> "DataFilesList":
564
+ base_path = base_path if base_path is not None else Path().resolve().as_posix()
565
+ return cls.from_patterns(
566
+ patterns, base_path=base_path, allowed_extensions=allowed_extensions, download_config=download_config
567
+ )
568
+
569
+ @classmethod
570
+ def from_patterns(
571
+ cls,
572
+ patterns: list[str],
573
+ base_path: Optional[str] = None,
574
+ allowed_extensions: Optional[list[str]] = None,
575
+ download_config: Optional[DownloadConfig] = None,
576
+ ) -> "DataFilesList":
577
+ base_path = base_path if base_path is not None else Path().resolve().as_posix()
578
+ data_files = []
579
+ for pattern in patterns:
580
+ try:
581
+ data_files.extend(
582
+ resolve_pattern(
583
+ pattern,
584
+ base_path=base_path,
585
+ allowed_extensions=allowed_extensions,
586
+ download_config=download_config,
587
+ )
588
+ )
589
+ except FileNotFoundError:
590
+ if not has_magic(pattern):
591
+ raise
592
+ origin_metadata = _get_origin_metadata(data_files, download_config=download_config)
593
+ return cls(data_files, origin_metadata)
594
+
595
+ def filter(
596
+ self, *, extensions: Optional[list[str]] = None, file_names: Optional[list[str]] = None
597
+ ) -> "DataFilesList":
598
+ patterns = []
599
+ if extensions:
600
+ ext_pattern = "|".join(re.escape(ext) for ext in extensions)
601
+ patterns.append(re.compile(f".*({ext_pattern})(\\..+)?$"))
602
+ if file_names:
603
+ fn_pattern = "|".join(re.escape(fn) for fn in file_names)
604
+ patterns.append(re.compile(rf".*[\/]?({fn_pattern})$"))
605
+ if patterns:
606
+ return DataFilesList(
607
+ [data_file for data_file in self if any(pattern.match(data_file) for pattern in patterns)],
608
+ origin_metadata=self.origin_metadata,
609
+ )
610
+ else:
611
+ return DataFilesList(list(self), origin_metadata=self.origin_metadata)
612
+
613
+
614
+ class DataFilesDict(dict[str, DataFilesList]):
615
+ """
616
+ Dict of split_name -> list of data files (absolute local paths or URLs).
617
+ It has two construction methods given the user's data files patterns :
618
+ - ``from_hf_repo``: resolve patterns inside a dataset repository
619
+ - ``from_local_or_remote``: resolve patterns from a local path
620
+
621
+ Moreover, each list is a DataFilesList. It is possible to hash the dictionary
622
+ and get a different hash if and only if at least one file changed.
623
+ For more info, see [`DataFilesList`].
624
+
625
+ This is useful for caching Dataset objects that are obtained from a list of data files.
626
+
627
+ Changing the order of the keys of this dictionary also doesn't change its hash.
628
+ """
629
+
630
+ @classmethod
631
+ def from_local_or_remote(
632
+ cls,
633
+ patterns: dict[str, Union[list[str], DataFilesList]],
634
+ base_path: Optional[str] = None,
635
+ allowed_extensions: Optional[list[str]] = None,
636
+ download_config: Optional[DownloadConfig] = None,
637
+ ) -> "DataFilesDict":
638
+ out = cls()
639
+ for key, patterns_for_key in patterns.items():
640
+ out[key] = (
641
+ patterns_for_key
642
+ if isinstance(patterns_for_key, DataFilesList)
643
+ else DataFilesList.from_local_or_remote(
644
+ patterns_for_key,
645
+ base_path=base_path,
646
+ allowed_extensions=allowed_extensions,
647
+ download_config=download_config,
648
+ )
649
+ )
650
+ return out
651
+
652
+ @classmethod
653
+ def from_hf_repo(
654
+ cls,
655
+ patterns: dict[str, Union[list[str], DataFilesList]],
656
+ dataset_info: huggingface_hub.hf_api.DatasetInfo,
657
+ base_path: Optional[str] = None,
658
+ allowed_extensions: Optional[list[str]] = None,
659
+ download_config: Optional[DownloadConfig] = None,
660
+ ) -> "DataFilesDict":
661
+ out = cls()
662
+ for key, patterns_for_key in patterns.items():
663
+ out[key] = (
664
+ patterns_for_key
665
+ if isinstance(patterns_for_key, DataFilesList)
666
+ else DataFilesList.from_hf_repo(
667
+ patterns_for_key,
668
+ dataset_info=dataset_info,
669
+ base_path=base_path,
670
+ allowed_extensions=allowed_extensions,
671
+ download_config=download_config,
672
+ )
673
+ )
674
+ return out
675
+
676
+ @classmethod
677
+ def from_patterns(
678
+ cls,
679
+ patterns: dict[str, Union[list[str], DataFilesList]],
680
+ base_path: Optional[str] = None,
681
+ allowed_extensions: Optional[list[str]] = None,
682
+ download_config: Optional[DownloadConfig] = None,
683
+ ) -> "DataFilesDict":
684
+ out = cls()
685
+ for key, patterns_for_key in patterns.items():
686
+ out[key] = (
687
+ patterns_for_key
688
+ if isinstance(patterns_for_key, DataFilesList)
689
+ else DataFilesList.from_patterns(
690
+ patterns_for_key,
691
+ base_path=base_path,
692
+ allowed_extensions=allowed_extensions,
693
+ download_config=download_config,
694
+ )
695
+ )
696
+ return out
697
+
698
+ def filter(
699
+ self, *, extensions: Optional[list[str]] = None, file_names: Optional[list[str]] = None
700
+ ) -> "DataFilesDict":
701
+ out = type(self)()
702
+ for key, data_files_list in self.items():
703
+ out[key] = data_files_list.filter(extensions=extensions, file_names=file_names)
704
+ return out
705
+
706
+
707
+ class DataFilesPatternsList(list[str]):
708
+ """
709
+ List of data files patterns (absolute local paths or URLs).
710
+ For each pattern there should also be a list of allowed extensions
711
+ to keep, or a None ot keep all the files for the pattern.
712
+ """
713
+
714
+ def __init__(
715
+ self,
716
+ patterns: list[str],
717
+ allowed_extensions: list[Optional[list[str]]],
718
+ ):
719
+ super().__init__(patterns)
720
+ self.allowed_extensions = allowed_extensions
721
+
722
+ def __add__(self, other):
723
+ return DataFilesList([*self, *other], self.allowed_extensions + other.allowed_extensions)
724
+
725
+ @classmethod
726
+ def from_patterns(
727
+ cls, patterns: list[str], allowed_extensions: Optional[list[str]] = None
728
+ ) -> "DataFilesPatternsList":
729
+ return cls(patterns, [allowed_extensions] * len(patterns))
730
+
731
+ def resolve(
732
+ self,
733
+ base_path: str,
734
+ download_config: Optional[DownloadConfig] = None,
735
+ ) -> "DataFilesList":
736
+ base_path = base_path if base_path is not None else Path().resolve().as_posix()
737
+ data_files = []
738
+ for pattern, allowed_extensions in zip(self, self.allowed_extensions):
739
+ try:
740
+ data_files.extend(
741
+ resolve_pattern(
742
+ pattern,
743
+ base_path=base_path,
744
+ allowed_extensions=allowed_extensions,
745
+ download_config=download_config,
746
+ )
747
+ )
748
+ except FileNotFoundError:
749
+ if not has_magic(pattern):
750
+ raise
751
+ origin_metadata = _get_origin_metadata(data_files, download_config=download_config)
752
+ return DataFilesList(data_files, origin_metadata)
753
+
754
+ def filter_extensions(self, extensions: list[str]) -> "DataFilesPatternsList":
755
+ return DataFilesPatternsList(
756
+ self, [allowed_extensions + extensions for allowed_extensions in self.allowed_extensions]
757
+ )
758
+
759
+
760
+ class DataFilesPatternsDict(dict[str, DataFilesPatternsList]):
761
+ """
762
+ Dict of split_name -> list of data files patterns (absolute local paths or URLs).
763
+ """
764
+
765
+ @classmethod
766
+ def from_patterns(
767
+ cls, patterns: dict[str, list[str]], allowed_extensions: Optional[list[str]] = None
768
+ ) -> "DataFilesPatternsDict":
769
+ out = cls()
770
+ for key, patterns_for_key in patterns.items():
771
+ out[key] = (
772
+ patterns_for_key
773
+ if isinstance(patterns_for_key, DataFilesPatternsList)
774
+ else DataFilesPatternsList.from_patterns(
775
+ patterns_for_key,
776
+ allowed_extensions=allowed_extensions,
777
+ )
778
+ )
779
+ return out
780
+
781
+ def resolve(
782
+ self,
783
+ base_path: str,
784
+ download_config: Optional[DownloadConfig] = None,
785
+ ) -> "DataFilesDict":
786
+ out = DataFilesDict()
787
+ for key, data_files_patterns_list in self.items():
788
+ out[key] = data_files_patterns_list.resolve(base_path, download_config)
789
+ return out
790
+
791
+ def filter_extensions(self, extensions: list[str]) -> "DataFilesPatternsDict":
792
+ out = type(self)()
793
+ for key, data_files_patterns_list in self.items():
794
+ out[key] = data_files_patterns_list.filter_extensions(extensions)
795
+ return out
lib/python3.12/site-packages/datasets/dataset_dict.py ADDED
The diff for this file is too large to render. See raw diff
 
lib/python3.12/site-packages/datasets/distributed.py ADDED
@@ -0,0 +1,39 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import TypeVar
2
+
3
+ from .arrow_dataset import Dataset, _split_by_node_map_style_dataset
4
+ from .iterable_dataset import IterableDataset, _split_by_node_iterable_dataset
5
+
6
+
7
+ DatasetType = TypeVar("DatasetType", Dataset, IterableDataset)
8
+
9
+
10
+ def split_dataset_by_node(dataset: DatasetType, rank: int, world_size: int) -> DatasetType:
11
+ """
12
+ Split a dataset for the node at rank `rank` in a pool of nodes of size `world_size`.
13
+
14
+ For map-style datasets:
15
+
16
+ Each node is assigned a chunk of data, e.g. rank 0 is given the first chunk of the dataset.
17
+ To maximize data loading throughput, chunks are made of contiguous data on disk if possible.
18
+
19
+ For iterable datasets:
20
+
21
+ If the dataset has a number of shards that is a factor of `world_size` (i.e. if `dataset.num_shards % world_size == 0`),
22
+ then the shards are evenly assigned across the nodes, which is the most optimized.
23
+ Otherwise, each node keeps 1 example out of `world_size`, skipping the other examples.
24
+
25
+ Args:
26
+ dataset ([`Dataset`] or [`IterableDataset`]):
27
+ The dataset to split by node.
28
+ rank (`int`):
29
+ Rank of the current node.
30
+ world_size (`int`):
31
+ Total number of nodes.
32
+
33
+ Returns:
34
+ [`Dataset`] or [`IterableDataset`]: The dataset to be used on the node at rank `rank`.
35
+ """
36
+ if isinstance(dataset, Dataset):
37
+ return _split_by_node_map_style_dataset(dataset, rank=rank, world_size=world_size)
38
+ else:
39
+ return _split_by_node_iterable_dataset(dataset, rank=rank, world_size=world_size)
lib/python3.12/site-packages/datasets/exceptions.py ADDED
@@ -0,0 +1,119 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ # Copyright 2023 The HuggingFace Authors.
3
+ from typing import Any, Optional, Union
4
+
5
+ from huggingface_hub import HfFileSystem
6
+
7
+ from . import config
8
+ from .table import CastError
9
+ from .utils.track import TrackedIterableFromGenerator, tracked_list, tracked_str
10
+
11
+
12
+ class DatasetsError(Exception):
13
+ """Base class for exceptions in this library."""
14
+
15
+
16
+ class DefunctDatasetError(DatasetsError):
17
+ """The dataset has been defunct."""
18
+
19
+
20
+ class FileNotFoundDatasetsError(DatasetsError, FileNotFoundError):
21
+ """FileNotFoundError raised by this library."""
22
+
23
+
24
+ class DataFilesNotFoundError(FileNotFoundDatasetsError):
25
+ """No (supported) data files found."""
26
+
27
+
28
+ class DatasetNotFoundError(FileNotFoundDatasetsError):
29
+ """Dataset not found.
30
+
31
+ Raised when trying to access:
32
+ - a missing dataset, or
33
+ - a private/gated dataset and the user is not authenticated.
34
+ """
35
+
36
+
37
+ class DatasetBuildError(DatasetsError):
38
+ pass
39
+
40
+
41
+ class ManualDownloadError(DatasetBuildError):
42
+ pass
43
+
44
+
45
+ class FileFormatError(DatasetBuildError):
46
+ pass
47
+
48
+
49
+ class DatasetGenerationError(DatasetBuildError):
50
+ pass
51
+
52
+
53
+ class DatasetGenerationCastError(DatasetGenerationError):
54
+ @classmethod
55
+ def from_cast_error(
56
+ cls,
57
+ cast_error: CastError,
58
+ builder_name: str,
59
+ gen_kwargs: dict[str, Any],
60
+ token: Optional[Union[bool, str]],
61
+ ) -> "DatasetGenerationCastError":
62
+ explanation_message = (
63
+ f"\n\nAll the data files must have the same columns, but at some point {cast_error.details()}"
64
+ )
65
+ formatted_tracked_gen_kwargs: list[str] = []
66
+ for gen_kwarg in gen_kwargs.values():
67
+ if not isinstance(gen_kwarg, (tracked_str, tracked_list, TrackedIterableFromGenerator)):
68
+ continue
69
+ while (
70
+ isinstance(gen_kwarg, (tracked_list, TrackedIterableFromGenerator)) and gen_kwarg.last_item is not None
71
+ ):
72
+ gen_kwarg = gen_kwarg.last_item
73
+ if isinstance(gen_kwarg, tracked_str):
74
+ gen_kwarg = gen_kwarg.get_origin()
75
+ if isinstance(gen_kwarg, str) and gen_kwarg.startswith("hf://"):
76
+ resolved_path = HfFileSystem(endpoint=config.HF_ENDPOINT, token=token).resolve_path(gen_kwarg)
77
+ gen_kwarg = "hf://" + resolved_path.unresolve()
78
+ if "@" + resolved_path.revision in gen_kwarg:
79
+ gen_kwarg = (
80
+ gen_kwarg.replace("@" + resolved_path.revision, "", 1)
81
+ + f" (at revision {resolved_path.revision})"
82
+ )
83
+ formatted_tracked_gen_kwargs.append(str(gen_kwarg))
84
+ if formatted_tracked_gen_kwargs:
85
+ explanation_message += f"\n\nThis happened while the {builder_name} dataset builder was generating data using\n\n{', '.join(formatted_tracked_gen_kwargs)}"
86
+ help_message = "\n\nPlease either edit the data files to have matching columns, or separate them into different configurations (see docs at https://hf.co/docs/hub/datasets-manual-configuration#multiple-configurations)"
87
+ return cls("An error occurred while generating the dataset" + explanation_message + help_message)
88
+
89
+
90
+ class ChecksumVerificationError(DatasetsError):
91
+ """Error raised during checksums verifications of downloaded files."""
92
+
93
+
94
+ class UnexpectedDownloadedFileError(ChecksumVerificationError):
95
+ """Some downloaded files were not expected."""
96
+
97
+
98
+ class ExpectedMoreDownloadedFilesError(ChecksumVerificationError):
99
+ """Some files were supposed to be downloaded but were not."""
100
+
101
+
102
+ class NonMatchingChecksumError(ChecksumVerificationError):
103
+ """The downloaded file checksum don't match the expected checksum."""
104
+
105
+
106
+ class SplitsVerificationError(DatasetsError):
107
+ """Error raised during splits verifications."""
108
+
109
+
110
+ class UnexpectedSplitsError(SplitsVerificationError):
111
+ """The expected splits of the downloaded file is missing."""
112
+
113
+
114
+ class ExpectedMoreSplitsError(SplitsVerificationError):
115
+ """Some recorded splits are missing."""
116
+
117
+
118
+ class NonMatchingSplitsSizesError(SplitsVerificationError):
119
+ """The splits sizes don't match the expected splits sizes."""
lib/python3.12/site-packages/datasets/fingerprint.py ADDED
@@ -0,0 +1,454 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import inspect
2
+ import os
3
+ import random
4
+ import shutil
5
+ import tempfile
6
+ import weakref
7
+ from functools import wraps
8
+ from pathlib import Path
9
+ from typing import TYPE_CHECKING, Any, Callable, Optional, Union
10
+
11
+ import numpy as np
12
+ import xxhash
13
+
14
+ from . import config
15
+ from .naming import INVALID_WINDOWS_CHARACTERS_IN_PATH
16
+ from .utils._dill import dumps
17
+ from .utils.logging import get_logger
18
+
19
+
20
+ if TYPE_CHECKING:
21
+ from .arrow_dataset import Dataset
22
+
23
+
24
+ logger = get_logger(__name__)
25
+
26
+
27
+ # Fingerprinting allows to have one deterministic fingerprint per dataset state.
28
+ # A dataset fingerprint is updated after each transform.
29
+ # Re-running the same transforms on a dataset in a different session results in the same fingerprint.
30
+ # This is possible thanks to a custom hashing function that works with most python objects.
31
+
32
+ # Fingerprinting is the main mechanism that enables caching.
33
+ # The caching mechanism allows to reload an existing cache file if it's already been computed.
34
+
35
+
36
+ #################
37
+ # Caching
38
+ #################
39
+
40
+ _CACHING_ENABLED = True
41
+ _TEMP_DIR_FOR_TEMP_CACHE_FILES: Optional["_TempCacheDir"] = None
42
+ _DATASETS_WITH_TABLE_IN_TEMP_DIR: Optional[weakref.WeakSet] = None
43
+
44
+
45
+ class _TempCacheDir:
46
+ """
47
+ A temporary directory for storing cached Arrow files with a cleanup that frees references to the Arrow files
48
+ before deleting the directory itself to avoid permission errors on Windows.
49
+ """
50
+
51
+ def __init__(self):
52
+ self.name = tempfile.mkdtemp(prefix=config.TEMP_CACHE_DIR_PREFIX)
53
+ self._finalizer = weakref.finalize(self, self._cleanup)
54
+
55
+ def _cleanup(self):
56
+ for dset in get_datasets_with_cache_file_in_temp_dir():
57
+ dset.__del__()
58
+ if os.path.exists(self.name):
59
+ try:
60
+ shutil.rmtree(self.name)
61
+ except Exception as e:
62
+ raise OSError(
63
+ f"An error occured while trying to delete temporary cache directory {self.name}. Please delete it manually."
64
+ ) from e
65
+
66
+ def cleanup(self):
67
+ if self._finalizer.detach():
68
+ self._cleanup()
69
+
70
+
71
+ def maybe_register_dataset_for_temp_dir_deletion(dataset):
72
+ """
73
+ This function registers the datasets that have cache files in _TEMP_DIR_FOR_TEMP_CACHE_FILES in order
74
+ to properly delete them before deleting the temporary directory.
75
+ The temporary directory _TEMP_DIR_FOR_TEMP_CACHE_FILES is used when caching is disabled.
76
+ """
77
+ if _TEMP_DIR_FOR_TEMP_CACHE_FILES is None:
78
+ return
79
+
80
+ global _DATASETS_WITH_TABLE_IN_TEMP_DIR
81
+ if _DATASETS_WITH_TABLE_IN_TEMP_DIR is None:
82
+ _DATASETS_WITH_TABLE_IN_TEMP_DIR = weakref.WeakSet()
83
+ if any(
84
+ Path(_TEMP_DIR_FOR_TEMP_CACHE_FILES.name) in Path(cache_file["filename"]).parents
85
+ for cache_file in dataset.cache_files
86
+ ):
87
+ _DATASETS_WITH_TABLE_IN_TEMP_DIR.add(dataset)
88
+
89
+
90
+ def get_datasets_with_cache_file_in_temp_dir():
91
+ return list(_DATASETS_WITH_TABLE_IN_TEMP_DIR) if _DATASETS_WITH_TABLE_IN_TEMP_DIR is not None else []
92
+
93
+
94
+ def enable_caching():
95
+ """
96
+ When applying transforms on a dataset, the data are stored in cache files.
97
+ The caching mechanism allows to reload an existing cache file if it's already been computed.
98
+
99
+ Reloading a dataset is possible since the cache files are named using the dataset fingerprint, which is updated
100
+ after each transform.
101
+
102
+ If disabled, the library will no longer reload cached datasets files when applying transforms to the datasets.
103
+ More precisely, if the caching is disabled:
104
+ - cache files are always recreated
105
+ - cache files are written to a temporary directory that is deleted when session closes
106
+ - cache files are named using a random hash instead of the dataset fingerprint
107
+ - use [`~datasets.Dataset.save_to_disk`] to save a transformed dataset or it will be deleted when session closes
108
+ - caching doesn't affect [`~datasets.load_dataset`]. If you want to regenerate a dataset from scratch you should use
109
+ the `download_mode` parameter in [`~datasets.load_dataset`].
110
+ """
111
+ global _CACHING_ENABLED
112
+ _CACHING_ENABLED = True
113
+
114
+
115
+ def disable_caching():
116
+ """
117
+ When applying transforms on a dataset, the data are stored in cache files.
118
+ The caching mechanism allows to reload an existing cache file if it's already been computed.
119
+
120
+ Reloading a dataset is possible since the cache files are named using the dataset fingerprint, which is updated
121
+ after each transform.
122
+
123
+ If disabled, the library will no longer reload cached datasets files when applying transforms to the datasets.
124
+ More precisely, if the caching is disabled:
125
+ - cache files are always recreated
126
+ - cache files are written to a temporary directory that is deleted when session closes
127
+ - cache files are named using a random hash instead of the dataset fingerprint
128
+ - use [`~datasets.Dataset.save_to_disk`] to save a transformed dataset or it will be deleted when session closes
129
+ - caching doesn't affect [`~datasets.load_dataset`]. If you want to regenerate a dataset from scratch you should use
130
+ the `download_mode` parameter in [`~datasets.load_dataset`].
131
+ """
132
+ global _CACHING_ENABLED
133
+ _CACHING_ENABLED = False
134
+
135
+
136
+ def is_caching_enabled() -> bool:
137
+ """
138
+ When applying transforms on a dataset, the data are stored in cache files.
139
+ The caching mechanism allows to reload an existing cache file if it's already been computed.
140
+
141
+ Reloading a dataset is possible since the cache files are named using the dataset fingerprint, which is updated
142
+ after each transform.
143
+
144
+ If disabled, the library will no longer reload cached datasets files when applying transforms to the datasets.
145
+ More precisely, if the caching is disabled:
146
+ - cache files are always recreated
147
+ - cache files are written to a temporary directory that is deleted when session closes
148
+ - cache files are named using a random hash instead of the dataset fingerprint
149
+ - use [`~datasets.Dataset.save_to_disk`]] to save a transformed dataset or it will be deleted when session closes
150
+ - caching doesn't affect [`~datasets.load_dataset`]. If you want to regenerate a dataset from scratch you should use
151
+ the `download_mode` parameter in [`~datasets.load_dataset`].
152
+ """
153
+ global _CACHING_ENABLED
154
+ return bool(_CACHING_ENABLED)
155
+
156
+
157
+ def get_temporary_cache_files_directory() -> str:
158
+ """Return a directory that is deleted when session closes."""
159
+ global _TEMP_DIR_FOR_TEMP_CACHE_FILES
160
+ if _TEMP_DIR_FOR_TEMP_CACHE_FILES is None:
161
+ _TEMP_DIR_FOR_TEMP_CACHE_FILES = _TempCacheDir()
162
+ return _TEMP_DIR_FOR_TEMP_CACHE_FILES.name
163
+
164
+
165
+ #################
166
+ # Hashing
167
+ #################
168
+
169
+
170
+ class Hasher:
171
+ """Hasher that accepts python objects as inputs."""
172
+
173
+ dispatch: dict = {}
174
+
175
+ def __init__(self):
176
+ self.m = xxhash.xxh64()
177
+
178
+ @classmethod
179
+ def hash_bytes(cls, value: Union[bytes, list[bytes]]) -> str:
180
+ value = [value] if isinstance(value, bytes) else value
181
+ m = xxhash.xxh64()
182
+ for x in value:
183
+ m.update(x)
184
+ return m.hexdigest()
185
+
186
+ @classmethod
187
+ def hash(cls, value: Any) -> str:
188
+ return cls.hash_bytes(dumps(value))
189
+
190
+ def update(self, value: Any) -> None:
191
+ header_for_update = f"=={type(value)}=="
192
+ value_for_update = self.hash(value)
193
+ self.m.update(header_for_update.encode("utf8"))
194
+ self.m.update(value_for_update.encode("utf-8"))
195
+
196
+ def hexdigest(self) -> str:
197
+ return self.m.hexdigest()
198
+
199
+
200
+ #################
201
+ # Fingerprinting
202
+ #################
203
+
204
+ fingerprint_rng = random.Random()
205
+ # we show a warning only once when fingerprinting fails to avoid spam
206
+ fingerprint_warnings: dict[str, bool] = {}
207
+
208
+
209
+ def generate_fingerprint(dataset: "Dataset") -> str:
210
+ state = dataset.__dict__
211
+ hasher = Hasher()
212
+ for key in sorted(state):
213
+ if key == "_fingerprint":
214
+ continue
215
+ hasher.update(key)
216
+ hasher.update(state[key])
217
+ # hash data files last modification timestamps as well
218
+ for cache_file in dataset.cache_files:
219
+ hasher.update(os.path.getmtime(cache_file["filename"]))
220
+ return hasher.hexdigest()
221
+
222
+
223
+ def generate_random_fingerprint(nbits: int = 64) -> str:
224
+ return f"{fingerprint_rng.getrandbits(nbits):0{nbits // 4}x}"
225
+
226
+
227
+ def update_fingerprint(fingerprint, transform, transform_args):
228
+ global fingerprint_warnings
229
+ hasher = Hasher()
230
+ hasher.update(fingerprint)
231
+ try:
232
+ hasher.update(transform)
233
+ except: # noqa various errors might raise here from pickle or dill
234
+ if _CACHING_ENABLED:
235
+ if not fingerprint_warnings.get("update_fingerprint_transform_hash_failed", False):
236
+ logger.warning(
237
+ f"Transform {transform} couldn't be hashed properly, a random hash was used instead. "
238
+ "Make sure your transforms and parameters are serializable with pickle or dill for the dataset fingerprinting and caching to work. "
239
+ "If you reuse this transform, the caching mechanism will consider it to be different from the previous calls and recompute everything. "
240
+ "This warning is only showed once. Subsequent hashing failures won't be showed."
241
+ )
242
+ fingerprint_warnings["update_fingerprint_transform_hash_failed"] = True
243
+ else:
244
+ logger.info(f"Transform {transform} couldn't be hashed properly, a random hash was used instead.")
245
+ else:
246
+ logger.info(
247
+ f"Transform {transform} couldn't be hashed properly, a random hash was used instead. This doesn't affect caching since it's disabled."
248
+ )
249
+
250
+ return generate_random_fingerprint()
251
+ for key in sorted(transform_args):
252
+ hasher.update(key)
253
+ try:
254
+ hasher.update(transform_args[key])
255
+ except: # noqa various errors might raise here from pickle or dill
256
+ if _CACHING_ENABLED:
257
+ if not fingerprint_warnings.get("update_fingerprint_transform_hash_failed", False):
258
+ logger.warning(
259
+ f"Parameter '{key}'={transform_args[key]} of the transform {transform} couldn't be hashed properly, a random hash was used instead. "
260
+ "Make sure your transforms and parameters are serializable with pickle or dill for the dataset fingerprinting and caching to work. "
261
+ "If you reuse this transform, the caching mechanism will consider it to be different from the previous calls and recompute everything. "
262
+ "This warning is only showed once. Subsequent hashing failures won't be showed."
263
+ )
264
+ fingerprint_warnings["update_fingerprint_transform_hash_failed"] = True
265
+ else:
266
+ logger.info(
267
+ f"Parameter '{key}'={transform_args[key]} of the transform {transform} couldn't be hashed properly, a random hash was used instead."
268
+ )
269
+ else:
270
+ logger.info(
271
+ f"Parameter '{key}'={transform_args[key]} of the transform {transform} couldn't be hashed properly, a random hash was used instead. This doesn't affect caching since it's disabled."
272
+ )
273
+ return generate_random_fingerprint()
274
+ return hasher.hexdigest()
275
+
276
+
277
+ def validate_fingerprint(fingerprint: str, max_length=64):
278
+ """
279
+ Make sure the fingerprint is a non-empty string that is not longer that max_length=64 by default,
280
+ so that the fingerprint can be used to name cache files without issues.
281
+ """
282
+ if not isinstance(fingerprint, str) or not fingerprint:
283
+ raise ValueError(f"Invalid fingerprint '{fingerprint}': it should be a non-empty string.")
284
+ for invalid_char in INVALID_WINDOWS_CHARACTERS_IN_PATH:
285
+ if invalid_char in fingerprint:
286
+ raise ValueError(
287
+ f"Invalid fingerprint. Bad characters from black list '{INVALID_WINDOWS_CHARACTERS_IN_PATH}' found in '{fingerprint}'. "
288
+ f"They could create issues when creating cache files."
289
+ )
290
+ if len(fingerprint) > max_length:
291
+ raise ValueError(
292
+ f"Invalid fingerprint. Maximum lenth is {max_length} but '{fingerprint}' has length {len(fingerprint)}."
293
+ "It could create issues when creating cache files."
294
+ )
295
+
296
+
297
+ def format_transform_for_fingerprint(func: Callable, version: Optional[str] = None) -> str:
298
+ """
299
+ Format a transform to the format that will be used to update the fingerprint.
300
+ """
301
+ transform = f"{func.__module__}.{func.__qualname__}"
302
+ if version is not None:
303
+ transform += f"@{version}"
304
+ return transform
305
+
306
+
307
+ def format_kwargs_for_fingerprint(
308
+ func: Callable,
309
+ args: tuple,
310
+ kwargs: dict[str, Any],
311
+ use_kwargs: Optional[list[str]] = None,
312
+ ignore_kwargs: Optional[list[str]] = None,
313
+ randomized_function: bool = False,
314
+ ) -> dict[str, Any]:
315
+ """
316
+ Format the kwargs of a transform to the format that will be used to update the fingerprint.
317
+ """
318
+ kwargs_for_fingerprint = kwargs.copy()
319
+ if args:
320
+ params = [p.name for p in inspect.signature(func).parameters.values() if p != p.VAR_KEYWORD]
321
+ args = args[1:] # assume the first argument is the dataset
322
+ params = params[1:]
323
+ kwargs_for_fingerprint.update(zip(params, args))
324
+ else:
325
+ del kwargs_for_fingerprint[
326
+ next(iter(inspect.signature(func).parameters))
327
+ ] # assume the first key is the dataset
328
+
329
+ # keep the right kwargs to be hashed to generate the fingerprint
330
+
331
+ if use_kwargs:
332
+ kwargs_for_fingerprint = {k: v for k, v in kwargs_for_fingerprint.items() if k in use_kwargs}
333
+ if ignore_kwargs:
334
+ kwargs_for_fingerprint = {k: v for k, v in kwargs_for_fingerprint.items() if k not in ignore_kwargs}
335
+ if randomized_function: # randomized functions have `seed` and `generator` parameters
336
+ if kwargs_for_fingerprint.get("seed") is None and kwargs_for_fingerprint.get("generator") is None:
337
+ _, seed, pos, *_ = np.random.get_state()
338
+ seed = seed[pos] if pos < 624 else seed[0]
339
+ kwargs_for_fingerprint["generator"] = np.random.default_rng(seed)
340
+
341
+ # remove kwargs that are the default values
342
+
343
+ default_values = {
344
+ p.name: p.default for p in inspect.signature(func).parameters.values() if p.default != inspect._empty
345
+ }
346
+ for default_varname, default_value in default_values.items():
347
+ if default_varname in kwargs_for_fingerprint and kwargs_for_fingerprint[default_varname] == default_value:
348
+ kwargs_for_fingerprint.pop(default_varname)
349
+ return kwargs_for_fingerprint
350
+
351
+
352
+ def fingerprint_transform(
353
+ inplace: bool,
354
+ use_kwargs: Optional[list[str]] = None,
355
+ ignore_kwargs: Optional[list[str]] = None,
356
+ fingerprint_names: Optional[list[str]] = None,
357
+ randomized_function: bool = False,
358
+ version: Optional[str] = None,
359
+ ):
360
+ """
361
+ Wrapper for dataset transforms to update the dataset fingerprint using ``update_fingerprint``
362
+ Args:
363
+ inplace (:obj:`bool`): If inplace is True, the fingerprint of the dataset is updated inplace.
364
+ Otherwise, a parameter "new_fingerprint" is passed to the wrapped method that should take care of
365
+ setting the fingerprint of the returned Dataset.
366
+ use_kwargs (:obj:`List[str]`, optional): optional white list of argument names to take into account
367
+ to update the fingerprint to the wrapped method that should take care of
368
+ setting the fingerprint of the returned Dataset. By default all the arguments are used.
369
+ ignore_kwargs (:obj:`List[str]`, optional): optional black list of argument names to take into account
370
+ to update the fingerprint. Note that ignore_kwargs prevails on use_kwargs.
371
+ fingerprint_names (:obj:`List[str]`, optional, defaults to ["new_fingerprint"]):
372
+ If the dataset transforms is not inplace and returns a DatasetDict, then it can require
373
+ several fingerprints (one per dataset in the DatasetDict). By specifying fingerprint_names,
374
+ one fingerprint named after each element of fingerprint_names is going to be passed.
375
+ randomized_function (:obj:`bool`, defaults to False): If the dataset transform is random and has
376
+ optional parameters "seed" and "generator", then you can set randomized_function to True.
377
+ This way, even if users set "seed" and "generator" to None, then the fingerprint is
378
+ going to be randomly generated depending on numpy's current state. In this case, the
379
+ generator is set to np.random.default_rng(np.random.get_state()[1][0]).
380
+ version (:obj:`str`, optional): version of the transform. The version is taken into account when
381
+ computing the fingerprint. If a datase transform changes (or at least if the output data
382
+ that are cached changes), then one should increase the version. If the version stays the
383
+ same, then old cached data could be reused that are not compatible with the new transform.
384
+ It should be in the format "MAJOR.MINOR.PATCH".
385
+ """
386
+
387
+ if use_kwargs is not None and not isinstance(use_kwargs, list):
388
+ raise ValueError(f"use_kwargs is supposed to be a list, not {type(use_kwargs)}")
389
+
390
+ if ignore_kwargs is not None and not isinstance(ignore_kwargs, list):
391
+ raise ValueError(f"ignore_kwargs is supposed to be a list, not {type(use_kwargs)}")
392
+
393
+ if inplace and fingerprint_names:
394
+ raise ValueError("fingerprint_names are only used when inplace is False")
395
+
396
+ fingerprint_names = fingerprint_names if fingerprint_names is not None else ["new_fingerprint"]
397
+
398
+ def _fingerprint(func):
399
+ if not inplace and not all(name in func.__code__.co_varnames for name in fingerprint_names):
400
+ raise ValueError(f"function {func} is missing parameters {fingerprint_names} in signature")
401
+
402
+ if randomized_function: # randomized function have seed and generator parameters
403
+ if "seed" not in func.__code__.co_varnames:
404
+ raise ValueError(f"'seed' must be in {func}'s signature")
405
+ if "generator" not in func.__code__.co_varnames:
406
+ raise ValueError(f"'generator' must be in {func}'s signature")
407
+ # this call has to be outside the wrapper or since __qualname__ changes in multiprocessing
408
+ transform = format_transform_for_fingerprint(func, version=version)
409
+
410
+ @wraps(func)
411
+ def wrapper(*args, **kwargs):
412
+ kwargs_for_fingerprint = format_kwargs_for_fingerprint(
413
+ func,
414
+ args,
415
+ kwargs,
416
+ use_kwargs=use_kwargs,
417
+ ignore_kwargs=ignore_kwargs,
418
+ randomized_function=randomized_function,
419
+ )
420
+
421
+ if args:
422
+ dataset: Dataset = args[0]
423
+ args = args[1:]
424
+ else:
425
+ dataset: Dataset = kwargs.pop(next(iter(inspect.signature(func).parameters)))
426
+
427
+ # compute new_fingerprint and add it to the args of not in-place transforms
428
+ if inplace:
429
+ new_fingerprint = update_fingerprint(dataset._fingerprint, transform, kwargs_for_fingerprint)
430
+ else:
431
+ for fingerprint_name in fingerprint_names: # transforms like `train_test_split` have several hashes
432
+ if kwargs.get(fingerprint_name) is None:
433
+ kwargs_for_fingerprint["fingerprint_name"] = fingerprint_name
434
+ kwargs[fingerprint_name] = update_fingerprint(
435
+ dataset._fingerprint, transform, kwargs_for_fingerprint
436
+ )
437
+ else:
438
+ validate_fingerprint(kwargs[fingerprint_name])
439
+
440
+ # Call actual function
441
+
442
+ out = func(dataset, *args, **kwargs)
443
+
444
+ # Update fingerprint of in-place transforms + update in-place history of transforms
445
+
446
+ if inplace: # update after calling func so that the fingerprint doesn't change if the function fails
447
+ dataset._fingerprint = new_fingerprint
448
+
449
+ return out
450
+
451
+ wrapper._decorator_name_ = "fingerprint"
452
+ return wrapper
453
+
454
+ return _fingerprint
lib/python3.12/site-packages/datasets/hub.py ADDED
@@ -0,0 +1,230 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import time
2
+ from itertools import chain
3
+ from typing import Optional, Union
4
+
5
+ from huggingface_hub import (
6
+ CommitInfo,
7
+ CommitOperationAdd,
8
+ CommitOperationDelete,
9
+ DatasetCard,
10
+ DatasetCardData,
11
+ HfApi,
12
+ HfFileSystem,
13
+ )
14
+ from huggingface_hub.utils import HfHubHTTPError
15
+
16
+ import datasets.config
17
+ from datasets.info import DatasetInfosDict
18
+ from datasets.inspect import get_dataset_config_names, get_dataset_default_config_name
19
+ from datasets.load import load_dataset, load_dataset_builder
20
+ from datasets.utils.metadata import MetadataConfigs
21
+
22
+
23
+ def convert_to_parquet(
24
+ repo_id: str,
25
+ revision: Optional[str] = None,
26
+ token: Optional[Union[bool, str]] = None,
27
+ trust_remote_code: Optional[bool] = None,
28
+ ) -> CommitInfo:
29
+ """Convert Hub [script-based dataset](dataset_script) to Parquet [data-only dataset](repository_structure), so that
30
+ the dataset viewer will be supported.
31
+
32
+ This function:
33
+ - makes a copy of the script on the "main" branch into a dedicated branch called "script" (if it does not already exist)
34
+ - creates a pull request to the Hub dataset to convert it to Parquet files (and deletes the script from the main branch)
35
+
36
+ If in the future you need to recreate the Parquet files from the "script" branch, pass the `revision="script"` argument.
37
+
38
+ Note that you should pass the `trust_remote_code=True` argument only if you trust the remote code to be executed locally on your machine.
39
+
40
+ Args:
41
+ repo_id (`str`): ID of the source Hub dataset repository, in the following format: `<user>/<dataset_name>` or
42
+ `<org>/<dataset_name>`.
43
+ revision (`str`, *optional*): Branch of the source Hub dataset repository. Defaults to the `"main"` branch.
44
+ token (`bool` or `str`, *optional*): Authentication token for the Hugging Face Hub.
45
+ trust_remote_code (`bool`, defaults to `False`): Whether you trust the remote code of the Hub script-based
46
+ dataset to be executed locally on your machine. This option should only be set to `True` for repositories
47
+ where you have read the code and which you trust.
48
+
49
+ <Changed version="2.20.0">
50
+
51
+ `trust_remote_code` defaults to `False` if not specified.
52
+
53
+ </Changed>
54
+
55
+ Returns:
56
+ `huggingface_hub.CommitInfo`
57
+ """
58
+ print(f"{repo_id}")
59
+ configs = get_dataset_config_names(repo_id, token=token, revision=revision, trust_remote_code=trust_remote_code)
60
+ print(f"{configs = }")
61
+ default_config = get_dataset_default_config_name(
62
+ repo_id, token=token, revision=revision, trust_remote_code=trust_remote_code
63
+ )
64
+ print(f"{default_config = }")
65
+ if default_config:
66
+ config = default_config
67
+ configs.remove(default_config)
68
+ else:
69
+ config = configs.pop(0)
70
+ print(f"{config = }")
71
+ dataset = load_dataset(repo_id, config, revision=revision, trust_remote_code=trust_remote_code)
72
+ commit_info = dataset.push_to_hub(
73
+ repo_id,
74
+ config_name=config,
75
+ commit_message="Convert dataset to Parquet",
76
+ commit_description="Convert dataset to Parquet.",
77
+ create_pr=True,
78
+ token=token,
79
+ set_default=default_config is not None,
80
+ )
81
+ time.sleep(5)
82
+ pr_revision, pr_url = commit_info.pr_revision, commit_info.pr_url
83
+ for config in configs:
84
+ print(f"{config = }")
85
+ dataset = load_dataset(repo_id, config, revision=revision, trust_remote_code=trust_remote_code)
86
+ dataset.push_to_hub(
87
+ repo_id,
88
+ config_name=config,
89
+ commit_message=f"Add '{config}' config data files",
90
+ revision=pr_revision,
91
+ token=token,
92
+ )
93
+ time.sleep(5)
94
+ _delete_files(repo_id, revision=pr_revision, token=token)
95
+ if not revision:
96
+ api = HfApi(endpoint=datasets.config.HF_ENDPOINT, token=token)
97
+ try:
98
+ api.create_branch(repo_id, branch="script", repo_type="dataset", token=token, exist_ok=True)
99
+ except HfHubHTTPError:
100
+ pass
101
+ print(f"You can find your PR to convert the dataset to Parquet at: {pr_url}")
102
+ return commit_info
103
+
104
+
105
+ def delete_from_hub(
106
+ repo_id: str,
107
+ config_name: str,
108
+ revision: Optional[str] = None,
109
+ token: Optional[Union[bool, str]] = None,
110
+ ) -> CommitInfo:
111
+ """Delete a dataset configuration from a [data-only dataset](repository_structure) on the Hub.
112
+
113
+ Args:
114
+ repo_id (`str`): ID of the Hub dataset repository, in the following format: `<user>/<dataset_name>` or
115
+ `<org>/<dataset_name>`.
116
+ config_name (`str`): Name of the dataset configuration.
117
+ revision (`str`, *optional*): Branch to delete the configuration from. Defaults to the `"main"` branch.
118
+ token (`bool` or `str`, *optional*): Authentication token for the Hugging Face Hub.
119
+
120
+ Returns:
121
+ `huggingface_hub.CommitInfo`
122
+ """
123
+ operations = []
124
+ # data_files
125
+ fs = HfFileSystem(endpoint=datasets.config.HF_ENDPOINT, token=token)
126
+ builder = load_dataset_builder(repo_id, config_name, revision=revision, token=token, trust_remote_code=False)
127
+ for data_file in chain(*builder.config.data_files.values()):
128
+ data_file_resolved_path = fs.resolve_path(data_file)
129
+ if data_file_resolved_path.repo_id == repo_id:
130
+ operations.append(CommitOperationDelete(path_in_repo=data_file_resolved_path.path_in_repo))
131
+ # README.md
132
+ dataset_card = DatasetCard.load(repo_id)
133
+ # config_names
134
+ if dataset_card.data.get("config_names", None) and config_name in dataset_card.data["config_names"]:
135
+ dataset_card.data["config_names"].remove(config_name)
136
+ # metadata_configs
137
+ metadata_configs = MetadataConfigs.from_dataset_card_data(dataset_card.data)
138
+ if metadata_configs:
139
+ _ = metadata_configs.pop(config_name, None)
140
+ dataset_card_data = DatasetCardData()
141
+ metadata_configs.to_dataset_card_data(dataset_card_data)
142
+ if datasets.config.METADATA_CONFIGS_FIELD in dataset_card_data:
143
+ dataset_card.data[datasets.config.METADATA_CONFIGS_FIELD] = dataset_card_data[
144
+ datasets.config.METADATA_CONFIGS_FIELD
145
+ ]
146
+ else:
147
+ _ = dataset_card.data.pop(datasets.config.METADATA_CONFIGS_FIELD, None)
148
+ # dataset_info
149
+ dataset_infos: DatasetInfosDict = DatasetInfosDict.from_dataset_card_data(dataset_card.data)
150
+ if dataset_infos:
151
+ _ = dataset_infos.pop(config_name, None)
152
+ dataset_card_data = DatasetCardData()
153
+ dataset_infos.to_dataset_card_data(dataset_card_data)
154
+ if "dataset_info" in dataset_card_data:
155
+ dataset_card.data["dataset_info"] = dataset_card_data["dataset_info"]
156
+ else:
157
+ _ = dataset_card.data.pop("dataset_info", None)
158
+ # Commit
159
+ operations.append(
160
+ CommitOperationAdd(path_in_repo=datasets.config.REPOCARD_FILENAME, path_or_fileobj=str(dataset_card).encode())
161
+ )
162
+ api = HfApi(endpoint=datasets.config.HF_ENDPOINT, token=token)
163
+ commit_info = api.create_commit(
164
+ repo_id,
165
+ operations=operations,
166
+ commit_message=f"Delete '{config_name}' config",
167
+ commit_description=f"Delete '{config_name}' config.",
168
+ token=token,
169
+ repo_type="dataset",
170
+ revision=revision,
171
+ create_pr=True,
172
+ )
173
+ print(f"You can find your PR to delete the dataset config at: {commit_info.pr_url}")
174
+ return commit_info
175
+
176
+
177
+ def _delete_files(dataset_id, revision=None, token=None):
178
+ dataset_name = dataset_id.split("/")[-1]
179
+ hf_api = HfApi(endpoint=datasets.config.HF_ENDPOINT, token=token)
180
+ repo_files = hf_api.list_repo_files(
181
+ dataset_id,
182
+ repo_type="dataset",
183
+ )
184
+ if repo_files:
185
+ legacy_json_file = []
186
+ python_files = []
187
+ data_files = []
188
+ for filename in repo_files:
189
+ if filename in {".gitattributes", "README.md"}:
190
+ continue
191
+ elif filename == f"{dataset_name}.py":
192
+ hf_api.delete_file(
193
+ filename,
194
+ dataset_id,
195
+ repo_type="dataset",
196
+ revision=revision,
197
+ commit_message="Delete loading script",
198
+ )
199
+ elif filename == "dataset_infos.json":
200
+ legacy_json_file.append(filename)
201
+ elif filename.endswith(".py"):
202
+ python_files.append(filename)
203
+ else:
204
+ data_files.append(filename)
205
+ if legacy_json_file:
206
+ hf_api.delete_file(
207
+ "dataset_infos.json",
208
+ dataset_id,
209
+ repo_type="dataset",
210
+ revision=revision,
211
+ commit_message="Delete legacy dataset_infos.json",
212
+ )
213
+ if python_files:
214
+ for filename in python_files:
215
+ hf_api.delete_file(
216
+ filename,
217
+ dataset_id,
218
+ repo_type="dataset",
219
+ revision=revision,
220
+ commit_message="Delete loading script auxiliary file",
221
+ )
222
+ if data_files:
223
+ for filename in data_files:
224
+ hf_api.delete_file(
225
+ filename,
226
+ dataset_id,
227
+ repo_type="dataset",
228
+ revision=revision,
229
+ commit_message="Delete data file",
230
+ )
lib/python3.12/site-packages/datasets/info.py ADDED
@@ -0,0 +1,430 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ # Lint as: python3
16
+ """DatasetInfo record information we know about a dataset.
17
+
18
+ This includes things that we know about the dataset statically, i.e.:
19
+ - description
20
+ - canonical location
21
+ - does it have validation and tests splits
22
+ - size
23
+ - etc.
24
+
25
+ This also includes the things that can and should be computed once we've
26
+ processed the dataset as well:
27
+ - number of examples (in each split)
28
+ - etc.
29
+ """
30
+
31
+ import copy
32
+ import dataclasses
33
+ import json
34
+ import os
35
+ import posixpath
36
+ from dataclasses import dataclass
37
+ from pathlib import Path
38
+ from typing import ClassVar, Optional, Union
39
+
40
+ import fsspec
41
+ from fsspec.core import url_to_fs
42
+ from huggingface_hub import DatasetCard, DatasetCardData
43
+
44
+ from . import config
45
+ from .features import Features
46
+ from .splits import SplitDict
47
+ from .utils import Version
48
+ from .utils.logging import get_logger
49
+ from .utils.py_utils import asdict, unique_values
50
+
51
+
52
+ logger = get_logger(__name__)
53
+
54
+
55
+ @dataclass
56
+ class SupervisedKeysData:
57
+ input: str = ""
58
+ output: str = ""
59
+
60
+
61
+ @dataclass
62
+ class DownloadChecksumsEntryData:
63
+ key: str = ""
64
+ value: str = ""
65
+
66
+
67
+ class MissingCachedSizesConfigError(Exception):
68
+ """The expected cached sizes of the download file are missing."""
69
+
70
+
71
+ class NonMatchingCachedSizesError(Exception):
72
+ """The prepared split doesn't have expected sizes."""
73
+
74
+
75
+ @dataclass
76
+ class PostProcessedInfo:
77
+ features: Optional[Features] = None
78
+ resources_checksums: Optional[dict] = None
79
+
80
+ def __post_init__(self):
81
+ # Convert back to the correct classes when we reload from dict
82
+ if self.features is not None and not isinstance(self.features, Features):
83
+ self.features = Features.from_dict(self.features)
84
+
85
+ @classmethod
86
+ def from_dict(cls, post_processed_info_dict: dict) -> "PostProcessedInfo":
87
+ field_names = {f.name for f in dataclasses.fields(cls)}
88
+ return cls(**{k: v for k, v in post_processed_info_dict.items() if k in field_names})
89
+
90
+
91
+ @dataclass
92
+ class DatasetInfo:
93
+ """Information about a dataset.
94
+
95
+ `DatasetInfo` documents datasets, including its name, version, and features.
96
+ See the constructor arguments and properties for a full list.
97
+
98
+ Not all fields are known on construction and may be updated later.
99
+
100
+ Attributes:
101
+ description (`str`):
102
+ A description of the dataset.
103
+ citation (`str`):
104
+ A BibTeX citation of the dataset.
105
+ homepage (`str`):
106
+ A URL to the official homepage for the dataset.
107
+ license (`str`):
108
+ The dataset's license. It can be the name of the license or a paragraph containing the terms of the license.
109
+ features ([`Features`], *optional*):
110
+ The features used to specify the dataset's column types.
111
+ post_processed (`PostProcessedInfo`, *optional*):
112
+ Information regarding the resources of a possible post-processing of a dataset. For example, it can contain the information of an index.
113
+ supervised_keys (`SupervisedKeysData`, *optional*):
114
+ Specifies the input feature and the label for supervised learning if applicable for the dataset (legacy from TFDS).
115
+ builder_name (`str`, *optional*):
116
+ The name of the `GeneratorBasedBuilder` subclass used to create the dataset. Usually matched to the corresponding script name. It is also the snake_case version of the dataset builder class name.
117
+ config_name (`str`, *optional*):
118
+ The name of the configuration derived from [`BuilderConfig`].
119
+ version (`str` or [`Version`], *optional*):
120
+ The version of the dataset.
121
+ splits (`dict`, *optional*):
122
+ The mapping between split name and metadata.
123
+ download_checksums (`dict`, *optional*):
124
+ The mapping between the URL to download the dataset's checksums and corresponding metadata.
125
+ download_size (`int`, *optional*):
126
+ The size of the files to download to generate the dataset, in bytes.
127
+ post_processing_size (`int`, *optional*):
128
+ Size of the dataset in bytes after post-processing, if any.
129
+ dataset_size (`int`, *optional*):
130
+ The combined size in bytes of the Arrow tables for all splits.
131
+ size_in_bytes (`int`, *optional*):
132
+ The combined size in bytes of all files associated with the dataset (downloaded files + Arrow files).
133
+ **config_kwargs (additional keyword arguments):
134
+ Keyword arguments to be passed to the [`BuilderConfig`] and used in the [`DatasetBuilder`].
135
+ """
136
+
137
+ # Set in the dataset scripts
138
+ description: str = dataclasses.field(default_factory=str)
139
+ citation: str = dataclasses.field(default_factory=str)
140
+ homepage: str = dataclasses.field(default_factory=str)
141
+ license: str = dataclasses.field(default_factory=str)
142
+ features: Optional[Features] = None
143
+ post_processed: Optional[PostProcessedInfo] = None
144
+ supervised_keys: Optional[SupervisedKeysData] = None
145
+
146
+ # Set later by the builder
147
+ builder_name: Optional[str] = None
148
+ dataset_name: Optional[str] = None # for packaged builders, to be different from builder_name
149
+ config_name: Optional[str] = None
150
+ version: Optional[Union[str, Version]] = None
151
+ # Set later by `download_and_prepare`
152
+ splits: Optional[dict] = None
153
+ download_checksums: Optional[dict] = None
154
+ download_size: Optional[int] = None
155
+ post_processing_size: Optional[int] = None
156
+ dataset_size: Optional[int] = None
157
+ size_in_bytes: Optional[int] = None
158
+
159
+ _INCLUDED_INFO_IN_YAML: ClassVar[list[str]] = [
160
+ "config_name",
161
+ "download_size",
162
+ "dataset_size",
163
+ "features",
164
+ "splits",
165
+ ]
166
+
167
+ def __post_init__(self):
168
+ # Convert back to the correct classes when we reload from dict
169
+ if self.features is not None and not isinstance(self.features, Features):
170
+ self.features = Features.from_dict(self.features)
171
+ if self.post_processed is not None and not isinstance(self.post_processed, PostProcessedInfo):
172
+ self.post_processed = PostProcessedInfo.from_dict(self.post_processed)
173
+ if self.version is not None and not isinstance(self.version, Version):
174
+ if isinstance(self.version, str):
175
+ self.version = Version(self.version)
176
+ else:
177
+ self.version = Version.from_dict(self.version)
178
+ if self.splits is not None and not isinstance(self.splits, SplitDict):
179
+ self.splits = SplitDict.from_split_dict(self.splits)
180
+ if self.supervised_keys is not None and not isinstance(self.supervised_keys, SupervisedKeysData):
181
+ if isinstance(self.supervised_keys, (tuple, list)):
182
+ self.supervised_keys = SupervisedKeysData(*self.supervised_keys)
183
+ else:
184
+ self.supervised_keys = SupervisedKeysData(**self.supervised_keys)
185
+
186
+ def write_to_directory(self, dataset_info_dir, pretty_print=False, storage_options: Optional[dict] = None):
187
+ """Write `DatasetInfo` and license (if present) as JSON files to `dataset_info_dir`.
188
+
189
+ Args:
190
+ dataset_info_dir (`str`):
191
+ Destination directory.
192
+ pretty_print (`bool`, defaults to `False`):
193
+ If `True`, the JSON will be pretty-printed with the indent level of 4.
194
+ storage_options (`dict`, *optional*):
195
+ Key/value pairs to be passed on to the file-system backend, if any.
196
+
197
+ <Added version="2.9.0"/>
198
+
199
+ Example:
200
+
201
+ ```py
202
+ >>> from datasets import load_dataset
203
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
204
+ >>> ds.info.write_to_directory("/path/to/directory/")
205
+ ```
206
+ """
207
+ fs: fsspec.AbstractFileSystem
208
+ fs, *_ = url_to_fs(dataset_info_dir, **(storage_options or {}))
209
+ with fs.open(posixpath.join(dataset_info_dir, config.DATASET_INFO_FILENAME), "wb") as f:
210
+ self._dump_info(f, pretty_print=pretty_print)
211
+ if self.license:
212
+ with fs.open(posixpath.join(dataset_info_dir, config.LICENSE_FILENAME), "wb") as f:
213
+ self._dump_license(f)
214
+
215
+ def _dump_info(self, file, pretty_print=False):
216
+ """Dump info in `file` file-like object open in bytes mode (to support remote files)"""
217
+ file.write(json.dumps(asdict(self), indent=4 if pretty_print else None).encode("utf-8"))
218
+
219
+ def _dump_license(self, file):
220
+ """Dump license in `file` file-like object open in bytes mode (to support remote files)"""
221
+ file.write(self.license.encode("utf-8"))
222
+
223
+ @classmethod
224
+ def from_merge(cls, dataset_infos: list["DatasetInfo"]):
225
+ dataset_infos = [dset_info.copy() for dset_info in dataset_infos if dset_info is not None]
226
+
227
+ if len(dataset_infos) > 0 and all(dataset_infos[0] == dset_info for dset_info in dataset_infos):
228
+ # if all dataset_infos are equal we don't need to merge. Just return the first.
229
+ return dataset_infos[0]
230
+
231
+ description = "\n\n".join(unique_values(info.description for info in dataset_infos)).strip()
232
+ citation = "\n\n".join(unique_values(info.citation for info in dataset_infos)).strip()
233
+ homepage = "\n\n".join(unique_values(info.homepage for info in dataset_infos)).strip()
234
+ license = "\n\n".join(unique_values(info.license for info in dataset_infos)).strip()
235
+ features = None
236
+ supervised_keys = None
237
+
238
+ return cls(
239
+ description=description,
240
+ citation=citation,
241
+ homepage=homepage,
242
+ license=license,
243
+ features=features,
244
+ supervised_keys=supervised_keys,
245
+ )
246
+
247
+ @classmethod
248
+ def from_directory(cls, dataset_info_dir: str, storage_options: Optional[dict] = None) -> "DatasetInfo":
249
+ """Create [`DatasetInfo`] from the JSON file in `dataset_info_dir`.
250
+
251
+ This function updates all the dynamically generated fields (num_examples,
252
+ hash, time of creation,...) of the [`DatasetInfo`].
253
+
254
+ This will overwrite all previous metadata.
255
+
256
+ Args:
257
+ dataset_info_dir (`str`):
258
+ The directory containing the metadata file. This
259
+ should be the root directory of a specific dataset version.
260
+ storage_options (`dict`, *optional*):
261
+ Key/value pairs to be passed on to the file-system backend, if any.
262
+
263
+ <Added version="2.9.0"/>
264
+
265
+ Example:
266
+
267
+ ```py
268
+ >>> from datasets import DatasetInfo
269
+ >>> ds_info = DatasetInfo.from_directory("/path/to/directory/")
270
+ ```
271
+ """
272
+ fs: fsspec.AbstractFileSystem
273
+ fs, *_ = url_to_fs(dataset_info_dir, **(storage_options or {}))
274
+ logger.info(f"Loading Dataset info from {dataset_info_dir}")
275
+ if not dataset_info_dir:
276
+ raise ValueError("Calling DatasetInfo.from_directory() with undefined dataset_info_dir.")
277
+ with fs.open(posixpath.join(dataset_info_dir, config.DATASET_INFO_FILENAME), "r", encoding="utf-8") as f:
278
+ dataset_info_dict = json.load(f)
279
+ return cls.from_dict(dataset_info_dict)
280
+
281
+ @classmethod
282
+ def from_dict(cls, dataset_info_dict: dict) -> "DatasetInfo":
283
+ field_names = {f.name for f in dataclasses.fields(cls)}
284
+ return cls(**{k: v for k, v in dataset_info_dict.items() if k in field_names})
285
+
286
+ def update(self, other_dataset_info: "DatasetInfo", ignore_none=True):
287
+ self_dict = self.__dict__
288
+ self_dict.update(
289
+ **{
290
+ k: copy.deepcopy(v)
291
+ for k, v in other_dataset_info.__dict__.items()
292
+ if (v is not None or not ignore_none)
293
+ }
294
+ )
295
+
296
+ def copy(self) -> "DatasetInfo":
297
+ return self.__class__(**{k: copy.deepcopy(v) for k, v in self.__dict__.items()})
298
+
299
+ def _to_yaml_dict(self) -> dict:
300
+ yaml_dict = {}
301
+ dataset_info_dict = asdict(self)
302
+ for key in dataset_info_dict:
303
+ if key in self._INCLUDED_INFO_IN_YAML:
304
+ value = getattr(self, key)
305
+ if hasattr(value, "_to_yaml_list"): # Features, SplitDict
306
+ yaml_dict[key] = value._to_yaml_list()
307
+ elif hasattr(value, "_to_yaml_string"): # Version
308
+ yaml_dict[key] = value._to_yaml_string()
309
+ else:
310
+ yaml_dict[key] = value
311
+ return yaml_dict
312
+
313
+ @classmethod
314
+ def _from_yaml_dict(cls, yaml_data: dict) -> "DatasetInfo":
315
+ yaml_data = copy.deepcopy(yaml_data)
316
+ if yaml_data.get("features") is not None:
317
+ yaml_data["features"] = Features._from_yaml_list(yaml_data["features"])
318
+ if yaml_data.get("splits") is not None:
319
+ yaml_data["splits"] = SplitDict._from_yaml_list(yaml_data["splits"])
320
+ field_names = {f.name for f in dataclasses.fields(cls)}
321
+ return cls(**{k: v for k, v in yaml_data.items() if k in field_names})
322
+
323
+
324
+ class DatasetInfosDict(dict[str, DatasetInfo]):
325
+ def write_to_directory(self, dataset_infos_dir, overwrite=False, pretty_print=False) -> None:
326
+ total_dataset_infos = {}
327
+ dataset_infos_path = os.path.join(dataset_infos_dir, config.DATASETDICT_INFOS_FILENAME)
328
+ dataset_readme_path = os.path.join(dataset_infos_dir, config.REPOCARD_FILENAME)
329
+ if not overwrite:
330
+ total_dataset_infos = self.from_directory(dataset_infos_dir)
331
+ total_dataset_infos.update(self)
332
+ if os.path.exists(dataset_infos_path):
333
+ # for backward compatibility, let's update the JSON file if it exists
334
+ with open(dataset_infos_path, "w", encoding="utf-8") as f:
335
+ dataset_infos_dict = {
336
+ config_name: asdict(dset_info) for config_name, dset_info in total_dataset_infos.items()
337
+ }
338
+ json.dump(dataset_infos_dict, f, indent=4 if pretty_print else None)
339
+ # Dump the infos in the YAML part of the README.md file
340
+ if os.path.exists(dataset_readme_path):
341
+ dataset_card = DatasetCard.load(dataset_readme_path)
342
+ dataset_card_data = dataset_card.data
343
+ else:
344
+ dataset_card = None
345
+ dataset_card_data = DatasetCardData()
346
+ if total_dataset_infos:
347
+ total_dataset_infos.to_dataset_card_data(dataset_card_data)
348
+ dataset_card = (
349
+ DatasetCard("---\n" + str(dataset_card_data) + "\n---\n") if dataset_card is None else dataset_card
350
+ )
351
+ dataset_card.save(Path(dataset_readme_path))
352
+
353
+ @classmethod
354
+ def from_directory(cls, dataset_infos_dir) -> "DatasetInfosDict":
355
+ logger.info(f"Loading Dataset Infos from {dataset_infos_dir}")
356
+ # Load the info from the YAML part of README.md
357
+ if os.path.exists(os.path.join(dataset_infos_dir, config.REPOCARD_FILENAME)):
358
+ dataset_card_data = DatasetCard.load(Path(dataset_infos_dir) / config.REPOCARD_FILENAME).data
359
+ if "dataset_info" in dataset_card_data:
360
+ return cls.from_dataset_card_data(dataset_card_data)
361
+ if os.path.exists(os.path.join(dataset_infos_dir, config.DATASETDICT_INFOS_FILENAME)):
362
+ # this is just to have backward compatibility with dataset_infos.json files
363
+ with open(os.path.join(dataset_infos_dir, config.DATASETDICT_INFOS_FILENAME), encoding="utf-8") as f:
364
+ return cls(
365
+ {
366
+ config_name: DatasetInfo.from_dict(dataset_info_dict)
367
+ for config_name, dataset_info_dict in json.load(f).items()
368
+ }
369
+ )
370
+ else:
371
+ return cls()
372
+
373
+ @classmethod
374
+ def from_dataset_card_data(cls, dataset_card_data: DatasetCardData) -> "DatasetInfosDict":
375
+ if isinstance(dataset_card_data.get("dataset_info"), (list, dict)):
376
+ if isinstance(dataset_card_data["dataset_info"], list):
377
+ return cls(
378
+ {
379
+ dataset_info_yaml_dict.get("config_name", "default"): DatasetInfo._from_yaml_dict(
380
+ dataset_info_yaml_dict
381
+ )
382
+ for dataset_info_yaml_dict in dataset_card_data["dataset_info"]
383
+ }
384
+ )
385
+ else:
386
+ dataset_info = DatasetInfo._from_yaml_dict(dataset_card_data["dataset_info"])
387
+ dataset_info.config_name = dataset_card_data["dataset_info"].get("config_name", "default")
388
+ return cls({dataset_info.config_name: dataset_info})
389
+ else:
390
+ return cls()
391
+
392
+ def to_dataset_card_data(self, dataset_card_data: DatasetCardData) -> None:
393
+ if self:
394
+ # first get existing metadata info
395
+ if "dataset_info" in dataset_card_data and isinstance(dataset_card_data["dataset_info"], dict):
396
+ dataset_metadata_infos = {
397
+ dataset_card_data["dataset_info"].get("config_name", "default"): dataset_card_data["dataset_info"]
398
+ }
399
+ elif "dataset_info" in dataset_card_data and isinstance(dataset_card_data["dataset_info"], list):
400
+ dataset_metadata_infos = {
401
+ config_metadata["config_name"]: config_metadata
402
+ for config_metadata in dataset_card_data["dataset_info"]
403
+ }
404
+ else:
405
+ dataset_metadata_infos = {}
406
+ # update/rewrite existing metadata info with the one to dump
407
+ total_dataset_infos = {
408
+ **dataset_metadata_infos,
409
+ **{config_name: dset_info._to_yaml_dict() for config_name, dset_info in self.items()},
410
+ }
411
+ # the config_name from the dataset_infos_dict takes over the config_name of the DatasetInfo
412
+ for config_name, dset_info_yaml_dict in total_dataset_infos.items():
413
+ dset_info_yaml_dict["config_name"] = config_name
414
+ if len(total_dataset_infos) == 1:
415
+ # use a struct instead of a list of configurations, since there's only one
416
+ dataset_card_data["dataset_info"] = next(iter(total_dataset_infos.values()))
417
+ config_name = dataset_card_data["dataset_info"].pop("config_name", None)
418
+ if config_name != "default":
419
+ # if config_name is not "default" preserve it and put at the first position
420
+ dataset_card_data["dataset_info"] = {
421
+ "config_name": config_name,
422
+ **dataset_card_data["dataset_info"],
423
+ }
424
+ else:
425
+ dataset_card_data["dataset_info"] = []
426
+ for config_name, dataset_info_yaml_dict in sorted(total_dataset_infos.items()):
427
+ # add the config_name field in first position
428
+ dataset_info_yaml_dict.pop("config_name", None)
429
+ dataset_info_yaml_dict = {"config_name": config_name, **dataset_info_yaml_dict}
430
+ dataset_card_data["dataset_info"].append(dataset_info_yaml_dict)
lib/python3.12/site-packages/datasets/inspect.py ADDED
@@ -0,0 +1,363 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2020 The HuggingFace Datasets Authors.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ # Lint as: python3
16
+ """List and inspect datasets."""
17
+
18
+ import os
19
+ from collections.abc import Mapping, Sequence
20
+ from typing import Optional, Union
21
+
22
+ from .download.download_config import DownloadConfig
23
+ from .download.download_manager import DownloadMode
24
+ from .download.streaming_download_manager import StreamingDownloadManager
25
+ from .info import DatasetInfo
26
+ from .load import (
27
+ dataset_module_factory,
28
+ get_dataset_builder_class,
29
+ load_dataset_builder,
30
+ )
31
+ from .utils.logging import get_logger
32
+ from .utils.version import Version
33
+
34
+
35
+ logger = get_logger(__name__)
36
+
37
+
38
+ class SplitsNotFoundError(ValueError):
39
+ pass
40
+
41
+
42
+ def get_dataset_infos(
43
+ path: str,
44
+ data_files: Optional[Union[dict, list, str]] = None,
45
+ download_config: Optional[DownloadConfig] = None,
46
+ download_mode: Optional[Union[DownloadMode, str]] = None,
47
+ revision: Optional[Union[str, Version]] = None,
48
+ token: Optional[Union[bool, str]] = None,
49
+ **config_kwargs,
50
+ ):
51
+ """Get the meta information about a dataset, returned as a dict mapping config name to DatasetInfoDict.
52
+
53
+ Args:
54
+ path (`str`): path to the dataset processing script with the dataset builder. Can be either:
55
+
56
+ - a local path to processing script or the directory containing the script (if the script has the same name as the directory),
57
+ e.g. `'./dataset/squad'` or `'./dataset/squad/squad.py'`
58
+ - a dataset identifier on the Hugging Face Hub (list all available datasets and ids with [`huggingface_hub.list_datasets`]),
59
+ e.g. `'rajpurkar/squad'`, `'nyu-mll/glue'` or``'openai/webtext'`
60
+ revision (`Union[str, datasets.Version]`, *optional*):
61
+ If specified, the dataset module will be loaded from the datasets repository at this version.
62
+ By default:
63
+ - it is set to the local version of the lib.
64
+ - it will also try to load it from the main branch if it's not available at the local version of the lib.
65
+ Specifying a version that is different from your local version of the lib might cause compatibility issues.
66
+ download_config ([`DownloadConfig`], *optional*):
67
+ Specific download configuration parameters.
68
+ download_mode ([`DownloadMode`] or `str`, defaults to `REUSE_DATASET_IF_EXISTS`):
69
+ Download/generate mode.
70
+ data_files (`Union[Dict, List, str]`, *optional*):
71
+ Defining the data_files of the dataset configuration.
72
+ token (`str` or `bool`, *optional*):
73
+ Optional string or boolean to use as Bearer token for remote files on the Datasets Hub.
74
+ If `True`, or not specified, will get token from `"~/.huggingface"`.
75
+ **config_kwargs (additional keyword arguments):
76
+ Optional attributes for builder class which will override the attributes if supplied.
77
+
78
+ Example:
79
+
80
+ ```py
81
+ >>> from datasets import get_dataset_infos
82
+ >>> get_dataset_infos('cornell-movie-review-data/rotten_tomatoes')
83
+ {'default': DatasetInfo(description="Movie Review Dataset.\nThis is a dataset of containing 5,331 positive and 5,331 negative processed\nsentences from Rotten Tomatoes movie reviews...), ...}
84
+ ```
85
+ """
86
+ config_names = get_dataset_config_names(
87
+ path=path,
88
+ revision=revision,
89
+ download_config=download_config,
90
+ download_mode=download_mode,
91
+ data_files=data_files,
92
+ token=token,
93
+ )
94
+ return {
95
+ config_name: get_dataset_config_info(
96
+ path=path,
97
+ config_name=config_name,
98
+ data_files=data_files,
99
+ download_config=download_config,
100
+ download_mode=download_mode,
101
+ revision=revision,
102
+ token=token,
103
+ **config_kwargs,
104
+ )
105
+ for config_name in config_names
106
+ }
107
+
108
+
109
+ def get_dataset_config_names(
110
+ path: str,
111
+ revision: Optional[Union[str, Version]] = None,
112
+ download_config: Optional[DownloadConfig] = None,
113
+ download_mode: Optional[Union[DownloadMode, str]] = None,
114
+ dynamic_modules_path: Optional[str] = None,
115
+ data_files: Optional[Union[dict, list, str]] = None,
116
+ **download_kwargs,
117
+ ):
118
+ """Get the list of available config names for a particular dataset.
119
+
120
+ Args:
121
+ path (`str`): path to the dataset processing script with the dataset builder. Can be either:
122
+
123
+ - a local path to processing script or the directory containing the script (if the script has the same name as the directory),
124
+ e.g. `'./dataset/squad'` or `'./dataset/squad/squad.py'`
125
+ - a dataset identifier on the Hugging Face Hub (list all available datasets and ids with [`huggingface_hub.list_datasets`]),
126
+ e.g. `'rajpurkar/squad'`, `'nyu-mll/glue'` or `'openai/webtext'`
127
+ revision (`Union[str, datasets.Version]`, *optional*):
128
+ If specified, the dataset module will be loaded from the datasets repository at this version.
129
+ By default:
130
+ - it is set to the local version of the lib.
131
+ - it will also try to load it from the main branch if it's not available at the local version of the lib.
132
+ Specifying a version that is different from your local version of the lib might cause compatibility issues.
133
+ download_config ([`DownloadConfig`], *optional*):
134
+ Specific download configuration parameters.
135
+ download_mode ([`DownloadMode`] or `str`, defaults to `REUSE_DATASET_IF_EXISTS`):
136
+ Download/generate mode.
137
+ dynamic_modules_path (`str`, defaults to `~/.cache/huggingface/modules/datasets_modules`):
138
+ Optional path to the directory in which the dynamic modules are saved. It must have been initialized with `init_dynamic_modules`.
139
+ By default the datasets are stored inside the `datasets_modules` module.
140
+ data_files (`Union[Dict, List, str]`, *optional*):
141
+ Defining the data_files of the dataset configuration.
142
+ **download_kwargs (additional keyword arguments):
143
+ Optional attributes for [`DownloadConfig`] which will override the attributes in `download_config` if supplied,
144
+ for example `token`.
145
+
146
+ Example:
147
+
148
+ ```py
149
+ >>> from datasets import get_dataset_config_names
150
+ >>> get_dataset_config_names("nyu-mll/glue")
151
+ ['cola',
152
+ 'sst2',
153
+ 'mrpc',
154
+ 'qqp',
155
+ 'stsb',
156
+ 'mnli',
157
+ 'mnli_mismatched',
158
+ 'mnli_matched',
159
+ 'qnli',
160
+ 'rte',
161
+ 'wnli',
162
+ 'ax']
163
+ ```
164
+ """
165
+ dataset_module = dataset_module_factory(
166
+ path,
167
+ revision=revision,
168
+ download_config=download_config,
169
+ download_mode=download_mode,
170
+ dynamic_modules_path=dynamic_modules_path,
171
+ data_files=data_files,
172
+ **download_kwargs,
173
+ )
174
+ builder_cls = get_dataset_builder_class(dataset_module, dataset_name=os.path.basename(path))
175
+ return list(builder_cls.builder_configs.keys()) or [
176
+ dataset_module.builder_kwargs.get("config_name", builder_cls.DEFAULT_CONFIG_NAME or "default")
177
+ ]
178
+
179
+
180
+ def get_dataset_default_config_name(
181
+ path: str,
182
+ revision: Optional[Union[str, Version]] = None,
183
+ download_config: Optional[DownloadConfig] = None,
184
+ download_mode: Optional[Union[DownloadMode, str]] = None,
185
+ dynamic_modules_path: Optional[str] = None,
186
+ data_files: Optional[Union[dict, list, str]] = None,
187
+ **download_kwargs,
188
+ ) -> Optional[str]:
189
+ """Get the default config name for a particular dataset.
190
+ Can return None only if the dataset has multiple configurations and no default configuration.
191
+
192
+ Args:
193
+ path (`str`): path to the dataset processing script with the dataset builder. Can be either:
194
+
195
+ - a local path to processing script or the directory containing the script (if the script has the same name as the directory),
196
+ e.g. `'./dataset/squad'` or `'./dataset/squad/squad.py'`
197
+ - a dataset identifier on the Hugging Face Hub (list all available datasets and ids with [`huggingface_hub.list_datasets`]),
198
+ e.g. `'rajpurkar/squad'`, `'nyu-mll/glue'` or `'openai/webtext'`
199
+ revision (`Union[str, datasets.Version]`, *optional*):
200
+ If specified, the dataset module will be loaded from the datasets repository at this version.
201
+ By default:
202
+ - it is set to the local version of the lib.
203
+ - it will also try to load it from the main branch if it's not available at the local version of the lib.
204
+ Specifying a version that is different from your local version of the lib might cause compatibility issues.
205
+ download_config ([`DownloadConfig`], *optional*):
206
+ Specific download configuration parameters.
207
+ download_mode ([`DownloadMode`] or `str`, defaults to `REUSE_DATASET_IF_EXISTS`):
208
+ Download/generate mode.
209
+ dynamic_modules_path (`str`, defaults to `~/.cache/huggingface/modules/datasets_modules`):
210
+ Optional path to the directory in which the dynamic modules are saved. It must have been initialized with `init_dynamic_modules`.
211
+ By default the datasets are stored inside the `datasets_modules` module.
212
+ data_files (`Union[Dict, List, str]`, *optional*):
213
+ Defining the data_files of the dataset configuration.
214
+ **download_kwargs (additional keyword arguments):
215
+ Optional attributes for [`DownloadConfig`] which will override the attributes in `download_config` if supplied,
216
+ for example `token`.
217
+
218
+ Returns:
219
+ Optional[str]: the default config name if there is one
220
+
221
+ Example:
222
+
223
+ ```py
224
+ >>> from datasets import get_dataset_default_config_name
225
+ >>> get_dataset_default_config_name("openbookqa")
226
+ 'main'
227
+ ```
228
+ """
229
+ dataset_module = dataset_module_factory(
230
+ path,
231
+ revision=revision,
232
+ download_config=download_config,
233
+ download_mode=download_mode,
234
+ dynamic_modules_path=dynamic_modules_path,
235
+ data_files=data_files,
236
+ **download_kwargs,
237
+ )
238
+ builder_cls = get_dataset_builder_class(dataset_module, dataset_name=os.path.basename(path))
239
+ builder_configs = list(builder_cls.builder_configs.keys())
240
+ if builder_configs:
241
+ default_config_name = builder_configs[0] if len(builder_configs) == 1 else None
242
+ else:
243
+ default_config_name = "default"
244
+ return builder_cls.DEFAULT_CONFIG_NAME or default_config_name
245
+
246
+
247
+ def get_dataset_config_info(
248
+ path: str,
249
+ config_name: Optional[str] = None,
250
+ data_files: Optional[Union[str, Sequence[str], Mapping[str, Union[str, Sequence[str]]]]] = None,
251
+ download_config: Optional[DownloadConfig] = None,
252
+ download_mode: Optional[Union[DownloadMode, str]] = None,
253
+ revision: Optional[Union[str, Version]] = None,
254
+ token: Optional[Union[bool, str]] = None,
255
+ **config_kwargs,
256
+ ) -> DatasetInfo:
257
+ """Get the meta information (DatasetInfo) about a dataset for a particular config
258
+
259
+ Args:
260
+ path (``str``): path to the dataset processing script with the dataset builder. Can be either:
261
+
262
+ - a local path to processing script or the directory containing the script (if the script has the same name as the directory),
263
+ e.g. ``'./dataset/squad'`` or ``'./dataset/squad/squad.py'``
264
+ - a dataset identifier on the Hugging Face Hub (list all available datasets and ids with [`huggingface_hub.list_datasets`]),
265
+ e.g. ``'rajpurkar/squad'``, ``'nyu-mll/glue'`` or ``'openai/webtext'``
266
+ config_name (:obj:`str`, optional): Defining the name of the dataset configuration.
267
+ data_files (:obj:`str` or :obj:`Sequence` or :obj:`Mapping`, optional): Path(s) to source data file(s).
268
+ download_config (:class:`~download.DownloadConfig`, optional): Specific download configuration parameters.
269
+ download_mode (:class:`DownloadMode` or :obj:`str`, default ``REUSE_DATASET_IF_EXISTS``): Download/generate mode.
270
+ revision (:class:`~utils.Version` or :obj:`str`, optional): Version of the dataset script to load.
271
+ As datasets have their own git repository on the Datasets Hub, the default version "main" corresponds to their "main" branch.
272
+ You can specify a different version than the default "main" by using a commit SHA or a git tag of the dataset repository.
273
+ token (``str`` or :obj:`bool`, optional): Optional string or boolean to use as Bearer token for remote files on the Datasets Hub.
274
+ If True, or not specified, will get token from `"~/.huggingface"`.
275
+ **config_kwargs (additional keyword arguments): optional attributes for builder class which will override the attributes if supplied.
276
+
277
+ """
278
+ builder = load_dataset_builder(
279
+ path,
280
+ name=config_name,
281
+ data_files=data_files,
282
+ download_config=download_config,
283
+ download_mode=download_mode,
284
+ revision=revision,
285
+ token=token,
286
+ **config_kwargs,
287
+ )
288
+ info = builder.info
289
+ if info.splits is None:
290
+ download_config = download_config.copy() if download_config else DownloadConfig()
291
+ if token is not None:
292
+ download_config.token = token
293
+ builder._check_manual_download(
294
+ StreamingDownloadManager(base_path=builder.base_path, download_config=download_config)
295
+ )
296
+ try:
297
+ info.splits = {
298
+ split_generator.name: {"name": split_generator.name, "dataset_name": path}
299
+ for split_generator in builder._split_generators(
300
+ StreamingDownloadManager(base_path=builder.base_path, download_config=download_config)
301
+ )
302
+ }
303
+ except Exception as err:
304
+ raise SplitsNotFoundError("The split names could not be parsed from the dataset config.") from err
305
+ return info
306
+
307
+
308
+ def get_dataset_split_names(
309
+ path: str,
310
+ config_name: Optional[str] = None,
311
+ data_files: Optional[Union[str, Sequence[str], Mapping[str, Union[str, Sequence[str]]]]] = None,
312
+ download_config: Optional[DownloadConfig] = None,
313
+ download_mode: Optional[Union[DownloadMode, str]] = None,
314
+ revision: Optional[Union[str, Version]] = None,
315
+ token: Optional[Union[bool, str]] = None,
316
+ **config_kwargs,
317
+ ):
318
+ """Get the list of available splits for a particular config and dataset.
319
+
320
+ Args:
321
+ path (`str`): path to the dataset processing script with the dataset builder. Can be either:
322
+
323
+ - a local path to processing script or the directory containing the script (if the script has the same name as the directory),
324
+ e.g. `'./dataset/squad'` or `'./dataset/squad/squad.py'`
325
+ - a dataset identifier on the Hugging Face Hub (list all available datasets and ids with [`huggingface_hub.list_datasets`]),
326
+ e.g. `'rajpurkar/squad'`, `'nyu-mll/glue'` or `'openai/webtext'`
327
+ config_name (`str`, *optional*):
328
+ Defining the name of the dataset configuration.
329
+ data_files (`str` or `Sequence` or `Mapping`, *optional*):
330
+ Path(s) to source data file(s).
331
+ download_config ([`DownloadConfig`], *optional*):
332
+ Specific download configuration parameters.
333
+ download_mode ([`DownloadMode`] or `str`, defaults to `REUSE_DATASET_IF_EXISTS`):
334
+ Download/generate mode.
335
+ revision ([`Version`] or `str`, *optional*):
336
+ Version of the dataset script to load.
337
+ As datasets have their own git repository on the Datasets Hub, the default version "main" corresponds to their "main" branch.
338
+ You can specify a different version than the default "main" by using a commit SHA or a git tag of the dataset repository.
339
+ token (`str` or `bool`, *optional*):
340
+ Optional string or boolean to use as Bearer token for remote files on the Datasets Hub.
341
+ If `True`, or not specified, will get token from `"~/.huggingface"`.
342
+ **config_kwargs (additional keyword arguments):
343
+ Optional attributes for builder class which will override the attributes if supplied.
344
+
345
+ Example:
346
+
347
+ ```py
348
+ >>> from datasets import get_dataset_split_names
349
+ >>> get_dataset_split_names('cornell-movie-review-data/rotten_tomatoes')
350
+ ['train', 'validation', 'test']
351
+ ```
352
+ """
353
+ info = get_dataset_config_info(
354
+ path,
355
+ config_name=config_name,
356
+ data_files=data_files,
357
+ download_config=download_config,
358
+ download_mode=download_mode,
359
+ revision=revision,
360
+ token=token,
361
+ **config_kwargs,
362
+ )
363
+ return list(info.splits.keys())
lib/python3.12/site-packages/datasets/iterable_dataset.py ADDED
The diff for this file is too large to render. See raw diff
 
lib/python3.12/site-packages/datasets/keyhash.py ADDED
@@ -0,0 +1,104 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ # Lint as: python3
16
+
17
+ """
18
+ Hashing function for dataset keys using `hashlib.md5`
19
+
20
+ Requirements for the hash function:
21
+
22
+ - Provides a uniformly distributed hash from random space
23
+ - Adequately fast speed
24
+ - Working with multiple input types (in this case, `str`, `int` or `bytes`)
25
+ - Should be platform independent (generates same hash on different OS and systems)
26
+
27
+ The hashing function provides a unique 128-bit integer hash of the key provided.
28
+
29
+ The split name is being used here as the hash salt to avoid having same hashes
30
+ in different splits due to same keys
31
+ """
32
+
33
+ from typing import Union
34
+
35
+ from huggingface_hub.utils import insecure_hashlib
36
+
37
+
38
+ def _as_bytes(hash_data: Union[str, int, bytes, bytearray]) -> bytes:
39
+ """
40
+ Returns the input hash_data in its bytes form
41
+
42
+ Args:
43
+ hash_data: the hash salt/key to be converted to bytes
44
+ """
45
+ if isinstance(hash_data, (bytes, bytearray)):
46
+ # Data already in bytes, returns as it as
47
+ return hash_data
48
+ elif isinstance(hash_data, str):
49
+ # We keep the data as it as for it ot be later encoded to UTF-8
50
+ # However replace `\\` with `/` for Windows compatibility
51
+ hash_data = hash_data.replace("\\", "/")
52
+ elif isinstance(hash_data, int):
53
+ hash_data = str(hash_data)
54
+ else:
55
+ # If data is not of the required type, raise error
56
+ raise InvalidKeyError(hash_data)
57
+
58
+ return hash_data.encode("utf-8")
59
+
60
+
61
+ class InvalidKeyError(Exception):
62
+ """Raises an error when given key is of invalid datatype."""
63
+
64
+ def __init__(self, hash_data):
65
+ self.prefix = "\nFAILURE TO GENERATE DATASET: Invalid key type detected"
66
+ self.err_msg = f"\nFound Key {hash_data} of type {type(hash_data)}"
67
+ self.suffix = "\nKeys should be either str, int or bytes type"
68
+ super().__init__(f"{self.prefix}{self.err_msg}{self.suffix}")
69
+
70
+
71
+ class DuplicatedKeysError(Exception):
72
+ """Raise an error when duplicate key found."""
73
+
74
+ def __init__(self, key, duplicate_key_indices, fix_msg=""):
75
+ self.key = key
76
+ self.duplicate_key_indices = duplicate_key_indices
77
+ self.fix_msg = fix_msg
78
+ self.prefix = "Found multiple examples generated with the same key"
79
+ if len(duplicate_key_indices) <= 20:
80
+ self.err_msg = f"\nThe examples at index {', '.join(duplicate_key_indices)} have the key {key}"
81
+ else:
82
+ self.err_msg = f"\nThe examples at index {', '.join(duplicate_key_indices[:20])}... ({len(duplicate_key_indices) - 20} more) have the key {key}"
83
+ self.suffix = "\n" + fix_msg if fix_msg else ""
84
+ super().__init__(f"{self.prefix}{self.err_msg}{self.suffix}")
85
+
86
+
87
+ class KeyHasher:
88
+ """KeyHasher class for providing hash using md5"""
89
+
90
+ def __init__(self, hash_salt: str):
91
+ self._split_md5 = insecure_hashlib.md5(_as_bytes(hash_salt))
92
+
93
+ def hash(self, key: Union[str, int, bytes]) -> int:
94
+ """Returns 128-bits unique hash of input key
95
+
96
+ Args:
97
+ key: the input key to be hashed (should be str, int or bytes)
98
+
99
+ Returns: 128-bit int hash key"""
100
+ md5 = self._split_md5.copy()
101
+ byte_key = _as_bytes(key)
102
+ md5.update(byte_key)
103
+ # Convert to integer with hexadecimal conversion
104
+ return int(md5.hexdigest(), 16)
lib/python3.12/site-packages/datasets/load.py ADDED
The diff for this file is too large to render. See raw diff
 
lib/python3.12/site-packages/datasets/naming.py ADDED
@@ -0,0 +1,84 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ # Lint as: python3
16
+ """Utilities for file names."""
17
+
18
+ import itertools
19
+ import os
20
+ import re
21
+
22
+
23
+ _uppercase_uppercase_re = re.compile(r"([A-Z]+)([A-Z][a-z])")
24
+ _lowercase_uppercase_re = re.compile(r"([a-z\d])([A-Z])")
25
+
26
+ _single_underscore_re = re.compile(r"(?<!_)_(?!_)")
27
+ _multiple_underscores_re = re.compile(r"(_{2,})")
28
+
29
+ _split_re = r"^\w+(\.\w+)*$"
30
+
31
+ INVALID_WINDOWS_CHARACTERS_IN_PATH = r"<>:/\|?*"
32
+
33
+
34
+ def camelcase_to_snakecase(name):
35
+ """Convert camel-case string to snake-case."""
36
+ name = _uppercase_uppercase_re.sub(r"\1_\2", name)
37
+ name = _lowercase_uppercase_re.sub(r"\1_\2", name)
38
+ return name.lower()
39
+
40
+
41
+ def snakecase_to_camelcase(name):
42
+ """Convert snake-case string to camel-case string."""
43
+ name = _single_underscore_re.split(name)
44
+ name = [_multiple_underscores_re.split(n) for n in name]
45
+ return "".join(n.capitalize() for n in itertools.chain.from_iterable(name) if n != "")
46
+
47
+
48
+ def filename_prefix_for_name(name):
49
+ if os.path.basename(name) != name:
50
+ raise ValueError(f"Should be a dataset name, not a path: {name}")
51
+ return camelcase_to_snakecase(name)
52
+
53
+
54
+ def filename_prefix_for_split(name, split):
55
+ if os.path.basename(name) != name:
56
+ raise ValueError(f"Should be a dataset name, not a path: {name}")
57
+ if not re.match(_split_re, split):
58
+ raise ValueError(f"Split name should match '{_split_re}'' but got '{split}'.")
59
+ return f"{filename_prefix_for_name(name)}-{split}"
60
+
61
+
62
+ def filepattern_for_dataset_split(dataset_name, split, data_dir, filetype_suffix=None):
63
+ prefix = filename_prefix_for_split(dataset_name, split)
64
+ if filetype_suffix:
65
+ prefix += f".{filetype_suffix}"
66
+ filepath = os.path.join(data_dir, prefix)
67
+ return f"{filepath}*"
68
+
69
+
70
+ def filenames_for_dataset_split(path, dataset_name, split, filetype_suffix=None, shard_lengths=None):
71
+ prefix = filename_prefix_for_split(dataset_name, split)
72
+ prefix = os.path.join(path, prefix)
73
+
74
+ if shard_lengths:
75
+ num_shards = len(shard_lengths)
76
+ filenames = [f"{prefix}-{shard_id:05d}-of-{num_shards:05d}" for shard_id in range(num_shards)]
77
+ if filetype_suffix:
78
+ filenames = [filename + f".{filetype_suffix}" for filename in filenames]
79
+ return filenames
80
+ else:
81
+ filename = prefix
82
+ if filetype_suffix:
83
+ filename += f".{filetype_suffix}"
84
+ return [filename]
lib/python3.12/site-packages/datasets/packaged_modules/__init__.py ADDED
@@ -0,0 +1,104 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import inspect
2
+ import re
3
+ from typing import Dict, List, Tuple
4
+
5
+ from huggingface_hub.utils import insecure_hashlib
6
+
7
+ from .arrow import arrow
8
+ from .audiofolder import audiofolder
9
+ from .cache import cache
10
+ from .csv import csv
11
+ from .imagefolder import imagefolder
12
+ from .json import json
13
+ from .pandas import pandas
14
+ from .parquet import parquet
15
+ from .pdffolder import pdffolder
16
+ from .sql import sql
17
+ from .text import text
18
+ from .videofolder import videofolder
19
+ from .webdataset import webdataset
20
+ from .xml import xml
21
+
22
+
23
+ def _hash_python_lines(lines: list[str]) -> str:
24
+ filtered_lines = []
25
+ for line in lines:
26
+ line = re.sub(r"#.*", "", line) # remove comments
27
+ if line:
28
+ filtered_lines.append(line)
29
+ full_str = "\n".join(filtered_lines)
30
+
31
+ # Make a hash from all this code
32
+ full_bytes = full_str.encode("utf-8")
33
+ return insecure_hashlib.sha256(full_bytes).hexdigest()
34
+
35
+
36
+ # get importable module names and hash for caching
37
+ _PACKAGED_DATASETS_MODULES = {
38
+ "csv": (csv.__name__, _hash_python_lines(inspect.getsource(csv).splitlines())),
39
+ "json": (json.__name__, _hash_python_lines(inspect.getsource(json).splitlines())),
40
+ "pandas": (pandas.__name__, _hash_python_lines(inspect.getsource(pandas).splitlines())),
41
+ "parquet": (parquet.__name__, _hash_python_lines(inspect.getsource(parquet).splitlines())),
42
+ "arrow": (arrow.__name__, _hash_python_lines(inspect.getsource(arrow).splitlines())),
43
+ "text": (text.__name__, _hash_python_lines(inspect.getsource(text).splitlines())),
44
+ "imagefolder": (imagefolder.__name__, _hash_python_lines(inspect.getsource(imagefolder).splitlines())),
45
+ "audiofolder": (audiofolder.__name__, _hash_python_lines(inspect.getsource(audiofolder).splitlines())),
46
+ "videofolder": (videofolder.__name__, _hash_python_lines(inspect.getsource(videofolder).splitlines())),
47
+ "pdffolder": (pdffolder.__name__, _hash_python_lines(inspect.getsource(pdffolder).splitlines())),
48
+ "webdataset": (webdataset.__name__, _hash_python_lines(inspect.getsource(webdataset).splitlines())),
49
+ "xml": (xml.__name__, _hash_python_lines(inspect.getsource(xml).splitlines())),
50
+ }
51
+
52
+ # get importable module names and hash for caching
53
+ _PACKAGED_DATASETS_MODULES_2_15_HASHES = {
54
+ "csv": "eea64c71ca8b46dd3f537ed218fc9bf495d5707789152eb2764f5c78fa66d59d",
55
+ "json": "8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96",
56
+ "pandas": "3ac4ffc4563c796122ef66899b9485a3f1a977553e2d2a8a318c72b8cc6f2202",
57
+ "parquet": "ca31c69184d9832faed373922c2acccec0b13a0bb5bbbe19371385c3ff26f1d1",
58
+ "arrow": "74f69db2c14c2860059d39860b1f400a03d11bf7fb5a8258ca38c501c878c137",
59
+ "text": "c4a140d10f020282918b5dd1b8a49f0104729c6177f60a6b49ec2a365ec69f34",
60
+ "imagefolder": "7b7ce5247a942be131d49ad4f3de5866083399a0f250901bd8dc202f8c5f7ce5",
61
+ "audiofolder": "d3c1655c66c8f72e4efb5c79e952975fa6e2ce538473a6890241ddbddee9071c",
62
+ }
63
+
64
+ # Used to infer the module to use based on the data files extensions
65
+ _EXTENSION_TO_MODULE: dict[str, tuple[str, dict]] = {
66
+ ".csv": ("csv", {}),
67
+ ".tsv": ("csv", {"sep": "\t"}),
68
+ ".json": ("json", {}),
69
+ ".jsonl": ("json", {}),
70
+ # ndjson is no longer maintained (see: https://github.com/ndjson/ndjson-spec/issues/35#issuecomment-1285673417)
71
+ ".ndjson": ("json", {}),
72
+ ".parquet": ("parquet", {}),
73
+ ".geoparquet": ("parquet", {}),
74
+ ".gpq": ("parquet", {}),
75
+ ".arrow": ("arrow", {}),
76
+ ".txt": ("text", {}),
77
+ ".tar": ("webdataset", {}),
78
+ ".xml": ("xml", {}),
79
+ }
80
+ _EXTENSION_TO_MODULE.update({ext: ("imagefolder", {}) for ext in imagefolder.ImageFolder.EXTENSIONS})
81
+ _EXTENSION_TO_MODULE.update({ext.upper(): ("imagefolder", {}) for ext in imagefolder.ImageFolder.EXTENSIONS})
82
+ _EXTENSION_TO_MODULE.update({ext: ("audiofolder", {}) for ext in audiofolder.AudioFolder.EXTENSIONS})
83
+ _EXTENSION_TO_MODULE.update({ext.upper(): ("audiofolder", {}) for ext in audiofolder.AudioFolder.EXTENSIONS})
84
+ _EXTENSION_TO_MODULE.update({ext: ("videofolder", {}) for ext in videofolder.VideoFolder.EXTENSIONS})
85
+ _EXTENSION_TO_MODULE.update({ext.upper(): ("videofolder", {}) for ext in videofolder.VideoFolder.EXTENSIONS})
86
+ _EXTENSION_TO_MODULE.update({ext: ("pdffolder", {}) for ext in pdffolder.PdfFolder.EXTENSIONS})
87
+ _EXTENSION_TO_MODULE.update({ext.upper(): ("pdffolder", {}) for ext in pdffolder.PdfFolder.EXTENSIONS})
88
+
89
+ # Used to filter data files based on extensions given a module name
90
+ _MODULE_TO_EXTENSIONS: dict[str, list[str]] = {}
91
+ for _ext, (_module, _) in _EXTENSION_TO_MODULE.items():
92
+ _MODULE_TO_EXTENSIONS.setdefault(_module, []).append(_ext)
93
+
94
+ for _module in _MODULE_TO_EXTENSIONS:
95
+ _MODULE_TO_EXTENSIONS[_module].append(".zip")
96
+
97
+ # Used to filter data files based on file names
98
+ _MODULE_TO_METADATA_FILE_NAMES: Dict[str, List[str]] = {}
99
+ for _module in _MODULE_TO_EXTENSIONS:
100
+ _MODULE_TO_METADATA_FILE_NAMES[_module] = []
101
+ _MODULE_TO_METADATA_FILE_NAMES["imagefolder"] = imagefolder.ImageFolder.METADATA_FILENAMES
102
+ _MODULE_TO_METADATA_FILE_NAMES["audiofolder"] = imagefolder.ImageFolder.METADATA_FILENAMES
103
+ _MODULE_TO_METADATA_FILE_NAMES["videofolder"] = imagefolder.ImageFolder.METADATA_FILENAMES
104
+ _MODULE_TO_METADATA_FILE_NAMES["pdffolder"] = imagefolder.ImageFolder.METADATA_FILENAMES
lib/python3.12/site-packages/datasets/packaged_modules/csv/__init__.py ADDED
File without changes
lib/python3.12/site-packages/datasets/packaged_modules/csv/__pycache__/__init__.cpython-312.pyc ADDED
Binary file (208 Bytes). View file
 
lib/python3.12/site-packages/datasets/packaged_modules/csv/__pycache__/csv.cpython-312.pyc ADDED
Binary file (11.7 kB). View file
 
lib/python3.12/site-packages/datasets/packaged_modules/csv/csv.py ADDED
@@ -0,0 +1,198 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import itertools
2
+ from dataclasses import dataclass
3
+ from typing import Any, Callable, Optional, Union
4
+
5
+ import pandas as pd
6
+ import pyarrow as pa
7
+
8
+ import datasets
9
+ import datasets.config
10
+ from datasets.features.features import require_storage_cast
11
+ from datasets.table import table_cast
12
+ from datasets.utils.py_utils import Literal
13
+
14
+
15
+ logger = datasets.utils.logging.get_logger(__name__)
16
+
17
+ _PANDAS_READ_CSV_NO_DEFAULT_PARAMETERS = ["names", "prefix"]
18
+ _PANDAS_READ_CSV_DEPRECATED_PARAMETERS = ["warn_bad_lines", "error_bad_lines", "mangle_dupe_cols"]
19
+ _PANDAS_READ_CSV_NEW_1_3_0_PARAMETERS = ["encoding_errors", "on_bad_lines"]
20
+ _PANDAS_READ_CSV_NEW_2_0_0_PARAMETERS = ["date_format"]
21
+ _PANDAS_READ_CSV_DEPRECATED_2_2_0_PARAMETERS = ["verbose"]
22
+
23
+
24
+ @dataclass
25
+ class CsvConfig(datasets.BuilderConfig):
26
+ """BuilderConfig for CSV."""
27
+
28
+ sep: str = ","
29
+ delimiter: Optional[str] = None
30
+ header: Optional[Union[int, list[int], str]] = "infer"
31
+ names: Optional[list[str]] = None
32
+ column_names: Optional[list[str]] = None
33
+ index_col: Optional[Union[int, str, list[int], list[str]]] = None
34
+ usecols: Optional[Union[list[int], list[str]]] = None
35
+ prefix: Optional[str] = None
36
+ mangle_dupe_cols: bool = True
37
+ engine: Optional[Literal["c", "python", "pyarrow"]] = None
38
+ converters: dict[Union[int, str], Callable[[Any], Any]] = None
39
+ true_values: Optional[list] = None
40
+ false_values: Optional[list] = None
41
+ skipinitialspace: bool = False
42
+ skiprows: Optional[Union[int, list[int]]] = None
43
+ nrows: Optional[int] = None
44
+ na_values: Optional[Union[str, list[str]]] = None
45
+ keep_default_na: bool = True
46
+ na_filter: bool = True
47
+ verbose: bool = False
48
+ skip_blank_lines: bool = True
49
+ thousands: Optional[str] = None
50
+ decimal: str = "."
51
+ lineterminator: Optional[str] = None
52
+ quotechar: str = '"'
53
+ quoting: int = 0
54
+ escapechar: Optional[str] = None
55
+ comment: Optional[str] = None
56
+ encoding: Optional[str] = None
57
+ dialect: Optional[str] = None
58
+ error_bad_lines: bool = True
59
+ warn_bad_lines: bool = True
60
+ skipfooter: int = 0
61
+ doublequote: bool = True
62
+ memory_map: bool = False
63
+ float_precision: Optional[str] = None
64
+ chunksize: int = 10_000
65
+ features: Optional[datasets.Features] = None
66
+ encoding_errors: Optional[str] = "strict"
67
+ on_bad_lines: Literal["error", "warn", "skip"] = "error"
68
+ date_format: Optional[str] = None
69
+
70
+ def __post_init__(self):
71
+ super().__post_init__()
72
+ if self.delimiter is not None:
73
+ self.sep = self.delimiter
74
+ if self.column_names is not None:
75
+ self.names = self.column_names
76
+
77
+ @property
78
+ def pd_read_csv_kwargs(self):
79
+ pd_read_csv_kwargs = {
80
+ "sep": self.sep,
81
+ "header": self.header,
82
+ "names": self.names,
83
+ "index_col": self.index_col,
84
+ "usecols": self.usecols,
85
+ "prefix": self.prefix,
86
+ "mangle_dupe_cols": self.mangle_dupe_cols,
87
+ "engine": self.engine,
88
+ "converters": self.converters,
89
+ "true_values": self.true_values,
90
+ "false_values": self.false_values,
91
+ "skipinitialspace": self.skipinitialspace,
92
+ "skiprows": self.skiprows,
93
+ "nrows": self.nrows,
94
+ "na_values": self.na_values,
95
+ "keep_default_na": self.keep_default_na,
96
+ "na_filter": self.na_filter,
97
+ "verbose": self.verbose,
98
+ "skip_blank_lines": self.skip_blank_lines,
99
+ "thousands": self.thousands,
100
+ "decimal": self.decimal,
101
+ "lineterminator": self.lineterminator,
102
+ "quotechar": self.quotechar,
103
+ "quoting": self.quoting,
104
+ "escapechar": self.escapechar,
105
+ "comment": self.comment,
106
+ "encoding": self.encoding,
107
+ "dialect": self.dialect,
108
+ "error_bad_lines": self.error_bad_lines,
109
+ "warn_bad_lines": self.warn_bad_lines,
110
+ "skipfooter": self.skipfooter,
111
+ "doublequote": self.doublequote,
112
+ "memory_map": self.memory_map,
113
+ "float_precision": self.float_precision,
114
+ "chunksize": self.chunksize,
115
+ "encoding_errors": self.encoding_errors,
116
+ "on_bad_lines": self.on_bad_lines,
117
+ "date_format": self.date_format,
118
+ }
119
+
120
+ # some kwargs must not be passed if they don't have a default value
121
+ # some others are deprecated and we can also not pass them if they are the default value
122
+ for pd_read_csv_parameter in _PANDAS_READ_CSV_NO_DEFAULT_PARAMETERS + _PANDAS_READ_CSV_DEPRECATED_PARAMETERS:
123
+ if pd_read_csv_kwargs[pd_read_csv_parameter] == getattr(CsvConfig(), pd_read_csv_parameter):
124
+ del pd_read_csv_kwargs[pd_read_csv_parameter]
125
+
126
+ # Remove 1.3 new arguments
127
+ if not (datasets.config.PANDAS_VERSION.major >= 1 and datasets.config.PANDAS_VERSION.minor >= 3):
128
+ for pd_read_csv_parameter in _PANDAS_READ_CSV_NEW_1_3_0_PARAMETERS:
129
+ del pd_read_csv_kwargs[pd_read_csv_parameter]
130
+
131
+ # Remove 2.0 new arguments
132
+ if not (datasets.config.PANDAS_VERSION.major >= 2):
133
+ for pd_read_csv_parameter in _PANDAS_READ_CSV_NEW_2_0_0_PARAMETERS:
134
+ del pd_read_csv_kwargs[pd_read_csv_parameter]
135
+
136
+ # Remove 2.2 deprecated arguments
137
+ if datasets.config.PANDAS_VERSION.release >= (2, 2):
138
+ for pd_read_csv_parameter in _PANDAS_READ_CSV_DEPRECATED_2_2_0_PARAMETERS:
139
+ if pd_read_csv_kwargs[pd_read_csv_parameter] == getattr(CsvConfig(), pd_read_csv_parameter):
140
+ del pd_read_csv_kwargs[pd_read_csv_parameter]
141
+
142
+ return pd_read_csv_kwargs
143
+
144
+
145
+ class Csv(datasets.ArrowBasedBuilder):
146
+ BUILDER_CONFIG_CLASS = CsvConfig
147
+
148
+ def _info(self):
149
+ return datasets.DatasetInfo(features=self.config.features)
150
+
151
+ def _split_generators(self, dl_manager):
152
+ """We handle string, list and dicts in datafiles"""
153
+ if not self.config.data_files:
154
+ raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}")
155
+ dl_manager.download_config.extract_on_the_fly = True
156
+ data_files = dl_manager.download_and_extract(self.config.data_files)
157
+ splits = []
158
+ for split_name, files in data_files.items():
159
+ if isinstance(files, str):
160
+ files = [files]
161
+ files = [dl_manager.iter_files(file) for file in files]
162
+ splits.append(datasets.SplitGenerator(name=split_name, gen_kwargs={"files": files}))
163
+ return splits
164
+
165
+ def _cast_table(self, pa_table: pa.Table) -> pa.Table:
166
+ if self.config.features is not None:
167
+ schema = self.config.features.arrow_schema
168
+ if all(not require_storage_cast(feature) for feature in self.config.features.values()):
169
+ # cheaper cast
170
+ pa_table = pa.Table.from_arrays([pa_table[field.name] for field in schema], schema=schema)
171
+ else:
172
+ # more expensive cast; allows str <-> int/float or str to Audio for example
173
+ pa_table = table_cast(pa_table, schema)
174
+ return pa_table
175
+
176
+ def _generate_tables(self, files):
177
+ schema = self.config.features.arrow_schema if self.config.features else None
178
+ # dtype allows reading an int column as str
179
+ dtype = (
180
+ {
181
+ name: dtype.to_pandas_dtype() if not require_storage_cast(feature) else object
182
+ for name, dtype, feature in zip(schema.names, schema.types, self.config.features.values())
183
+ }
184
+ if schema is not None
185
+ else None
186
+ )
187
+ for file_idx, file in enumerate(itertools.chain.from_iterable(files)):
188
+ csv_file_reader = pd.read_csv(file, iterator=True, dtype=dtype, **self.config.pd_read_csv_kwargs)
189
+ try:
190
+ for batch_idx, df in enumerate(csv_file_reader):
191
+ pa_table = pa.Table.from_pandas(df)
192
+ # Uncomment for debugging (will print the Arrow table size and elements)
193
+ # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
194
+ # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
195
+ yield (file_idx, batch_idx), self._cast_table(pa_table)
196
+ except ValueError as e:
197
+ logger.error(f"Failed to read file '{file}' with error {type(e)}: {e}")
198
+ raise
lib/python3.12/site-packages/datasets/packaged_modules/folder_based_builder/__init__.py ADDED
File without changes
lib/python3.12/site-packages/datasets/packaged_modules/folder_based_builder/__pycache__/__init__.cpython-312.pyc ADDED
Binary file (225 Bytes). View file
 
lib/python3.12/site-packages/datasets/packaged_modules/folder_based_builder/__pycache__/folder_based_builder.cpython-312.pyc ADDED
Binary file (23 kB). View file
 
lib/python3.12/site-packages/datasets/packaged_modules/folder_based_builder/folder_based_builder.py ADDED
@@ -0,0 +1,417 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import collections
2
+ import io
3
+ import itertools
4
+ import os
5
+ from dataclasses import dataclass
6
+ from typing import Any, Callable, Iterator, Optional, Union
7
+
8
+ import pandas as pd
9
+ import pyarrow as pa
10
+ import pyarrow.dataset as ds
11
+ import pyarrow.json as paj
12
+ import pyarrow.parquet as pq
13
+
14
+ import datasets
15
+ from datasets import config
16
+ from datasets.features.features import FeatureType, _visit, _visit_with_path, _VisitPath, require_storage_cast
17
+ from datasets.utils.file_utils import readline
18
+
19
+
20
+ logger = datasets.utils.logging.get_logger(__name__)
21
+
22
+
23
+ def count_path_segments(path):
24
+ return path.replace("\\", "/").count("/")
25
+
26
+
27
+ @dataclass
28
+ class FolderBasedBuilderConfig(datasets.BuilderConfig):
29
+ """BuilderConfig for AutoFolder."""
30
+
31
+ features: Optional[datasets.Features] = None
32
+ drop_labels: bool = None
33
+ drop_metadata: bool = None
34
+ filters: Optional[Union[ds.Expression, list[tuple], list[list[tuple]]]] = None
35
+
36
+ def __post_init__(self):
37
+ super().__post_init__()
38
+
39
+
40
+ class FolderBasedBuilder(datasets.GeneratorBasedBuilder):
41
+ """
42
+ Base class for generic data loaders for vision and image data.
43
+
44
+
45
+ Abstract class attributes to be overridden by a child class:
46
+ BASE_FEATURE: feature object to decode data (i.e. datasets.Image, datasets.Audio, ...)
47
+ BASE_COLUMN_NAME: string key name of a base feature (i.e. "image", "audio", ...)
48
+ BUILDER_CONFIG_CLASS: builder config inherited from `folder_based_builder.FolderBasedBuilderConfig`
49
+ EXTENSIONS: list of allowed extensions (only files with these extensions and METADATA_FILENAME files
50
+ will be included in a dataset)
51
+ """
52
+
53
+ BASE_FEATURE: type[FeatureType]
54
+ BASE_COLUMN_NAME: str
55
+ BUILDER_CONFIG_CLASS: FolderBasedBuilderConfig
56
+ EXTENSIONS: list[str]
57
+
58
+ METADATA_FILENAMES: list[str] = ["metadata.csv", "metadata.jsonl", "metadata.parquet"]
59
+
60
+ def _info(self):
61
+ return datasets.DatasetInfo(features=self.config.features)
62
+
63
+ def _split_generators(self, dl_manager):
64
+ if not self.config.data_files:
65
+ raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}")
66
+ dl_manager.download_config.extract_on_the_fly = True
67
+ # Do an early pass if:
68
+ # * `drop_labels` is None (default) or False, to infer the class labels
69
+ # * `drop_metadata` is None (default) or False, to find the metadata files
70
+ do_analyze = not self.config.drop_labels or not self.config.drop_metadata
71
+ labels, path_depths = set(), set()
72
+ metadata_files = collections.defaultdict(set)
73
+
74
+ def analyze(files_or_archives, downloaded_files_or_dirs, split):
75
+ if len(downloaded_files_or_dirs) == 0:
76
+ return
77
+ # The files are separated from the archives at this point, so check the first sample
78
+ # to see if it's a file or a directory and iterate accordingly
79
+ if os.path.isfile(downloaded_files_or_dirs[0]):
80
+ original_files, downloaded_files = files_or_archives, downloaded_files_or_dirs
81
+ for original_file, downloaded_file in zip(original_files, downloaded_files):
82
+ original_file, downloaded_file = str(original_file), str(downloaded_file)
83
+ _, original_file_ext = os.path.splitext(original_file)
84
+ if original_file_ext.lower() in self.EXTENSIONS:
85
+ if not self.config.drop_labels:
86
+ labels.add(os.path.basename(os.path.dirname(original_file)))
87
+ path_depths.add(count_path_segments(original_file))
88
+ elif os.path.basename(original_file) in self.METADATA_FILENAMES:
89
+ metadata_files[split].add((original_file, downloaded_file))
90
+ else:
91
+ original_file_name = os.path.basename(original_file)
92
+ logger.debug(
93
+ f"The file '{original_file_name}' was ignored: it is not a {self.BASE_COLUMN_NAME}, and is not {self.METADATA_FILENAMES} either."
94
+ )
95
+ else:
96
+ archives, downloaded_dirs = files_or_archives, downloaded_files_or_dirs
97
+ for archive, downloaded_dir in zip(archives, downloaded_dirs):
98
+ archive, downloaded_dir = str(archive), str(downloaded_dir)
99
+ for downloaded_dir_file in dl_manager.iter_files(downloaded_dir):
100
+ _, downloaded_dir_file_ext = os.path.splitext(downloaded_dir_file)
101
+ if downloaded_dir_file_ext in self.EXTENSIONS:
102
+ if not self.config.drop_labels:
103
+ labels.add(os.path.basename(os.path.dirname(downloaded_dir_file)))
104
+ path_depths.add(count_path_segments(downloaded_dir_file))
105
+ elif os.path.basename(downloaded_dir_file) in self.METADATA_FILENAMES:
106
+ metadata_files[split].add((None, downloaded_dir_file))
107
+ else:
108
+ archive_file_name = os.path.basename(archive)
109
+ original_file_name = os.path.basename(downloaded_dir_file)
110
+ logger.debug(
111
+ f"The file '{original_file_name}' from the archive '{archive_file_name}' was ignored: it is not a {self.BASE_COLUMN_NAME}, and is not {self.METADATA_FILENAMES} either."
112
+ )
113
+
114
+ data_files = self.config.data_files
115
+ splits = []
116
+ for split_name, files in data_files.items():
117
+ if isinstance(files, str):
118
+ files = [files]
119
+ files, archives = self._split_files_and_archives(files)
120
+ downloaded_files = dl_manager.download(files)
121
+ downloaded_dirs = dl_manager.download_and_extract(archives)
122
+ if do_analyze: # drop_metadata is None or False, drop_labels is None or False
123
+ logger.info(f"Searching for labels and/or metadata files in {split_name} data files...")
124
+ analyze(files, downloaded_files, split_name)
125
+ analyze(archives, downloaded_dirs, split_name)
126
+
127
+ if metadata_files:
128
+ # add metadata if `metadata_files` are found and `drop_metadata` is None (default) or False
129
+ add_metadata = not self.config.drop_metadata
130
+ # if `metadata_files` are found, don't add labels
131
+ add_labels = False
132
+ else:
133
+ # if `metadata_files` are not found, don't add metadata
134
+ add_metadata = False
135
+ # if `metadata_files` are not found and `drop_labels` is None (default) -
136
+ # add labels if files are on the same level in directory hierarchy and there is more than one label
137
+ add_labels = (
138
+ (len(labels) > 1 and len(path_depths) == 1)
139
+ if self.config.drop_labels is None
140
+ else not self.config.drop_labels
141
+ )
142
+
143
+ if add_labels:
144
+ logger.info("Adding the labels inferred from data directories to the dataset's features...")
145
+ if add_metadata:
146
+ logger.info("Adding metadata to the dataset...")
147
+ else:
148
+ add_labels, add_metadata, metadata_files = False, False, {}
149
+
150
+ splits.append(
151
+ datasets.SplitGenerator(
152
+ name=split_name,
153
+ gen_kwargs={
154
+ "files": tuple(zip(files, downloaded_files))
155
+ + tuple((None, dl_manager.iter_files(downloaded_dir)) for downloaded_dir in downloaded_dirs),
156
+ "metadata_files": metadata_files.get(split_name, []),
157
+ "add_labels": add_labels,
158
+ "add_metadata": add_metadata,
159
+ },
160
+ )
161
+ )
162
+
163
+ if add_metadata:
164
+ # Verify that:
165
+ # * all metadata files have the same set of features in each split
166
+ # * the `file_name` key is one of the metadata keys and is of type string
167
+ features_per_metadata_file: list[tuple[str, datasets.Features]] = []
168
+
169
+ # Check that all metadata files share the same format
170
+ metadata_ext = {
171
+ os.path.splitext(original_metadata_file or downloaded_metadata_file)[-1]
172
+ for original_metadata_file, downloaded_metadata_file in itertools.chain.from_iterable(
173
+ metadata_files.values()
174
+ )
175
+ }
176
+ if len(metadata_ext) > 1:
177
+ raise ValueError(f"Found metadata files with different extensions: {list(metadata_ext)}")
178
+ metadata_ext = metadata_ext.pop()
179
+
180
+ for split_metadata_files in metadata_files.values():
181
+ pa_metadata_table = None
182
+ for _, downloaded_metadata_file in split_metadata_files:
183
+ for pa_metadata_table in self._read_metadata(downloaded_metadata_file, metadata_ext=metadata_ext):
184
+ break # just fetch the first rows
185
+ if pa_metadata_table is not None:
186
+ features_per_metadata_file.append(
187
+ (downloaded_metadata_file, datasets.Features.from_arrow_schema(pa_metadata_table.schema))
188
+ )
189
+ break # no need to fetch all the files
190
+ for downloaded_metadata_file, metadata_features in features_per_metadata_file:
191
+ if metadata_features != features_per_metadata_file[0][1]:
192
+ raise ValueError(
193
+ f"Metadata files {downloaded_metadata_file} and {features_per_metadata_file[0][0]} have different features: {features_per_metadata_file[0]} != {metadata_features}"
194
+ )
195
+ metadata_features = features_per_metadata_file[0][1]
196
+ feature_not_found = True
197
+
198
+ def _set_feature(feature):
199
+ nonlocal feature_not_found
200
+ if isinstance(feature, dict):
201
+ out = type(feature)()
202
+ for key in feature:
203
+ if (key == "file_name" or key.endswith("_file_name")) and feature[key] == datasets.Value(
204
+ "string"
205
+ ):
206
+ key = key[: -len("_file_name")] or self.BASE_COLUMN_NAME
207
+ out[key] = self.BASE_FEATURE()
208
+ feature_not_found = False
209
+ elif (key == "file_names" or key.endswith("_file_names")) and feature[
210
+ key
211
+ ] == datasets.Sequence(datasets.Value("string")):
212
+ key = key[: -len("_file_names")] or (self.BASE_COLUMN_NAME + "s")
213
+ out[key] = datasets.Sequence(self.BASE_FEATURE())
214
+ feature_not_found = False
215
+ elif (key == "file_names" or key.endswith("_file_names")) and feature[key] == [
216
+ datasets.Value("string")
217
+ ]:
218
+ key = key[: -len("_file_names")] or (self.BASE_COLUMN_NAME + "s")
219
+ out[key] = [self.BASE_FEATURE()]
220
+ feature_not_found = False
221
+ else:
222
+ out[key] = feature[key]
223
+ return out
224
+ return feature
225
+
226
+ metadata_features = _visit(metadata_features, _set_feature)
227
+
228
+ if feature_not_found:
229
+ raise ValueError(
230
+ "`file_name` or `*_file_name` must be present as dictionary key (with type string) in metadata files"
231
+ )
232
+ else:
233
+ metadata_features = None
234
+
235
+ # Normally, we would do this in _info, but we need to know the labels and/or metadata
236
+ # before building the features
237
+ if self.config.features is None:
238
+ if add_metadata:
239
+ self.info.features = metadata_features
240
+ elif add_labels:
241
+ self.info.features = datasets.Features(
242
+ {
243
+ self.BASE_COLUMN_NAME: self.BASE_FEATURE(),
244
+ "label": datasets.ClassLabel(names=sorted(labels)),
245
+ }
246
+ )
247
+ else:
248
+ self.info.features = datasets.Features({self.BASE_COLUMN_NAME: self.BASE_FEATURE()})
249
+
250
+ return splits
251
+
252
+ def _split_files_and_archives(self, data_files):
253
+ files, archives = [], []
254
+ for data_file in data_files:
255
+ _, data_file_ext = os.path.splitext(data_file)
256
+ if data_file_ext.lower() in self.EXTENSIONS:
257
+ files.append(data_file)
258
+ elif os.path.basename(data_file) in self.METADATA_FILENAMES:
259
+ files.append(data_file)
260
+ else:
261
+ archives.append(data_file)
262
+ return files, archives
263
+
264
+ def _read_metadata(self, metadata_file: str, metadata_ext: str = "") -> Iterator[pa.Table]:
265
+ """using the same logic as the Csv, Json and Parquet dataset builders to stream the data"""
266
+ if self.config.filters is not None:
267
+ filter_expr = (
268
+ pq.filters_to_expression(self.config.filters)
269
+ if isinstance(self.config.filters, list)
270
+ else self.config.filters
271
+ )
272
+ else:
273
+ filter_expr = None
274
+ if metadata_ext == ".csv":
275
+ chunksize = 10_000 # 10k lines
276
+ schema = self.config.features.arrow_schema if self.config.features else None
277
+ # dtype allows reading an int column as str
278
+ dtype = (
279
+ {
280
+ name: dtype.to_pandas_dtype() if not require_storage_cast(feature) else object
281
+ for name, dtype, feature in zip(schema.names, schema.types, self.config.features.values())
282
+ }
283
+ if schema is not None
284
+ else None
285
+ )
286
+ csv_file_reader = pd.read_csv(metadata_file, iterator=True, dtype=dtype, chunksize=chunksize)
287
+ for df in csv_file_reader:
288
+ pa_table = pa.Table.from_pandas(df)
289
+ if self.config.filters is not None:
290
+ pa_table = pa_table.filter(filter_expr)
291
+ if len(pa_table) > 0:
292
+ yield pa_table
293
+ elif metadata_ext == ".jsonl":
294
+ with open(metadata_file, "rb") as f:
295
+ chunksize: int = 10 << 20 # 10MB
296
+ # Use block_size equal to the chunk size divided by 32 to leverage multithreading
297
+ # Set a default minimum value of 16kB if the chunk size is really small
298
+ block_size = max(chunksize // 32, 16 << 10)
299
+ while True:
300
+ batch = f.read(chunksize)
301
+ if not batch:
302
+ break
303
+ # Finish current line
304
+ try:
305
+ batch += f.readline()
306
+ except (AttributeError, io.UnsupportedOperation):
307
+ batch += readline(f)
308
+ while True:
309
+ try:
310
+ pa_table = paj.read_json(
311
+ io.BytesIO(batch), read_options=paj.ReadOptions(block_size=block_size)
312
+ )
313
+ break
314
+ except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
315
+ if (
316
+ isinstance(e, pa.ArrowInvalid)
317
+ and "straddling" not in str(e)
318
+ or block_size > len(batch)
319
+ ):
320
+ raise
321
+ else:
322
+ # Increase the block size in case it was too small.
323
+ # The block size will be reset for the next file.
324
+ logger.debug(
325
+ f"Batch of {len(batch)} bytes couldn't be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}."
326
+ )
327
+ block_size *= 2
328
+ if self.config.filters is not None:
329
+ pa_table = pa_table.filter(filter_expr)
330
+ if len(pa_table) > 0:
331
+ yield pa_table
332
+ else:
333
+ with open(metadata_file, "rb") as f:
334
+ parquet_fragment = ds.ParquetFileFormat().make_fragment(f)
335
+ if parquet_fragment.row_groups:
336
+ batch_size = parquet_fragment.row_groups[0].num_rows
337
+ else:
338
+ batch_size = config.DEFAULT_MAX_BATCH_SIZE
339
+ for record_batch in parquet_fragment.to_batches(
340
+ batch_size=batch_size,
341
+ filter=filter_expr,
342
+ batch_readahead=0,
343
+ fragment_readahead=0,
344
+ ):
345
+ yield pa.Table.from_batches([record_batch])
346
+
347
+ def _generate_examples(self, files, metadata_files, add_metadata, add_labels):
348
+ sample_idx = 0
349
+ if add_metadata:
350
+ feature_paths = []
351
+
352
+ def find_feature_path(feature, feature_path):
353
+ nonlocal feature_paths
354
+ if feature_path and isinstance(feature, self.BASE_FEATURE):
355
+ feature_paths.append(feature_path)
356
+
357
+ _visit_with_path(self.info.features, find_feature_path)
358
+
359
+ for original_metadata_file, downloaded_metadata_file in metadata_files:
360
+ metadata_ext = os.path.splitext(original_metadata_file or downloaded_metadata_file)[-1]
361
+ downloaded_metadata_dir = os.path.dirname(downloaded_metadata_file)
362
+
363
+ def set_feature(item, feature_path: _VisitPath):
364
+ if len(feature_path) == 2 and isinstance(feature_path[0], str) and feature_path[1] == 0:
365
+ item[feature_path[0]] = item.pop("file_names", None) or item.pop(
366
+ feature_path[0] + "_file_names", None
367
+ )
368
+ elif len(feature_path) == 1 and isinstance(feature_path[0], str):
369
+ item[feature_path[0]] = item.pop("file_name", None) or item.pop(
370
+ feature_path[0] + "_file_name", None
371
+ )
372
+ elif len(feature_path) == 0:
373
+ file_relpath = os.path.normpath(item).replace("\\", "/")
374
+ item = os.path.join(downloaded_metadata_dir, file_relpath)
375
+ return item
376
+
377
+ for pa_metadata_table in self._read_metadata(downloaded_metadata_file, metadata_ext=metadata_ext):
378
+ for sample in pa_metadata_table.to_pylist():
379
+ for feature_path in feature_paths:
380
+ _nested_apply(sample, feature_path, set_feature)
381
+ yield sample_idx, sample
382
+ sample_idx += 1
383
+ else:
384
+ if self.config.filters is not None:
385
+ filter_expr = (
386
+ pq.filters_to_expression(self.config.filters)
387
+ if isinstance(self.config.filters, list)
388
+ else self.config.filters
389
+ )
390
+ for original_file, downloaded_file_or_dir in files:
391
+ downloaded_files = [downloaded_file_or_dir] if original_file else downloaded_file_or_dir
392
+ for downloaded_file in downloaded_files:
393
+ original_file_ext = os.path.splitext(original_file or downloaded_file)[-1]
394
+ if original_file_ext.lower() not in self.EXTENSIONS:
395
+ continue
396
+ sample = {self.BASE_COLUMN_NAME: downloaded_file}
397
+ if add_labels:
398
+ sample["label"] = os.path.basename(os.path.dirname(original_file or downloaded_file))
399
+ if self.config.filters is not None:
400
+ pa_table = pa.Table.from_pylist([sample]).filter(filter_expr)
401
+ if len(pa_table) == 0:
402
+ continue
403
+ yield sample_idx, sample
404
+ sample_idx += 1
405
+
406
+
407
+ def _nested_apply(item: Any, feature_path: _VisitPath, func: Callable[[Any, _VisitPath], Any]):
408
+ # see _visit_with_path() to see how feature paths are constructed
409
+ item = func(item, feature_path)
410
+ if feature_path:
411
+ key = feature_path[0]
412
+ if key == 0:
413
+ for i in range(len(item)):
414
+ item[i] = _nested_apply(item[i], feature_path[1:], func)
415
+ else:
416
+ item[key] = _nested_apply(item[key], feature_path[1:], func)
417
+ return item
lib/python3.12/site-packages/datasets/packaged_modules/generator/__init__.py ADDED
File without changes
lib/python3.12/site-packages/datasets/packaged_modules/generator/__pycache__/generator.cpython-312.pyc ADDED
Binary file (2.56 kB). View file
 
lib/python3.12/site-packages/datasets/packaged_modules/generator/generator.py ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from dataclasses import dataclass
2
+ from typing import Callable, Optional
3
+
4
+ import datasets
5
+
6
+
7
+ @dataclass
8
+ class GeneratorConfig(datasets.BuilderConfig):
9
+ generator: Optional[Callable] = None
10
+ gen_kwargs: Optional[dict] = None
11
+ features: Optional[datasets.Features] = None
12
+ split: datasets.NamedSplit = datasets.Split.TRAIN
13
+
14
+ def __post_init__(self):
15
+ super().__post_init__()
16
+ if self.generator is None:
17
+ raise ValueError("generator must be specified")
18
+
19
+ if self.gen_kwargs is None:
20
+ self.gen_kwargs = {}
21
+
22
+
23
+ class Generator(datasets.GeneratorBasedBuilder):
24
+ BUILDER_CONFIG_CLASS = GeneratorConfig
25
+
26
+ def _info(self):
27
+ return datasets.DatasetInfo(features=self.config.features)
28
+
29
+ def _split_generators(self, dl_manager):
30
+ return [datasets.SplitGenerator(name=self.config.split, gen_kwargs=self.config.gen_kwargs)]
31
+
32
+ def _generate_examples(self, **gen_kwargs):
33
+ yield from enumerate(self.config.generator(**gen_kwargs))
lib/python3.12/site-packages/datasets/packaged_modules/imagefolder/__init__.py ADDED
File without changes
lib/python3.12/site-packages/datasets/packaged_modules/imagefolder/__pycache__/__init__.cpython-312.pyc ADDED
Binary file (216 Bytes). View file
 
lib/python3.12/site-packages/datasets/packaged_modules/imagefolder/__pycache__/imagefolder.cpython-312.pyc ADDED
Binary file (1.9 kB). View file
 
lib/python3.12/site-packages/datasets/packaged_modules/imagefolder/imagefolder.py ADDED
@@ -0,0 +1,103 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import datasets
2
+
3
+ from ..folder_based_builder import folder_based_builder
4
+
5
+
6
+ logger = datasets.utils.logging.get_logger(__name__)
7
+
8
+
9
+ class ImageFolderConfig(folder_based_builder.FolderBasedBuilderConfig):
10
+ """BuilderConfig for ImageFolder."""
11
+
12
+ drop_labels: bool = None
13
+ drop_metadata: bool = None
14
+
15
+ def __post_init__(self):
16
+ super().__post_init__()
17
+
18
+
19
+ class ImageFolder(folder_based_builder.FolderBasedBuilder):
20
+ BASE_FEATURE = datasets.Image
21
+ BASE_COLUMN_NAME = "image"
22
+ BUILDER_CONFIG_CLASS = ImageFolderConfig
23
+ EXTENSIONS: list[str] # definition at the bottom of the script
24
+
25
+
26
+ # Obtained with:
27
+ # ```
28
+ # import PIL.Image
29
+ # IMAGE_EXTENSIONS = []
30
+ # PIL.Image.init()
31
+ # for ext, format in PIL.Image.EXTENSION.items():
32
+ # if format in PIL.Image.OPEN:
33
+ # IMAGE_EXTENSIONS.append(ext[1:])
34
+ # ```
35
+ # We intentionally do not run this code on launch because:
36
+ # (1) Pillow is an optional dependency, so importing Pillow in global namespace is not allowed
37
+ # (2) To ensure the list of supported extensions is deterministic
38
+ IMAGE_EXTENSIONS = [
39
+ ".blp",
40
+ ".bmp",
41
+ ".dib",
42
+ ".bufr",
43
+ ".cur",
44
+ ".pcx",
45
+ ".dcx",
46
+ ".dds",
47
+ ".ps",
48
+ ".eps",
49
+ ".fit",
50
+ ".fits",
51
+ ".fli",
52
+ ".flc",
53
+ ".ftc",
54
+ ".ftu",
55
+ ".gbr",
56
+ ".gif",
57
+ ".grib",
58
+ # ".h5", # may contain zero or several images
59
+ # ".hdf", # may contain zero or several images
60
+ ".png",
61
+ ".apng",
62
+ ".jp2",
63
+ ".j2k",
64
+ ".jpc",
65
+ ".jpf",
66
+ ".jpx",
67
+ ".j2c",
68
+ ".icns",
69
+ ".ico",
70
+ ".im",
71
+ ".iim",
72
+ ".tif",
73
+ ".tiff",
74
+ ".jfif",
75
+ ".jpe",
76
+ ".jpg",
77
+ ".jpeg",
78
+ ".mpg",
79
+ ".mpeg",
80
+ ".msp",
81
+ ".pcd",
82
+ ".pxr",
83
+ ".pbm",
84
+ ".pgm",
85
+ ".ppm",
86
+ ".pnm",
87
+ ".psd",
88
+ ".bw",
89
+ ".rgb",
90
+ ".rgba",
91
+ ".sgi",
92
+ ".ras",
93
+ ".tga",
94
+ ".icb",
95
+ ".vda",
96
+ ".vst",
97
+ ".webp",
98
+ ".wmf",
99
+ ".emf",
100
+ ".xbm",
101
+ ".xpm",
102
+ ]
103
+ ImageFolder.EXTENSIONS = IMAGE_EXTENSIONS
lib/python3.12/site-packages/datasets/packaged_modules/sql/__init__.py ADDED
File without changes