Download src/psi/data/dataset.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 8.67 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/src/psi/data/dataset.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/src/psi/data/dataset.py
-
curl -L -o dataset.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/src/psi/data/dataset.py
8.67 kB
| from __future__ import annotations | |
| from typing import Any, Dict, TYPE_CHECKING | |
| if TYPE_CHECKING: | |
| from psi.config.config import DataConfig | |
| from psi.data.sampler import DatasetSpec | |
| from collections.abc import Iterator, Sized | |
| import os | |
| import torch | |
| from torch.utils.data import Dataset as TorchDataset | |
| from torch.utils.data import IterableDataset as TorchIterableDataset | |
| class Dataset(torch.utils.data.Dataset): | |
| def __init__(self, dataCfg: DataConfig, dataset: TorchDataset|Any, **kwargs) -> None: | |
| self.raw_dataset = dataset | |
| self.transform = dataCfg.transform | |
| self.transform_kwargs = kwargs.get("transform_kwargs", {}) | |
| def __len__(self): | |
| return len(self.raw_dataset) # type: ignore | |
| def __getitem__(self, idx: int) -> dict[str, Any]: | |
| raw = self.raw_dataset[idx] | |
| data = self.transform(raw, **self.transform_kwargs) | |
| if isinstance(raw, dict): | |
| for key in ("task_name", "task_family"): | |
| if key in raw and key not in data: | |
| data[key] = raw[key] | |
| return data | |
| def __getitems__(self, indices) -> list[dict[str, Any]]: | |
| """Bulk-fetch a batch, including OSS shard hints when present. | |
| PyTorch calls this method for map-style datasets when a DataLoader | |
| uses automatic batching. The training ``MixtureDataset`` already had | |
| an equivalent fast path; validation used to call ``__getitem__`` one | |
| index at a time and therefore could not install OSS shard hints. The | |
| tuple format is emitted by ``OssTaskBatchSampler``: | |
| ``(index, current_shard, next_shard, source_group, source_key, | |
| local_index, offset, length)``. | |
| """ | |
| if not indices: | |
| return [] | |
| raw_indices = [] | |
| hints = None | |
| current_shard = None | |
| next_shard = None | |
| first = indices[0] | |
| if isinstance(first, (tuple, list)): | |
| if len(first) < 3: | |
| raise ValueError(f"Invalid batched dataset index: {first!r}") | |
| current_shard = str(first[1]) | |
| next_shard = str(first[2]) if first[2] else None | |
| raw_indices = [int(index[0]) for index in indices] | |
| if len(first) >= 8: | |
| hints = [ | |
| ( | |
| str(index[3]), | |
| str(index[4]), | |
| int(index[5]), | |
| int(index[6]), | |
| int(index[7]), | |
| ) | |
| for index in indices | |
| ] | |
| else: | |
| raw_indices = [int(index) for index in indices] | |
| installed = False | |
| if current_shard: | |
| if hints is not None: | |
| from psi.data.seed.omni_hidden_cache import ( | |
| install_omni_oss_batch_hints, | |
| prefetch_omni_oss_shard, | |
| ) | |
| # Validation consumes only a handful of batches per task, so | |
| # cloning a 1--4 GiB shard in every rank/worker costs far more | |
| # than reading the requested slices. Training's | |
| # MixtureDataset keeps the full-shard path below unchanged. | |
| eval_full_shard_cache = os.environ.get( | |
| "PSI_OMNI_OSS_EVAL_FULL_SHARD_CACHE", "0" | |
| ).strip().lower() in {"1", "true", "yes", "on"} | |
| install_omni_oss_batch_hints( | |
| current_shard, | |
| hints, | |
| materialize_full_shard=eval_full_shard_cache, | |
| ) | |
| prefetch_omni_oss_shard(next_shard) | |
| installed = True | |
| try: | |
| output = [] | |
| for raw_index in raw_indices: | |
| raw = self.raw_dataset[raw_index] | |
| data = self.transform(raw, **self.transform_kwargs) | |
| if isinstance(raw, dict): | |
| for key in ("task_name", "task_family"): | |
| if key in raw and key not in data: | |
| data[key] = raw[key] | |
| output.append(data) | |
| return output | |
| finally: | |
| if installed: | |
| from psi.data.seed.omni_hidden_cache import clear_omni_oss_batch_hints | |
| clear_omni_oss_batch_hints() | |
| def sample_shard_aware_indices( | |
| self, | |
| batch_size: int, | |
| generator: torch.Generator, | |
| lane_id: int = 0, | |
| ) -> list[int] | None: | |
| sampler = getattr(self.raw_dataset, "sample_shard_aware_indices", None) | |
| if callable(sampler): | |
| return sampler(batch_size, generator, lane_id=lane_id) | |
| return None | |
| def dataset_length(self) -> int: | |
| return len(self.raw_dataset) # type: ignore | |
| def dataset_statistics(self) -> Dict[str, Any]: | |
| return getattr(self.raw_dataset, "dataset_statistics", {}) | |
| class IterableDataset(torch.utils.data.IterableDataset): | |
| def __init__( | |
| self, dataCfg: DataConfig, dataset: TorchIterableDataset|Any, **kwargs | |
| ) -> None: | |
| self.raw_dataset = dataset | |
| self.transform = dataCfg.transform | |
| self.transform_kwargs = kwargs.get("transform_kwargs", {}) | |
| def __iter__(self) -> Iterator[Any]: | |
| for rlds_batch in self.raw_dataset: | |
| try: | |
| data = self.transform(rlds_batch, **self.transform_kwargs) | |
| if isinstance(rlds_batch, dict): | |
| for key in ("task_name", "task_family"): | |
| if key in rlds_batch and key not in data: | |
| data[key] = rlds_batch[key] | |
| yield data | |
| except Exception as e: | |
| print(f"Error processing batch: {e}") | |
| raise e | |
| def dataset_length(self) -> int: | |
| return len(self.raw_dataset) # type: ignore | |
| def dataset_statistics(self) -> Dict[str, Any]: | |
| return getattr(self.raw_dataset, "dataset_statistics", {}) | |
| class MixtureDataset(TorchDataset): | |
| specs: list[DatasetSpec] | |
| def __init__(self, datasets: dict[Dataset|IterableDataset, float], num_samples_per_epoch: int) -> None: | |
| self.datasets = list(datasets.keys()) | |
| self.ratios = list(datasets.values()) | |
| self.num_samples_per_epoch = num_samples_per_epoch | |
| def __getitem__(self, index): | |
| assert isinstance(index, tuple) and len(index) >= 2, "Maybe forget to use batch sampler? see ../data/sampler.py" | |
| dataset_id, sample_id = index[:2] | |
| return self.datasets[dataset_id][sample_id] | |
| def __getitems__(self, indices): | |
| if not indices: | |
| return [] | |
| dataset_ids = {int(index[0]) for index in indices} | |
| if len(dataset_ids) != 1: | |
| raise RuntimeError(f"MixtureDataset received a heterogeneous batch: dataset_ids={dataset_ids}") | |
| dataset_id = next(iter(dataset_ids)) | |
| dataset = self.datasets[dataset_id] | |
| first = indices[0] | |
| hints_installed = False | |
| if len(first) >= 4: | |
| current_shard = str(first[2]) | |
| next_shard = str(first[3]) if first[3] else None | |
| if len(first) >= 9: | |
| from psi.data.seed.omni_hidden_cache import ( | |
| install_omni_oss_batch_hints, | |
| prefetch_omni_oss_shard, | |
| ) | |
| sample_hints = [ | |
| (str(index[4]), str(index[5]), int(index[6]), int(index[7]), int(index[8])) | |
| for index in indices | |
| ] | |
| install_omni_oss_batch_hints(current_shard, sample_hints) | |
| prefetch_omni_oss_shard(next_shard) | |
| hints_installed = True | |
| try: | |
| return [dataset[int(index[1])] for index in indices] | |
| finally: | |
| if hints_installed: | |
| from psi.data.seed.omni_hidden_cache import clear_omni_oss_batch_hints | |
| clear_omni_oss_batch_hints() | |
| # Start after the current batch is complete so foreground | |
| # data stays latency-prioritized. Only source motion arrays | |
| # for the next video shard are staged; sample order and the | |
| # transformed batch queue are unchanged. | |
| from psi.data.seed.video_motion_local_cache import ( | |
| prefetch_video_motion_for_omni_shard, | |
| ) | |
| prefetch_video_motion_for_omni_shard(dataset, next_shard) | |
| def __len__(self): | |
| # raise RuntimeError("Length is defined by the sampler") | |
| # return sum(d.dataset_length for d in self.datasets) | |
| return self.num_samples_per_epoch | |