Download src/psi/data/dataset.py from Tinghong-Ye22/code-Psi0-yth-v22: direct link, hf CLI and curl.
- Browser
- Download file 3.19 kB
-
https://huggingface.co/Tinghong-Ye22/code-Psi0-yth-v22/resolve/main/src/psi/data/dataset.py
- Command line
-
hf download hf://Tinghong-Ye22/code-Psi0-yth-v22/src/psi/data/dataset.py
-
curl -L -o dataset.py https://huggingface.co/Tinghong-Ye22/code-Psi0-yth-v22/resolve/main/src/psi/data/dataset.py
3.19 kB
| from __future__ import annotations | |
| from typing import Any, Dict, TYPE_CHECKING | |
| if TYPE_CHECKING: | |
| from psi.config.config import DataConfig | |
| from psi.data.sampler import DatasetSpec | |
| from collections.abc import Iterator, Sized | |
| import torch | |
| from torch.utils.data import Dataset as TorchDataset | |
| from torch.utils.data import IterableDataset as TorchIterableDataset | |
| class Dataset(torch.utils.data.Dataset): | |
| def __init__(self, dataCfg: DataConfig, dataset: TorchDataset|Any, **kwargs) -> None: | |
| self.raw_dataset = dataset | |
| self.transform = dataCfg.transform | |
| self.transform_kwargs = kwargs.get("transform_kwargs", {}) | |
| def __len__(self): | |
| return len(self.raw_dataset) # type: ignore | |
| def __getitem__(self, idx: int) -> dict[str, Any]: | |
| raw = self.raw_dataset[idx] | |
| data = self.transform(raw, **self.transform_kwargs) | |
| if isinstance(raw, dict): | |
| for key in ("task_name", "task_family"): | |
| if key in raw and key not in data: | |
| data[key] = raw[key] | |
| return data | |
| def dataset_length(self) -> int: | |
| return len(self.raw_dataset) # type: ignore | |
| def dataset_statistics(self) -> Dict[str, Any]: | |
| return getattr(self.raw_dataset, "dataset_statistics", {}) | |
| class IterableDataset(torch.utils.data.IterableDataset): | |
| def __init__( | |
| self, dataCfg: DataConfig, dataset: TorchIterableDataset|Any, **kwargs | |
| ) -> None: | |
| self.raw_dataset = dataset | |
| self.transform = dataCfg.transform | |
| self.transform_kwargs = kwargs.get("transform_kwargs", {}) | |
| def __iter__(self) -> Iterator[Any]: | |
| for rlds_batch in self.raw_dataset: | |
| try: | |
| data = self.transform(rlds_batch, **self.transform_kwargs) | |
| if isinstance(rlds_batch, dict): | |
| for key in ("task_name", "task_family"): | |
| if key in rlds_batch and key not in data: | |
| data[key] = rlds_batch[key] | |
| yield data | |
| except Exception as e: | |
| print(f"Error processing batch: {e}") | |
| raise e | |
| def dataset_length(self) -> int: | |
| return len(self.raw_dataset) # type: ignore | |
| def dataset_statistics(self) -> Dict[str, Any]: | |
| return getattr(self.raw_dataset, "dataset_statistics", {}) | |
| class MixtureDataset(TorchDataset): | |
| specs: list[DatasetSpec] | |
| def __init__(self, datasets: dict[Dataset|IterableDataset, float], num_samples_per_epoch: int) -> None: | |
| self.datasets = list(datasets.keys()) | |
| self.ratios = list(datasets.values()) | |
| self.num_samples_per_epoch = num_samples_per_epoch | |
| def __getitem__(self, index): | |
| assert isinstance(index, tuple) and len(index) == 2, "Maybe forget to use batch sampler? see ../data/sampler.py" | |
| dataset_id, sample_id = index | |
| return self.datasets[dataset_id][sample_id] | |
| def __len__(self): | |
| # raise RuntimeError("Length is defined by the sampler") | |
| # return sum(d.dataset_length for d in self.datasets) | |
| return self.num_samples_per_epoch | |