code-HFD / src /psi /data /dataset.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
06f9561 verified
Raw History Blame Contribute Delete
8.67 kB
from __future__ import annotations
from typing import Any, Dict, TYPE_CHECKING
if TYPE_CHECKING:
from psi.config.config import DataConfig
from psi.data.sampler import DatasetSpec
from collections.abc import Iterator, Sized
import os
import torch
from torch.utils.data import Dataset as TorchDataset
from torch.utils.data import IterableDataset as TorchIterableDataset
class Dataset(torch.utils.data.Dataset):
def __init__(self, dataCfg: DataConfig, dataset: TorchDataset|Any, **kwargs) -> None:
self.raw_dataset = dataset
self.transform = dataCfg.transform
self.transform_kwargs = kwargs.get("transform_kwargs", {})
def __len__(self):
return len(self.raw_dataset) # type: ignore
def __getitem__(self, idx: int) -> dict[str, Any]:
raw = self.raw_dataset[idx]
data = self.transform(raw, **self.transform_kwargs)
if isinstance(raw, dict):
for key in ("task_name", "task_family"):
if key in raw and key not in data:
data[key] = raw[key]
return data
def __getitems__(self, indices) -> list[dict[str, Any]]:
"""Bulk-fetch a batch, including OSS shard hints when present.
PyTorch calls this method for map-style datasets when a DataLoader
uses automatic batching. The training ``MixtureDataset`` already had
an equivalent fast path; validation used to call ``__getitem__`` one
index at a time and therefore could not install OSS shard hints. The
tuple format is emitted by ``OssTaskBatchSampler``:
``(index, current_shard, next_shard, source_group, source_key,
local_index, offset, length)``.
"""
if not indices:
return []
raw_indices = []
hints = None
current_shard = None
next_shard = None
first = indices[0]
if isinstance(first, (tuple, list)):
if len(first) < 3:
raise ValueError(f"Invalid batched dataset index: {first!r}")
current_shard = str(first[1])
next_shard = str(first[2]) if first[2] else None
raw_indices = [int(index[0]) for index in indices]
if len(first) >= 8:
hints = [
(
str(index[3]),
str(index[4]),
int(index[5]),
int(index[6]),
int(index[7]),
)
for index in indices
]
else:
raw_indices = [int(index) for index in indices]
installed = False
if current_shard:
if hints is not None:
from psi.data.seed.omni_hidden_cache import (
install_omni_oss_batch_hints,
prefetch_omni_oss_shard,
)
# Validation consumes only a handful of batches per task, so
# cloning a 1--4 GiB shard in every rank/worker costs far more
# than reading the requested slices. Training's
# MixtureDataset keeps the full-shard path below unchanged.
eval_full_shard_cache = os.environ.get(
"PSI_OMNI_OSS_EVAL_FULL_SHARD_CACHE", "0"
).strip().lower() in {"1", "true", "yes", "on"}
install_omni_oss_batch_hints(
current_shard,
hints,
materialize_full_shard=eval_full_shard_cache,
)
prefetch_omni_oss_shard(next_shard)
installed = True
try:
output = []
for raw_index in raw_indices:
raw = self.raw_dataset[raw_index]
data = self.transform(raw, **self.transform_kwargs)
if isinstance(raw, dict):
for key in ("task_name", "task_family"):
if key in raw and key not in data:
data[key] = raw[key]
output.append(data)
return output
finally:
if installed:
from psi.data.seed.omni_hidden_cache import clear_omni_oss_batch_hints
clear_omni_oss_batch_hints()
def sample_shard_aware_indices(
self,
batch_size: int,
generator: torch.Generator,
lane_id: int = 0,
) -> list[int] | None:
sampler = getattr(self.raw_dataset, "sample_shard_aware_indices", None)
if callable(sampler):
return sampler(batch_size, generator, lane_id=lane_id)
return None
@property
def dataset_length(self) -> int:
return len(self.raw_dataset) # type: ignore
@property
def dataset_statistics(self) -> Dict[str, Any]:
return getattr(self.raw_dataset, "dataset_statistics", {})
class IterableDataset(torch.utils.data.IterableDataset):
def __init__(
self, dataCfg: DataConfig, dataset: TorchIterableDataset|Any, **kwargs
) -> None:
self.raw_dataset = dataset
self.transform = dataCfg.transform
self.transform_kwargs = kwargs.get("transform_kwargs", {})
def __iter__(self) -> Iterator[Any]:
for rlds_batch in self.raw_dataset:
try:
data = self.transform(rlds_batch, **self.transform_kwargs)
if isinstance(rlds_batch, dict):
for key in ("task_name", "task_family"):
if key in rlds_batch and key not in data:
data[key] = rlds_batch[key]
yield data
except Exception as e:
print(f"Error processing batch: {e}")
raise e
@property
def dataset_length(self) -> int:
return len(self.raw_dataset) # type: ignore
@property
def dataset_statistics(self) -> Dict[str, Any]:
return getattr(self.raw_dataset, "dataset_statistics", {})
class MixtureDataset(TorchDataset):
specs: list[DatasetSpec]
def __init__(self, datasets: dict[Dataset|IterableDataset, float], num_samples_per_epoch: int) -> None:
self.datasets = list(datasets.keys())
self.ratios = list(datasets.values())
self.num_samples_per_epoch = num_samples_per_epoch
def __getitem__(self, index):
assert isinstance(index, tuple) and len(index) >= 2, "Maybe forget to use batch sampler? see ../data/sampler.py"
dataset_id, sample_id = index[:2]
return self.datasets[dataset_id][sample_id]
def __getitems__(self, indices):
if not indices:
return []
dataset_ids = {int(index[0]) for index in indices}
if len(dataset_ids) != 1:
raise RuntimeError(f"MixtureDataset received a heterogeneous batch: dataset_ids={dataset_ids}")
dataset_id = next(iter(dataset_ids))
dataset = self.datasets[dataset_id]
first = indices[0]
hints_installed = False
if len(first) >= 4:
current_shard = str(first[2])
next_shard = str(first[3]) if first[3] else None
if len(first) >= 9:
from psi.data.seed.omni_hidden_cache import (
install_omni_oss_batch_hints,
prefetch_omni_oss_shard,
)
sample_hints = [
(str(index[4]), str(index[5]), int(index[6]), int(index[7]), int(index[8]))
for index in indices
]
install_omni_oss_batch_hints(current_shard, sample_hints)
prefetch_omni_oss_shard(next_shard)
hints_installed = True
try:
return [dataset[int(index[1])] for index in indices]
finally:
if hints_installed:
from psi.data.seed.omni_hidden_cache import clear_omni_oss_batch_hints
clear_omni_oss_batch_hints()
# Start after the current batch is complete so foreground
# data stays latency-prioritized. Only source motion arrays
# for the next video shard are staged; sample order and the
# transformed batch queue are unchanged.
from psi.data.seed.video_motion_local_cache import (
prefetch_video_motion_for_omni_shard,
)
prefetch_video_motion_for_omni_shard(dataset, next_shard)
def __len__(self):
# raise RuntimeError("Length is defined by the sampler")
# return sum(d.dataset_length for d in self.datasets)
return self.num_samples_per_epoch