from __future__ import annotations from typing import Any, Dict, TYPE_CHECKING if TYPE_CHECKING: from psi.config.config import DataConfig from psi.data.sampler import DatasetSpec from collections.abc import Iterator, Sized import os import torch from torch.utils.data import Dataset as TorchDataset from torch.utils.data import IterableDataset as TorchIterableDataset class Dataset(torch.utils.data.Dataset): def __init__(self, dataCfg: DataConfig, dataset: TorchDataset|Any, **kwargs) -> None: self.raw_dataset = dataset self.transform = dataCfg.transform self.transform_kwargs = kwargs.get("transform_kwargs", {}) def __len__(self): return len(self.raw_dataset) # type: ignore def __getitem__(self, idx: int) -> dict[str, Any]: raw = self.raw_dataset[idx] data = self.transform(raw, **self.transform_kwargs) if isinstance(raw, dict): for key in ("task_name", "task_family"): if key in raw and key not in data: data[key] = raw[key] return data def __getitems__(self, indices) -> list[dict[str, Any]]: """Bulk-fetch a batch, including OSS shard hints when present. PyTorch calls this method for map-style datasets when a DataLoader uses automatic batching. The training ``MixtureDataset`` already had an equivalent fast path; validation used to call ``__getitem__`` one index at a time and therefore could not install OSS shard hints. The tuple format is emitted by ``OssTaskBatchSampler``: ``(index, current_shard, next_shard, source_group, source_key, local_index, offset, length)``. """ if not indices: return [] raw_indices = [] hints = None current_shard = None next_shard = None first = indices[0] if isinstance(first, (tuple, list)): if len(first) < 3: raise ValueError(f"Invalid batched dataset index: {first!r}") current_shard = str(first[1]) next_shard = str(first[2]) if first[2] else None raw_indices = [int(index[0]) for index in indices] if len(first) >= 8: hints = [ ( str(index[3]), str(index[4]), int(index[5]), int(index[6]), int(index[7]), ) for index in indices ] else: raw_indices = [int(index) for index in indices] installed = False if current_shard: if hints is not None: from psi.data.seed.omni_hidden_cache import ( install_omni_oss_batch_hints, prefetch_omni_oss_shard, ) # Validation consumes only a handful of batches per task, so # cloning a 1--4 GiB shard in every rank/worker costs far more # than reading the requested slices. Training's # MixtureDataset keeps the full-shard path below unchanged. eval_full_shard_cache = os.environ.get( "PSI_OMNI_OSS_EVAL_FULL_SHARD_CACHE", "0" ).strip().lower() in {"1", "true", "yes", "on"} install_omni_oss_batch_hints( current_shard, hints, materialize_full_shard=eval_full_shard_cache, ) prefetch_omni_oss_shard(next_shard) installed = True try: output = [] for raw_index in raw_indices: raw = self.raw_dataset[raw_index] data = self.transform(raw, **self.transform_kwargs) if isinstance(raw, dict): for key in ("task_name", "task_family"): if key in raw and key not in data: data[key] = raw[key] output.append(data) return output finally: if installed: from psi.data.seed.omni_hidden_cache import clear_omni_oss_batch_hints clear_omni_oss_batch_hints() def sample_shard_aware_indices( self, batch_size: int, generator: torch.Generator, lane_id: int = 0, ) -> list[int] | None: sampler = getattr(self.raw_dataset, "sample_shard_aware_indices", None) if callable(sampler): return sampler(batch_size, generator, lane_id=lane_id) return None @property def dataset_length(self) -> int: return len(self.raw_dataset) # type: ignore @property def dataset_statistics(self) -> Dict[str, Any]: return getattr(self.raw_dataset, "dataset_statistics", {}) class IterableDataset(torch.utils.data.IterableDataset): def __init__( self, dataCfg: DataConfig, dataset: TorchIterableDataset|Any, **kwargs ) -> None: self.raw_dataset = dataset self.transform = dataCfg.transform self.transform_kwargs = kwargs.get("transform_kwargs", {}) def __iter__(self) -> Iterator[Any]: for rlds_batch in self.raw_dataset: try: data = self.transform(rlds_batch, **self.transform_kwargs) if isinstance(rlds_batch, dict): for key in ("task_name", "task_family"): if key in rlds_batch and key not in data: data[key] = rlds_batch[key] yield data except Exception as e: print(f"Error processing batch: {e}") raise e @property def dataset_length(self) -> int: return len(self.raw_dataset) # type: ignore @property def dataset_statistics(self) -> Dict[str, Any]: return getattr(self.raw_dataset, "dataset_statistics", {}) class MixtureDataset(TorchDataset): specs: list[DatasetSpec] def __init__(self, datasets: dict[Dataset|IterableDataset, float], num_samples_per_epoch: int) -> None: self.datasets = list(datasets.keys()) self.ratios = list(datasets.values()) self.num_samples_per_epoch = num_samples_per_epoch def __getitem__(self, index): assert isinstance(index, tuple) and len(index) >= 2, "Maybe forget to use batch sampler? see ../data/sampler.py" dataset_id, sample_id = index[:2] return self.datasets[dataset_id][sample_id] def __getitems__(self, indices): if not indices: return [] dataset_ids = {int(index[0]) for index in indices} if len(dataset_ids) != 1: raise RuntimeError(f"MixtureDataset received a heterogeneous batch: dataset_ids={dataset_ids}") dataset_id = next(iter(dataset_ids)) dataset = self.datasets[dataset_id] first = indices[0] hints_installed = False if len(first) >= 4: current_shard = str(first[2]) next_shard = str(first[3]) if first[3] else None if len(first) >= 9: from psi.data.seed.omni_hidden_cache import ( install_omni_oss_batch_hints, prefetch_omni_oss_shard, ) sample_hints = [ (str(index[4]), str(index[5]), int(index[6]), int(index[7]), int(index[8])) for index in indices ] install_omni_oss_batch_hints(current_shard, sample_hints) prefetch_omni_oss_shard(next_shard) hints_installed = True try: return [dataset[int(index[1])] for index in indices] finally: if hints_installed: from psi.data.seed.omni_hidden_cache import clear_omni_oss_batch_hints clear_omni_oss_batch_hints() # Start after the current batch is complete so foreground # data stays latency-prioritized. Only source motion arrays # for the next video shard are staged; sample order and the # transformed batch queue are unchanged. from psi.data.seed.video_motion_local_cache import ( prefetch_video_motion_for_omni_shard, ) prefetch_video_motion_for_omni_shard(dataset, next_shard) def __len__(self): # raise RuntimeError("Length is defined by the sampler") # return sum(d.dataset_length for d in self.datasets) return self.num_samples_per_epoch