File size: 8,668 Bytes
06f9561 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 | from __future__ import annotations
from typing import Any, Dict, TYPE_CHECKING
if TYPE_CHECKING:
from psi.config.config import DataConfig
from psi.data.sampler import DatasetSpec
from collections.abc import Iterator, Sized
import os
import torch
from torch.utils.data import Dataset as TorchDataset
from torch.utils.data import IterableDataset as TorchIterableDataset
class Dataset(torch.utils.data.Dataset):
def __init__(self, dataCfg: DataConfig, dataset: TorchDataset|Any, **kwargs) -> None:
self.raw_dataset = dataset
self.transform = dataCfg.transform
self.transform_kwargs = kwargs.get("transform_kwargs", {})
def __len__(self):
return len(self.raw_dataset) # type: ignore
def __getitem__(self, idx: int) -> dict[str, Any]:
raw = self.raw_dataset[idx]
data = self.transform(raw, **self.transform_kwargs)
if isinstance(raw, dict):
for key in ("task_name", "task_family"):
if key in raw and key not in data:
data[key] = raw[key]
return data
def __getitems__(self, indices) -> list[dict[str, Any]]:
"""Bulk-fetch a batch, including OSS shard hints when present.
PyTorch calls this method for map-style datasets when a DataLoader
uses automatic batching. The training ``MixtureDataset`` already had
an equivalent fast path; validation used to call ``__getitem__`` one
index at a time and therefore could not install OSS shard hints. The
tuple format is emitted by ``OssTaskBatchSampler``:
``(index, current_shard, next_shard, source_group, source_key,
local_index, offset, length)``.
"""
if not indices:
return []
raw_indices = []
hints = None
current_shard = None
next_shard = None
first = indices[0]
if isinstance(first, (tuple, list)):
if len(first) < 3:
raise ValueError(f"Invalid batched dataset index: {first!r}")
current_shard = str(first[1])
next_shard = str(first[2]) if first[2] else None
raw_indices = [int(index[0]) for index in indices]
if len(first) >= 8:
hints = [
(
str(index[3]),
str(index[4]),
int(index[5]),
int(index[6]),
int(index[7]),
)
for index in indices
]
else:
raw_indices = [int(index) for index in indices]
installed = False
if current_shard:
if hints is not None:
from psi.data.seed.omni_hidden_cache import (
install_omni_oss_batch_hints,
prefetch_omni_oss_shard,
)
# Validation consumes only a handful of batches per task, so
# cloning a 1--4 GiB shard in every rank/worker costs far more
# than reading the requested slices. Training's
# MixtureDataset keeps the full-shard path below unchanged.
eval_full_shard_cache = os.environ.get(
"PSI_OMNI_OSS_EVAL_FULL_SHARD_CACHE", "0"
).strip().lower() in {"1", "true", "yes", "on"}
install_omni_oss_batch_hints(
current_shard,
hints,
materialize_full_shard=eval_full_shard_cache,
)
prefetch_omni_oss_shard(next_shard)
installed = True
try:
output = []
for raw_index in raw_indices:
raw = self.raw_dataset[raw_index]
data = self.transform(raw, **self.transform_kwargs)
if isinstance(raw, dict):
for key in ("task_name", "task_family"):
if key in raw and key not in data:
data[key] = raw[key]
output.append(data)
return output
finally:
if installed:
from psi.data.seed.omni_hidden_cache import clear_omni_oss_batch_hints
clear_omni_oss_batch_hints()
def sample_shard_aware_indices(
self,
batch_size: int,
generator: torch.Generator,
lane_id: int = 0,
) -> list[int] | None:
sampler = getattr(self.raw_dataset, "sample_shard_aware_indices", None)
if callable(sampler):
return sampler(batch_size, generator, lane_id=lane_id)
return None
@property
def dataset_length(self) -> int:
return len(self.raw_dataset) # type: ignore
@property
def dataset_statistics(self) -> Dict[str, Any]:
return getattr(self.raw_dataset, "dataset_statistics", {})
class IterableDataset(torch.utils.data.IterableDataset):
def __init__(
self, dataCfg: DataConfig, dataset: TorchIterableDataset|Any, **kwargs
) -> None:
self.raw_dataset = dataset
self.transform = dataCfg.transform
self.transform_kwargs = kwargs.get("transform_kwargs", {})
def __iter__(self) -> Iterator[Any]:
for rlds_batch in self.raw_dataset:
try:
data = self.transform(rlds_batch, **self.transform_kwargs)
if isinstance(rlds_batch, dict):
for key in ("task_name", "task_family"):
if key in rlds_batch and key not in data:
data[key] = rlds_batch[key]
yield data
except Exception as e:
print(f"Error processing batch: {e}")
raise e
@property
def dataset_length(self) -> int:
return len(self.raw_dataset) # type: ignore
@property
def dataset_statistics(self) -> Dict[str, Any]:
return getattr(self.raw_dataset, "dataset_statistics", {})
class MixtureDataset(TorchDataset):
specs: list[DatasetSpec]
def __init__(self, datasets: dict[Dataset|IterableDataset, float], num_samples_per_epoch: int) -> None:
self.datasets = list(datasets.keys())
self.ratios = list(datasets.values())
self.num_samples_per_epoch = num_samples_per_epoch
def __getitem__(self, index):
assert isinstance(index, tuple) and len(index) >= 2, "Maybe forget to use batch sampler? see ../data/sampler.py"
dataset_id, sample_id = index[:2]
return self.datasets[dataset_id][sample_id]
def __getitems__(self, indices):
if not indices:
return []
dataset_ids = {int(index[0]) for index in indices}
if len(dataset_ids) != 1:
raise RuntimeError(f"MixtureDataset received a heterogeneous batch: dataset_ids={dataset_ids}")
dataset_id = next(iter(dataset_ids))
dataset = self.datasets[dataset_id]
first = indices[0]
hints_installed = False
if len(first) >= 4:
current_shard = str(first[2])
next_shard = str(first[3]) if first[3] else None
if len(first) >= 9:
from psi.data.seed.omni_hidden_cache import (
install_omni_oss_batch_hints,
prefetch_omni_oss_shard,
)
sample_hints = [
(str(index[4]), str(index[5]), int(index[6]), int(index[7]), int(index[8]))
for index in indices
]
install_omni_oss_batch_hints(current_shard, sample_hints)
prefetch_omni_oss_shard(next_shard)
hints_installed = True
try:
return [dataset[int(index[1])] for index in indices]
finally:
if hints_installed:
from psi.data.seed.omni_hidden_cache import clear_omni_oss_batch_hints
clear_omni_oss_batch_hints()
# Start after the current batch is complete so foreground
# data stays latency-prioritized. Only source motion arrays
# for the next video shard are staged; sample order and the
# transformed batch queue are unchanged.
from psi.data.seed.video_motion_local_cache import (
prefetch_video_motion_for_omni_shard,
)
prefetch_video_motion_for_omni_shard(dataset, next_shard)
def __len__(self):
# raise RuntimeError("Length is defined by the sampler")
# return sum(d.dataset_length for d in self.datasets)
return self.num_samples_per_epoch
|