File size: 8,668 Bytes
06f9561
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
from __future__ import annotations
from typing import Any, Dict, TYPE_CHECKING

if TYPE_CHECKING:
    from psi.config.config import DataConfig
    from psi.data.sampler import DatasetSpec


from collections.abc import Iterator, Sized
import os
import torch
from torch.utils.data import Dataset as TorchDataset
from torch.utils.data import IterableDataset as TorchIterableDataset

class Dataset(torch.utils.data.Dataset):
    def __init__(self, dataCfg: DataConfig, dataset: TorchDataset|Any, **kwargs) -> None:
        self.raw_dataset = dataset
        self.transform = dataCfg.transform
        self.transform_kwargs = kwargs.get("transform_kwargs", {})

    def __len__(self):
        return len(self.raw_dataset)  # type: ignore

    def __getitem__(self, idx: int) -> dict[str, Any]:
        raw = self.raw_dataset[idx]
        data = self.transform(raw, **self.transform_kwargs)
        if isinstance(raw, dict):
            for key in ("task_name", "task_family"):
                if key in raw and key not in data:
                    data[key] = raw[key]
        return data

    def __getitems__(self, indices) -> list[dict[str, Any]]:
        """Bulk-fetch a batch, including OSS shard hints when present.

        PyTorch calls this method for map-style datasets when a DataLoader
        uses automatic batching.  The training ``MixtureDataset`` already had
        an equivalent fast path; validation used to call ``__getitem__`` one
        index at a time and therefore could not install OSS shard hints.  The
        tuple format is emitted by ``OssTaskBatchSampler``:

        ``(index, current_shard, next_shard, source_group, source_key,
        local_index, offset, length)``.
        """
        if not indices:
            return []

        raw_indices = []
        hints = None
        current_shard = None
        next_shard = None
        first = indices[0]
        if isinstance(first, (tuple, list)):
            if len(first) < 3:
                raise ValueError(f"Invalid batched dataset index: {first!r}")
            current_shard = str(first[1])
            next_shard = str(first[2]) if first[2] else None
            raw_indices = [int(index[0]) for index in indices]
            if len(first) >= 8:
                hints = [
                    (
                        str(index[3]),
                        str(index[4]),
                        int(index[5]),
                        int(index[6]),
                        int(index[7]),
                    )
                    for index in indices
                ]
        else:
            raw_indices = [int(index) for index in indices]

        installed = False
        if current_shard:
            if hints is not None:
                from psi.data.seed.omni_hidden_cache import (
                    install_omni_oss_batch_hints,
                    prefetch_omni_oss_shard,
                )

                # Validation consumes only a handful of batches per task, so
                # cloning a 1--4 GiB shard in every rank/worker costs far more
                # than reading the requested slices.  Training's
                # MixtureDataset keeps the full-shard path below unchanged.
                eval_full_shard_cache = os.environ.get(
                    "PSI_OMNI_OSS_EVAL_FULL_SHARD_CACHE", "0"
                ).strip().lower() in {"1", "true", "yes", "on"}
                install_omni_oss_batch_hints(
                    current_shard,
                    hints,
                    materialize_full_shard=eval_full_shard_cache,
                )
                prefetch_omni_oss_shard(next_shard)
                installed = True

        try:
            output = []
            for raw_index in raw_indices:
                raw = self.raw_dataset[raw_index]
                data = self.transform(raw, **self.transform_kwargs)
                if isinstance(raw, dict):
                    for key in ("task_name", "task_family"):
                        if key in raw and key not in data:
                            data[key] = raw[key]
                output.append(data)
            return output
        finally:
            if installed:
                from psi.data.seed.omni_hidden_cache import clear_omni_oss_batch_hints

                clear_omni_oss_batch_hints()

    def sample_shard_aware_indices(
        self,
        batch_size: int,
        generator: torch.Generator,
        lane_id: int = 0,
    ) -> list[int] | None:
        sampler = getattr(self.raw_dataset, "sample_shard_aware_indices", None)
        if callable(sampler):
            return sampler(batch_size, generator, lane_id=lane_id)
        return None

    @property
    def dataset_length(self) -> int:
        return len(self.raw_dataset) # type: ignore

    @property
    def dataset_statistics(self) -> Dict[str, Any]:
        return getattr(self.raw_dataset, "dataset_statistics", {})

class IterableDataset(torch.utils.data.IterableDataset):

    def __init__(
        self, dataCfg: DataConfig, dataset: TorchIterableDataset|Any, **kwargs
    ) -> None:
        self.raw_dataset = dataset
        self.transform = dataCfg.transform
        self.transform_kwargs = kwargs.get("transform_kwargs", {})

    def __iter__(self) -> Iterator[Any]:
        for rlds_batch in self.raw_dataset:
            try:
                data = self.transform(rlds_batch, **self.transform_kwargs)
                if isinstance(rlds_batch, dict):
                    for key in ("task_name", "task_family"):
                        if key in rlds_batch and key not in data:
                            data[key] = rlds_batch[key]
                yield data
            except Exception as e:
                print(f"Error processing batch: {e}")
                raise e

    @property
    def dataset_length(self) -> int:
        return len(self.raw_dataset) # type: ignore

    @property
    def dataset_statistics(self) -> Dict[str, Any]:
        return getattr(self.raw_dataset, "dataset_statistics", {})


class MixtureDataset(TorchDataset):

    specs: list[DatasetSpec]
    
    def __init__(self, datasets: dict[Dataset|IterableDataset, float], num_samples_per_epoch: int) -> None:
        self.datasets = list(datasets.keys())
        self.ratios = list(datasets.values())
        self.num_samples_per_epoch = num_samples_per_epoch

    def __getitem__(self, index):
        assert isinstance(index, tuple) and len(index) >= 2, "Maybe forget to use batch sampler? see ../data/sampler.py"
        dataset_id, sample_id = index[:2]
        return self.datasets[dataset_id][sample_id]

    def __getitems__(self, indices):
        if not indices:
            return []
        dataset_ids = {int(index[0]) for index in indices}
        if len(dataset_ids) != 1:
            raise RuntimeError(f"MixtureDataset received a heterogeneous batch: dataset_ids={dataset_ids}")
        dataset_id = next(iter(dataset_ids))
        dataset = self.datasets[dataset_id]
        first = indices[0]
        hints_installed = False
        if len(first) >= 4:
            current_shard = str(first[2])
            next_shard = str(first[3]) if first[3] else None
            if len(first) >= 9:
                from psi.data.seed.omni_hidden_cache import (
                    install_omni_oss_batch_hints,
                    prefetch_omni_oss_shard,
                )

                sample_hints = [
                    (str(index[4]), str(index[5]), int(index[6]), int(index[7]), int(index[8]))
                    for index in indices
                ]
                install_omni_oss_batch_hints(current_shard, sample_hints)
                prefetch_omni_oss_shard(next_shard)
                hints_installed = True
        try:
            return [dataset[int(index[1])] for index in indices]
        finally:
            if hints_installed:
                from psi.data.seed.omni_hidden_cache import clear_omni_oss_batch_hints

                clear_omni_oss_batch_hints()
                # Start after the current batch is complete so foreground
                # data stays latency-prioritized. Only source motion arrays
                # for the next video shard are staged; sample order and the
                # transformed batch queue are unchanged.
                from psi.data.seed.video_motion_local_cache import (
                    prefetch_video_motion_for_omni_shard,
                )

                prefetch_video_motion_for_omni_shard(dataset, next_shard)

    def __len__(self):
        # raise RuntimeError("Length is defined by the sampler")
        # return sum(d.dataset_length for d in self.datasets)
        return self.num_samples_per_epoch