Download src/psi/data/sampler.py from Tinghong-Ye22/code-Movens-Zero_train: direct link, hf CLI and curl.
- Browser
- Download file 31.8 kB
-
https://huggingface.co/Tinghong-Ye22/code-Movens-Zero_train/resolve/main/src/psi/data/sampler.py
- Command line
-
hf download hf://Tinghong-Ye22/code-Movens-Zero_train/src/psi/data/sampler.py
-
curl -L -o sampler.py https://huggingface.co/Tinghong-Ye22/code-Movens-Zero_train/resolve/main/src/psi/data/sampler.py
31.8 kB
| import torch | |
| from torch.utils.data import Sampler | |
| from accelerate import Accelerator | |
| import math | |
| import os | |
| import random | |
| from dataclasses import dataclass | |
| from psi.utils import initialize_overwatch | |
| overwatch = initialize_overwatch(__name__) | |
| def _normalize_positive_weights(mixture_ratios): | |
| weights = torch.tensor(mixture_ratios, dtype=torch.double) | |
| if weights.numel() == 0: | |
| raise ValueError("Sampler received empty mixture ratios") | |
| if torch.any(weights < 0): | |
| raise ValueError(f"Sampler received negative mixture ratios: {mixture_ratios}") | |
| weight_sum = weights.sum() | |
| if weight_sum.item() <= 0: | |
| raise ValueError("Sampler received no positive mixture ratios") | |
| weights /= weight_sum | |
| return weights | |
| class BatchMixtureSampler(Sampler): | |
| """ | |
| Multi-node safe dataset mixture sampler. | |
| Example: datasets = [ds1, ds2], ratio = [4, 1] | |
| """ | |
| def __init__(self, dataset_lens, mixture_ratios, num_samples_per_epoch, batch_size, seed=42): | |
| self.dataset_lens = dataset_lens | |
| self.weights = _normalize_positive_weights(mixture_ratios) | |
| self.num_samples = num_samples_per_epoch | |
| self.batch_size = batch_size | |
| self.seed = seed | |
| # Distributed settings | |
| self.rank = int(os.environ.get("RANK", 0)) | |
| self.world_size = int(os.environ.get("WORLD_SIZE", 1)) | |
| # each rank gets this many samples | |
| self.num_samples_rank = math.ceil(self.num_samples / self.world_size) | |
| self.epoch = 0 | |
| self.batch_counts = self._build_batch_counts() | |
| def _build_batch_counts(self): | |
| positive_ids = [i for i, weight in enumerate(self.weights.tolist()) if weight > 0] | |
| if not positive_ids: | |
| raise ValueError("BatchMixtureSampler received no positive mixture ratios") | |
| counts = torch.zeros(len(self.dataset_lens), dtype=torch.long) | |
| if self.batch_size >= len(positive_ids): | |
| counts[positive_ids] = 1 | |
| remaining = self.batch_size - len(positive_ids) | |
| if remaining > 0: | |
| positive_weights = self.weights[positive_ids] | |
| positive_weights = positive_weights / positive_weights.sum() | |
| raw_extra = positive_weights * remaining | |
| extra_counts = torch.floor(raw_extra).long() | |
| counts[positive_ids] += extra_counts | |
| extra_left = remaining - int(extra_counts.sum().item()) | |
| if extra_left > 0: | |
| frac = raw_extra - extra_counts.double() | |
| for frac_index in torch.argsort(frac, descending=True)[:extra_left].tolist(): | |
| counts[positive_ids[frac_index]] += 1 | |
| else: | |
| for dataset_id in torch.argsort(self.weights, descending=True)[:self.batch_size].tolist(): | |
| counts[dataset_id] += 1 | |
| return counts.tolist() | |
| def set_epoch(self, epoch): | |
| self.epoch = epoch | |
| def __iter__(self): | |
| num_batches_rank = len(self) | |
| total_batches = num_batches_rank * self.world_size | |
| for batch_idx in range(total_batches): | |
| if batch_idx % self.world_size != self.rank: | |
| continue | |
| g = torch.Generator() | |
| g.manual_seed(self.seed + self.epoch * total_batches + batch_idx) | |
| batch = [] | |
| for dataset_id, count in enumerate(self.batch_counts): | |
| if count <= 0: | |
| continue | |
| dataset_len = int(self.dataset_lens[dataset_id]) | |
| rand_indices = torch.randint( | |
| low=0, | |
| high=dataset_len, | |
| size=(count,), | |
| generator=g, | |
| ).tolist() | |
| batch.extend((dataset_id, sample_id) for sample_id in rand_indices) | |
| shuffle_order = torch.randperm(len(batch), generator=g).tolist() | |
| yield [batch[i] for i in shuffle_order] | |
| def __len__(self): | |
| return math.ceil(self.num_samples_rank / self.batch_size) | |
| class TaskMixtureSampler(Sampler): | |
| """ | |
| Multi-node safe dataset mixture sampler that assigns exactly one dataset to | |
| each global batch, then samples a full homogeneous batch from that dataset. | |
| """ | |
| def __init__(self, dataset_lens, mixture_ratios, num_samples_per_epoch, batch_size, seed=42, datasets=None): | |
| self.dataset_lens = dataset_lens | |
| self.datasets = datasets | |
| self.weights = _normalize_positive_weights(mixture_ratios) | |
| self.num_samples = num_samples_per_epoch | |
| self.batch_size = batch_size | |
| self.seed = seed | |
| self.sync_across_ranks = os.environ.get("PSI_TASK_MIXTURE_SYNC_RANKS", "0").strip().lower() in { | |
| "1", | |
| "true", | |
| "yes", | |
| "on", | |
| } | |
| self.shard_aware = os.environ.get("PSI_TASK_MIXTURE_SHARD_AWARE", "0").strip().lower() in { | |
| "1", | |
| "true", | |
| "yes", | |
| "on", | |
| } | |
| self.task_block_batches = max(int(os.environ.get("PSI_TASK_MIXTURE_BLOCK_BATCHES", "1")), 1) | |
| self.worker_lanes = max(int(os.environ.get("PSI_OMNI_OSS_WORKER_LANES", "1")), 1) | |
| self.rank_local_blocks = os.environ.get("PSI_TASK_MIXTURE_RANK_LOCAL_BLOCKS", "0").strip().lower() in { | |
| "1", | |
| "true", | |
| "yes", | |
| "on", | |
| } | |
| # In OSS training, long task blocks are useful for shard reuse but can | |
| # make every rank repeatedly evict four tasks' resident shards. Exact | |
| # integer ratios keep each rank on one task. For fractional ratios we | |
| # keep floor(weight * world_size) core ranks fixed and use only the | |
| # remaining flex ranks to realize the residual weights. This preserves | |
| # the requested epoch mixture while keeping most OSS caches resident. | |
| self.balanced_rank_mode = os.environ.get( | |
| "PSI_TASK_MIXTURE_BALANCED_RANKS", "0" | |
| ).strip().lower() in {"1", "true", "yes", "on"} | |
| self.rank = int(os.environ.get("RANK", 0)) | |
| self.world_size = int(os.environ.get("WORLD_SIZE", 1)) | |
| self.num_samples_rank = math.ceil(self.num_samples / self.world_size) | |
| self.epoch = 0 | |
| # A resumed run can advance this sampler without materializing the | |
| # already-trained dataset batches. Keep the live iterator after the | |
| # seek so stateful OSS shard lanes continue from the exact same point. | |
| self._resume_iterator = None | |
| def _balanced_rank_plan(self): | |
| """Return this rank's fixed task or residual flex-task weights. | |
| The returned tuple is ``(fixed_dataset_id, flex_weights, | |
| flex_position, roles)``. Exactly one of ``fixed_dataset_id`` and | |
| ``flex_weights`` is non-None. ``roles`` contains task IDs for core | |
| ranks and ``-1`` for flex ranks, after deterministic epoch shuffling. | |
| """ | |
| if not self.balanced_rank_mode: | |
| return None | |
| positive_ids = [ | |
| i for i, weight in enumerate(self.weights.tolist()) if weight > 0 | |
| ] | |
| if not positive_ids or self.world_size <= 0: | |
| return None | |
| raw_counts = self.weights[positive_ids] * self.world_size | |
| core_counts = torch.floor(raw_counts + 1e-12).long() | |
| residual = raw_counts - core_counts.to(dtype=raw_counts.dtype) | |
| residual[torch.abs(residual) < 1e-12] = 0.0 | |
| core_roles = [ | |
| dataset_id | |
| for dataset_id, rank_count in zip(positive_ids, core_counts.tolist()) | |
| for _ in range(int(rank_count)) | |
| ] | |
| flex_count = self.world_size - len(core_roles) | |
| if flex_count < 0: | |
| raise RuntimeError( | |
| f"Invalid balanced rank plan: world={self.world_size} core={len(core_roles)}" | |
| ) | |
| roles = core_roles + [-1] * flex_count | |
| if len(roles) != self.world_size: | |
| raise RuntimeError( | |
| f"Balanced rank plan size mismatch: world={self.world_size} roles={len(roles)}" | |
| ) | |
| generator = torch.Generator() | |
| generator.manual_seed(self.seed + self.epoch * 1_000_003 + 17) | |
| permutation = torch.randperm(self.world_size, generator=generator).tolist() | |
| shuffled = [0] * self.world_size | |
| for position, rank in enumerate(permutation): | |
| shuffled[rank] = roles[position] | |
| role = int(shuffled[self.rank]) | |
| flex_weights = None | |
| flex_position = -1 | |
| if role < 0: | |
| if flex_count <= 0 or float(residual.sum().item()) <= 0.0: | |
| return None | |
| flex_weights = torch.zeros_like(self.weights) | |
| flex_weights[positive_ids] = residual / residual.sum() | |
| flex_ranks = [rank for rank, assigned in enumerate(shuffled) if assigned < 0] | |
| flex_position = flex_ranks.index(self.rank) | |
| if self.rank == 0 and self.epoch == 0: | |
| counts = { | |
| task_id: shuffled.count(task_id) | |
| for task_id in sorted(set(shuffled)) | |
| if task_id >= 0 | |
| } | |
| residual_text = ( | |
| "none" | |
| if flex_count == 0 | |
| else ",".join( | |
| f"{dataset_id}:{float(weight):.6f}" | |
| for dataset_id, weight in enumerate( | |
| (residual / residual.sum()).tolist() | |
| ) | |
| if weight > 0 | |
| ) | |
| ) | |
| print( | |
| f"[TASK MIXTURE] balanced rank plan: roles={shuffled} " | |
| f"core_counts={counts} flex_ranks={flex_count} " | |
| f"flex_weights={residual_text}", | |
| flush=True, | |
| ) | |
| return (role if role >= 0 else None), flex_weights, flex_position, shuffled | |
| def set_epoch(self, epoch): | |
| self.epoch = epoch | |
| self._resume_iterator = None | |
| def _sample_dataset_indices( | |
| self, | |
| dataset_id: int, | |
| generator: torch.Generator, | |
| lane_id: int = 0, | |
| ): | |
| dataset_len = int(self.dataset_lens[dataset_id]) | |
| if self.shard_aware and self.datasets is not None: | |
| dataset = self.datasets[dataset_id] | |
| sampler = getattr(dataset, "sample_shard_aware_indices", None) | |
| if callable(sampler): | |
| try: | |
| sampled = sampler(self.batch_size, generator, lane_id=lane_id) | |
| except TypeError as exc: | |
| if "lane_id" not in str(exc): | |
| raise | |
| sampled = sampler(self.batch_size, generator) | |
| if sampled is not None: | |
| indices = getattr(sampled, "indices", sampled) | |
| if len(indices) != self.batch_size: | |
| raise RuntimeError( | |
| f"Shard-aware sampler for dataset {dataset_id} returned {len(indices)} " | |
| f"indices, expected {self.batch_size}" | |
| ) | |
| if hasattr(sampled, "indices"): | |
| sampled.indices[:] = [int(i) for i in indices] | |
| return sampled | |
| return [int(i) for i in indices] | |
| return torch.randint( | |
| low=0, | |
| high=dataset_len, | |
| size=(self.batch_size,), | |
| generator=generator, | |
| ).tolist() | |
| def _initialize_shard_aware_datasets(self, dataset_ids) -> None: | |
| """Open flex-rank task indexes before the first yielded batch. | |
| ``OssTaskShardSampler.sample_batch(0)`` initializes only metadata and | |
| SQLite state; it does not choose a shard or consume a sample. Doing | |
| this while every rank is still inside the epoch first-batch barrier | |
| prevents a flex rank from lazily opening a cold modality halfway | |
| through training while the other ranks have already entered NCCL. | |
| """ | |
| if not self.shard_aware or self.datasets is None: | |
| return | |
| generator = torch.Generator() | |
| generator.manual_seed(self.seed + self.epoch * 1_000_003 + self.rank) | |
| for dataset_id in sorted({int(value) for value in dataset_ids}): | |
| dataset = self.datasets[dataset_id] | |
| sampler = getattr(dataset, "sample_shard_aware_indices", None) | |
| if not callable(sampler): | |
| continue | |
| try: | |
| sampler(0, generator, lane_id=0) | |
| except TypeError as exc: | |
| if "lane_id" not in str(exc): | |
| raise | |
| sampler(0, generator) | |
| def _build_batch_schedule(self, total_batches, *, seed_offset: int = 0, weights=None): | |
| schedule_weights = self.weights if weights is None else weights | |
| positive_ids = [i for i, weight in enumerate(schedule_weights.tolist()) if weight > 0] | |
| if not positive_ids: | |
| raise ValueError("TaskMixtureSampler received no positive mixture ratios") | |
| positive_weights = schedule_weights[positive_ids] | |
| positive_weights = positive_weights / positive_weights.sum() | |
| raw_counts = positive_weights * total_batches | |
| batch_counts = torch.floor(raw_counts).long() | |
| batches_left = total_batches - int(batch_counts.sum().item()) | |
| if batches_left > 0: | |
| frac = raw_counts - batch_counts.double() | |
| for frac_index in torch.argsort(frac, descending=True)[:batches_left].tolist(): | |
| batch_counts[frac_index] += 1 | |
| if self.task_block_batches <= 1: | |
| schedule = [] | |
| for local_idx, dataset_id in enumerate(positive_ids): | |
| schedule.extend([dataset_id] * int(batch_counts[local_idx].item())) | |
| g = torch.Generator() | |
| g.manual_seed(self.seed + self.epoch + int(seed_offset)) | |
| shuffle_order = torch.randperm(len(schedule), generator=g).tolist() | |
| schedule = [schedule[i] for i in shuffle_order] | |
| else: | |
| blocks = [] | |
| for local_idx, dataset_id in enumerate(positive_ids): | |
| remaining = int(batch_counts[local_idx].item()) | |
| while remaining > 0: | |
| block_size = min(self.task_block_batches, remaining) | |
| blocks.append([dataset_id] * block_size) | |
| remaining -= block_size | |
| g = torch.Generator() | |
| g.manual_seed(self.seed + self.epoch + int(seed_offset)) | |
| block_order = torch.randperm(len(blocks), generator=g).tolist() | |
| schedule = [dataset_id for block_idx in block_order for dataset_id in blocks[block_idx]] | |
| if len(schedule) != total_batches: | |
| raise RuntimeError( | |
| f"TaskMixtureSampler built invalid schedule: expected {total_batches} batches, got {len(schedule)}" | |
| ) | |
| return schedule | |
| def _format_batch(dataset_id: int, sampled): | |
| indices = getattr(sampled, "indices", sampled) | |
| current_shard = getattr(sampled, "current_shard", None) | |
| if current_shard: | |
| next_shard = getattr(sampled, "next_shard", None) | |
| sample_hints = getattr(sampled, "sample_hints", None) | |
| if sample_hints is not None: | |
| if len(sample_hints) != len(indices): | |
| raise RuntimeError( | |
| f"OSS sample hint count mismatch: hints={len(sample_hints)} indices={len(indices)}" | |
| ) | |
| return [ | |
| ( | |
| dataset_id, | |
| int(sample_id), | |
| str(current_shard), | |
| str(next_shard) if next_shard else "", | |
| str(hint[0]), | |
| str(hint[1]), | |
| int(hint[2]), | |
| int(hint[3]), | |
| int(hint[4]), | |
| ) | |
| for sample_id, hint in zip(indices, sample_hints) | |
| ] | |
| return [ | |
| (dataset_id, int(sample_id), str(current_shard), str(next_shard) if next_shard else "") | |
| for sample_id in indices | |
| ] | |
| return [(dataset_id, int(sample_id)) for sample_id in indices] | |
| def _iter_batches(self): | |
| num_batches_rank = len(self) | |
| balanced_plan = self._balanced_rank_plan() | |
| if balanced_plan is not None and balanced_plan[0] is not None: | |
| balanced_dataset_id = int(balanced_plan[0]) | |
| # Keep one task on each rank for the epoch. This gives an exact | |
| # core task allocation on every global optimizer step and preserves | |
| # OSS shard locality/prefetching instead of switching tasks every | |
| # few batches. | |
| for local_batch_idx in range(num_batches_rank): | |
| g = torch.Generator() | |
| g.manual_seed( | |
| self.seed | |
| + self.epoch * num_batches_rank * self.world_size | |
| + local_batch_idx * self.world_size | |
| + self.rank | |
| ) | |
| sampled = self._sample_dataset_indices( | |
| balanced_dataset_id, | |
| g, | |
| lane_id=local_batch_idx % self.worker_lanes, | |
| ) | |
| yield self._format_batch(balanced_dataset_id, sampled) | |
| return | |
| if balanced_plan is not None and balanced_plan[1] is not None: | |
| flex_weights = balanced_plan[1] | |
| flex_position = int(balanced_plan[2]) | |
| self._initialize_shard_aware_datasets( | |
| dataset_id | |
| for dataset_id, weight in enumerate(flex_weights.tolist()) | |
| if weight > 0 | |
| ) | |
| batch_schedule = self._build_batch_schedule( | |
| num_batches_rank, | |
| seed_offset=(flex_position + 1) * 1_000_003, | |
| weights=flex_weights, | |
| ) | |
| for local_batch_idx, dataset_id in enumerate(batch_schedule): | |
| g = torch.Generator() | |
| g.manual_seed( | |
| self.seed | |
| + self.epoch * num_batches_rank * self.world_size | |
| + local_batch_idx * self.world_size | |
| + self.rank | |
| ) | |
| sampled = self._sample_dataset_indices( | |
| dataset_id, | |
| g, | |
| lane_id=local_batch_idx % self.worker_lanes, | |
| ) | |
| yield self._format_batch(dataset_id, sampled) | |
| return | |
| if self.sync_across_ranks: | |
| batch_schedule = self._build_batch_schedule(num_batches_rank) | |
| for local_batch_idx, dataset_id in enumerate(batch_schedule): | |
| g = torch.Generator() | |
| g.manual_seed( | |
| self.seed | |
| + self.epoch * num_batches_rank * self.world_size | |
| + local_batch_idx * self.world_size | |
| + self.rank | |
| ) | |
| sampled = self._sample_dataset_indices( | |
| dataset_id, | |
| g, | |
| lane_id=local_batch_idx % self.worker_lanes, | |
| ) | |
| yield self._format_batch(dataset_id, sampled) | |
| return | |
| if self.rank_local_blocks and self.task_block_batches > 1: | |
| batch_schedule = self._build_batch_schedule( | |
| num_batches_rank, | |
| seed_offset=(self.rank + 1) * 1_000_003, | |
| ) | |
| for local_batch_idx, dataset_id in enumerate(batch_schedule): | |
| g = torch.Generator() | |
| g.manual_seed( | |
| self.seed | |
| + self.epoch * num_batches_rank * self.world_size | |
| + local_batch_idx * self.world_size | |
| + self.rank | |
| ) | |
| sampled = self._sample_dataset_indices( | |
| dataset_id, | |
| g, | |
| lane_id=local_batch_idx % self.worker_lanes, | |
| ) | |
| yield self._format_batch(dataset_id, sampled) | |
| return | |
| total_batches = num_batches_rank * self.world_size | |
| batch_schedule = self._build_batch_schedule(total_batches) | |
| local_batch_idx = 0 | |
| for batch_idx, dataset_id in enumerate(batch_schedule): | |
| if batch_idx % self.world_size != self.rank: | |
| continue | |
| g = torch.Generator() | |
| g.manual_seed(self.seed + self.epoch * total_batches + batch_idx) | |
| sampled = self._sample_dataset_indices( | |
| dataset_id, | |
| g, | |
| lane_id=local_batch_idx % self.worker_lanes, | |
| ) | |
| yield self._format_batch(dataset_id, sampled) | |
| local_batch_idx += 1 | |
| def fast_forward(self, num_batches: int, progress_callback=None) -> int: | |
| """Advance deterministic sampler state without loading batch tensors. | |
| DataLoader replay is unnecessarily expensive for a mid-epoch resume: | |
| it re-reads hidden states and motion targets that have already trained. | |
| Sampling the batch indices alone is sufficient to reconstruct the OSS | |
| lane cursors and the exact next batch. The partially consumed iterator | |
| is retained and handed to the next DataLoader iteration. | |
| """ | |
| num_batches = int(num_batches) | |
| if num_batches < 0 or num_batches > len(self): | |
| raise ValueError( | |
| f"Invalid sampler fast-forward: {num_batches} batches for epoch length {len(self)}" | |
| ) | |
| if self._resume_iterator is not None: | |
| raise RuntimeError("TaskMixtureSampler already has a pending resumed iterator") | |
| iterator = self._iter_batches() | |
| report_every = 256 | |
| for batch_idx in range(num_batches): | |
| try: | |
| next(iterator) | |
| except StopIteration as exc: | |
| raise RuntimeError( | |
| f"TaskMixtureSampler ended while fast-forwarding at " | |
| f"{batch_idx:,}/{num_batches:,} batches" | |
| ) from exc | |
| completed = batch_idx + 1 | |
| if progress_callback is not None and ( | |
| completed == num_batches or completed % report_every == 0 | |
| ): | |
| progress_callback(completed) | |
| self._resume_iterator = iterator | |
| return num_batches | |
| def __iter__(self): | |
| if self._resume_iterator is not None: | |
| iterator = self._resume_iterator | |
| self._resume_iterator = None | |
| yield from iterator | |
| return | |
| yield from self._iter_batches() | |
| def __len__(self): | |
| return math.ceil(self.num_samples_rank / self.batch_size) | |
| class OssTaskBatchSampler(Sampler): | |
| """Batch sampler that keeps evaluation batches inside one OSS shard. | |
| The normal validation ``RandomSampler`` scatters 64 indices over many | |
| large OSS files, causing unnecessary random slice reads. This sampler | |
| asks the dataset's existing ``sample_shard_aware_indices`` implementation | |
| for a batch and forwards the shard offsets as index metadata. ``Dataset`` | |
| consumes that metadata in ``__getitems__`` to avoid repeated manifest or | |
| SQLite lookups while loading the batch. | |
| ``num_batches`` is the number of batches before Accelerate shards them | |
| across ranks. Callers should round it up to a multiple of world size when | |
| they want a fixed global validation budget. | |
| """ | |
| def __init__( | |
| self, | |
| dataset, | |
| batch_size: int, | |
| num_batches: int, | |
| *, | |
| seed: int = 42, | |
| worker_lanes: int = 1, | |
| ): | |
| self.dataset = dataset | |
| self.batch_size = max(int(batch_size), 1) | |
| self.num_batches = max(int(num_batches), 0) | |
| self.seed = int(seed) | |
| self.worker_lanes = max(int(worker_lanes), 1) | |
| self._prepared_batches = None | |
| self._fixed_batches = None | |
| self.fixed_batches = os.environ.get( | |
| "PSI_OMNI_OSS_EVAL_FIXED_BATCHES", "0" | |
| ).strip().lower() in {"1", "true", "yes", "on"} | |
| def _copy_batches(batches): | |
| # Sample entries are immutable tuples. Copy the two list levels so | |
| # local-path remapping never mutates the canonical evaluation set. | |
| return [list(batch) for batch in batches] | |
| def _format_sampled(sampled): | |
| indices = getattr(sampled, "indices", sampled) | |
| current_shard = getattr(sampled, "current_shard", None) | |
| if not current_shard: | |
| return [int(index) for index in indices] | |
| next_shard = getattr(sampled, "next_shard", None) | |
| sample_hints = getattr(sampled, "sample_hints", None) | |
| if sample_hints is None: | |
| return [ | |
| ( | |
| int(index), | |
| str(current_shard), | |
| str(next_shard) if next_shard else "", | |
| ) | |
| for index in indices | |
| ] | |
| if len(sample_hints) != len(indices): | |
| raise RuntimeError( | |
| f"OSS eval sample hint count mismatch: hints={len(sample_hints)} " | |
| f"indices={len(indices)}" | |
| ) | |
| return [ | |
| ( | |
| int(index), | |
| str(current_shard), | |
| str(next_shard) if next_shard else "", | |
| str(hint[0]), | |
| str(hint[1]), | |
| int(hint[2]), | |
| int(hint[3]), | |
| int(hint[4]), | |
| ) | |
| for index, hint in zip(indices, sample_hints) | |
| ] | |
| def _iter_batches(self): | |
| sample_fn = getattr(self.dataset, "sample_shard_aware_indices", None) | |
| for batch_idx in range(self.num_batches): | |
| generator = torch.Generator() | |
| generator.manual_seed(self.seed + batch_idx) | |
| if callable(sample_fn): | |
| try: | |
| sampled = sample_fn( | |
| self.batch_size, | |
| generator, | |
| lane_id=batch_idx % self.worker_lanes, | |
| ) | |
| except TypeError as exc: | |
| if "lane_id" not in str(exc): | |
| raise | |
| sampled = sample_fn(self.batch_size, generator) | |
| else: | |
| sampled = torch.randint( | |
| low=0, | |
| high=max(len(self.dataset), 1), | |
| size=(self.batch_size,), | |
| generator=generator, | |
| ).tolist() | |
| if sampled is None: | |
| sampled = torch.randint( | |
| low=0, | |
| high=max(len(self.dataset), 1), | |
| size=(self.batch_size,), | |
| generator=generator, | |
| ).tolist() | |
| yield self._format_sampled(sampled) | |
| def prepare_batches(self): | |
| """Materialize the deterministic eval schedule before data loading. | |
| The schedule is consumed exactly once by ``__iter__``. This permits | |
| the trainer to stage its shard files locally while the last training | |
| steps continue, without sampling a second or different validation | |
| sequence. | |
| """ | |
| if self._prepared_batches is None: | |
| if self.fixed_batches: | |
| if self._fixed_batches is None: | |
| self._fixed_batches = list(self._iter_batches()) | |
| self._prepared_batches = self._copy_batches(self._fixed_batches) | |
| else: | |
| self._prepared_batches = list(self._iter_batches()) | |
| return self._prepared_batches | |
| def prepared_shard_paths(self) -> set[str]: | |
| batches = self.prepare_batches() | |
| return { | |
| str(batch[0][1]) | |
| for batch in batches | |
| if batch and isinstance(batch[0], (tuple, list)) and len(batch[0]) >= 3 | |
| } | |
| def remap_prepared_shards(self, path_mapping: dict[str, str]) -> None: | |
| batches = self.prepare_batches() | |
| remapped_batches = [] | |
| for batch in batches: | |
| remapped_batch = [] | |
| for sample in batch: | |
| if not isinstance(sample, (tuple, list)) or len(sample) < 3: | |
| remapped_batch.append(sample) | |
| continue | |
| values = list(sample) | |
| values[1] = path_mapping.get(str(values[1]), str(values[1])) | |
| if values[2]: | |
| values[2] = path_mapping.get(str(values[2]), str(values[2])) | |
| remapped_batch.append(tuple(values)) | |
| remapped_batches.append(remapped_batch) | |
| self._prepared_batches = remapped_batches | |
| def __iter__(self): | |
| if self._prepared_batches is not None: | |
| batches = self._prepared_batches | |
| self._prepared_batches = None | |
| yield from batches | |
| return | |
| if self.fixed_batches: | |
| if self._fixed_batches is None: | |
| self._fixed_batches = list(self._iter_batches()) | |
| yield from self._copy_batches(self._fixed_batches) | |
| return | |
| yield from self._iter_batches() | |
| def __len__(self): | |
| return self.num_batches | |
| class DatasetSpec: | |
| # dataset: torch.utils.data.Dataset | |
| dataset_length: int | |
| prob: float # mixture ratio | |
| image_size: tuple[int, int] # e.g. 224, 448 | |
| tokens_per_image: int # (image_size / patch)^2 | |
| class TokenMixtureSampler(Sampler): | |
| """ | |
| Multi-node safe dataset mixture sampler that samples from datasets with different image resolutoion while | |
| keeping the total number of tokens per batch (approximately) fixed. | |
| """ | |
| def __init__( | |
| self, | |
| specs: list[DatasetSpec], | |
| tokens_per_batch: int, # per rank | |
| num_batches_per_rank: int, | |
| seed: int = 0, | |
| ): | |
| self.specs = specs | |
| self.tokens_per_batch = tokens_per_batch | |
| self.num_batches_per_rank = num_batches_per_rank | |
| self.seed = seed | |
| self.probs = [s.prob for s in specs] | |
| self.epoch = 0 | |
| # Distributed settings | |
| self.rank = int(os.environ.get("RANK", 0)) | |
| self.world_size = int(os.environ.get("WORLD_SIZE", 1)) | |
| def set_epoch(self, epoch): | |
| self.epoch = epoch | |
| # overwatch.info(f"TokenMixtureSampler set to epoch {epoch}") | |
| def __iter__(self): | |
| # assert self.epoch is not None, "TokenMixtureSampler epoch not set. Please call set_epoch(epoch) before iterating." | |
| # For each global batch index, synchronize dataset_id selection across all ranks | |
| for batch_idx in range(self.num_batches_per_rank * self.world_size): | |
| if batch_idx % self.world_size != self.rank: | |
| continue | |
| # Use a deterministic RNG for this global! batch to pick dataset_id and indices | |
| batch_seed = self.seed + self.epoch + batch_idx # // self.world_size | |
| local_rng = random.Random(batch_seed) | |
| dataset_id = local_rng.choices(range(len(self.specs)), weights=self.probs, k=1)[0] | |
| # print(f"[TokenMixtureSampler] rank={self.rank} seed={batch_seed} dataset_id={dataset_id}") | |
| spec = self.specs[dataset_id] | |
| batch_size = max(1, self.tokens_per_batch // spec.tokens_per_image) | |
| local_batch_rng = random.Random(self.seed + self.epoch + batch_idx) | |
| indices = [ | |
| (dataset_id, local_batch_rng.randrange(spec.dataset_length)) | |
| for _ in range(batch_size) | |
| ] | |
| # overwatch.info(f"rank {overwatch.rank()}: {self.epoch} batch {batch_idx} dataset_id {dataset_id} batch_size {batch_size}") | |
| yield indices | |
| def __len__(self): | |
| return self.num_batches_per_rank | |