import torch from torch.utils.data import Sampler from accelerate import Accelerator import math import os import random from dataclasses import dataclass from psi.utils import initialize_overwatch overwatch = initialize_overwatch(__name__) def _normalize_positive_weights(mixture_ratios): weights = torch.tensor(mixture_ratios, dtype=torch.double) if weights.numel() == 0: raise ValueError("Sampler received empty mixture ratios") if torch.any(weights < 0): raise ValueError(f"Sampler received negative mixture ratios: {mixture_ratios}") weight_sum = weights.sum() if weight_sum.item() <= 0: raise ValueError("Sampler received no positive mixture ratios") weights /= weight_sum return weights class BatchMixtureSampler(Sampler): """ Multi-node safe dataset mixture sampler. Example: datasets = [ds1, ds2], ratio = [4, 1] """ def __init__(self, dataset_lens, mixture_ratios, num_samples_per_epoch, batch_size, seed=42): self.dataset_lens = dataset_lens self.weights = _normalize_positive_weights(mixture_ratios) self.num_samples = num_samples_per_epoch self.batch_size = batch_size self.seed = seed # Distributed settings self.rank = int(os.environ.get("RANK", 0)) self.world_size = int(os.environ.get("WORLD_SIZE", 1)) # each rank gets this many samples self.num_samples_rank = math.ceil(self.num_samples / self.world_size) self.epoch = 0 self.batch_counts = self._build_batch_counts() def _build_batch_counts(self): positive_ids = [i for i, weight in enumerate(self.weights.tolist()) if weight > 0] if not positive_ids: raise ValueError("BatchMixtureSampler received no positive mixture ratios") counts = torch.zeros(len(self.dataset_lens), dtype=torch.long) if self.batch_size >= len(positive_ids): counts[positive_ids] = 1 remaining = self.batch_size - len(positive_ids) if remaining > 0: positive_weights = self.weights[positive_ids] positive_weights = positive_weights / positive_weights.sum() raw_extra = positive_weights * remaining extra_counts = torch.floor(raw_extra).long() counts[positive_ids] += extra_counts extra_left = remaining - int(extra_counts.sum().item()) if extra_left > 0: frac = raw_extra - extra_counts.double() for frac_index in torch.argsort(frac, descending=True)[:extra_left].tolist(): counts[positive_ids[frac_index]] += 1 else: for dataset_id in torch.argsort(self.weights, descending=True)[:self.batch_size].tolist(): counts[dataset_id] += 1 return counts.tolist() def set_epoch(self, epoch): self.epoch = epoch def __iter__(self): num_batches_rank = len(self) total_batches = num_batches_rank * self.world_size for batch_idx in range(total_batches): if batch_idx % self.world_size != self.rank: continue g = torch.Generator() g.manual_seed(self.seed + self.epoch * total_batches + batch_idx) batch = [] for dataset_id, count in enumerate(self.batch_counts): if count <= 0: continue dataset_len = int(self.dataset_lens[dataset_id]) rand_indices = torch.randint( low=0, high=dataset_len, size=(count,), generator=g, ).tolist() batch.extend((dataset_id, sample_id) for sample_id in rand_indices) shuffle_order = torch.randperm(len(batch), generator=g).tolist() yield [batch[i] for i in shuffle_order] def __len__(self): return math.ceil(self.num_samples_rank / self.batch_size) class TaskMixtureSampler(Sampler): """ Multi-node safe dataset mixture sampler that assigns exactly one dataset to each global batch, then samples a full homogeneous batch from that dataset. """ def __init__(self, dataset_lens, mixture_ratios, num_samples_per_epoch, batch_size, seed=42, datasets=None): self.dataset_lens = dataset_lens self.datasets = datasets self.weights = _normalize_positive_weights(mixture_ratios) self.num_samples = num_samples_per_epoch self.batch_size = batch_size self.seed = seed self.sync_across_ranks = os.environ.get("PSI_TASK_MIXTURE_SYNC_RANKS", "0").strip().lower() in { "1", "true", "yes", "on", } self.shard_aware = os.environ.get("PSI_TASK_MIXTURE_SHARD_AWARE", "0").strip().lower() in { "1", "true", "yes", "on", } self.task_block_batches = max(int(os.environ.get("PSI_TASK_MIXTURE_BLOCK_BATCHES", "1")), 1) self.worker_lanes = max(int(os.environ.get("PSI_OMNI_OSS_WORKER_LANES", "1")), 1) self.rank_local_blocks = os.environ.get("PSI_TASK_MIXTURE_RANK_LOCAL_BLOCKS", "0").strip().lower() in { "1", "true", "yes", "on", } # In OSS training, long task blocks are useful for shard reuse but can # make the global loss jump when the set of tasks present on each rank # changes. When enabled, assign an equal number of ranks to every # equally weighted task for the whole epoch. Each global step then has # the same task composition while every rank keeps reusing one task's # resident OSS shard. Unsupported mixtures fall back to the legacy # block scheduler below. self.balanced_rank_mode = os.environ.get( "PSI_TASK_MIXTURE_BALANCED_RANKS", "0" ).strip().lower() in {"1", "true", "yes", "on"} self.rank = int(os.environ.get("RANK", 0)) self.world_size = int(os.environ.get("WORLD_SIZE", 1)) self.num_samples_rank = math.ceil(self.num_samples / self.world_size) self.epoch = 0 def _balanced_rank_assignment(self) -> list[int] | None: """Return a deterministic equal-task assignment for all ranks. The mode is intentionally limited to equal positive weights and a world size divisible by the number of enabled tasks. This keeps the old ratio-aware scheduler as a safe fallback for arbitrary mixtures. Every rank computes the same permutation from ``seed`` and ``epoch``; only its own entry is consumed in ``__iter__``. """ if not self.balanced_rank_mode: return None positive_ids = [ i for i, weight in enumerate(self.weights.tolist()) if weight > 0 ] task_count = len(positive_ids) if task_count <= 0 or self.world_size < task_count or self.world_size % task_count: return None expected = 1.0 / float(task_count) if not torch.allclose( self.weights[positive_ids], torch.full((task_count,), expected, dtype=self.weights.dtype), atol=1e-12, rtol=0.0, ): return None ranks_per_task = self.world_size // task_count assignment = [ dataset_id for dataset_id in positive_ids for _ in range(ranks_per_task) ] generator = torch.Generator() generator.manual_seed(self.seed + self.epoch * 1_000_003 + 17) permutation = torch.randperm(self.world_size, generator=generator).tolist() shuffled = [0] * self.world_size for position, rank in enumerate(permutation): shuffled[rank] = assignment[position] return shuffled def _balanced_dataset_for_rank(self) -> int | None: assignment = self._balanced_rank_assignment() if assignment is None: return None dataset_id = int(assignment[self.rank]) if self.rank == 0 and self.epoch == 0: counts = {task_id: assignment.count(task_id) for task_id in sorted(set(assignment))} print( f"[TASK MIXTURE] balanced rank mode enabled: assignment={assignment} " f"counts={counts}", flush=True, ) return dataset_id def set_epoch(self, epoch): self.epoch = epoch def _sample_dataset_indices( self, dataset_id: int, generator: torch.Generator, lane_id: int = 0, ): dataset_len = int(self.dataset_lens[dataset_id]) if self.shard_aware and self.datasets is not None: dataset = self.datasets[dataset_id] sampler = getattr(dataset, "sample_shard_aware_indices", None) if callable(sampler): try: sampled = sampler(self.batch_size, generator, lane_id=lane_id) except TypeError as exc: if "lane_id" not in str(exc): raise sampled = sampler(self.batch_size, generator) if sampled is not None: indices = getattr(sampled, "indices", sampled) if len(indices) != self.batch_size: raise RuntimeError( f"Shard-aware sampler for dataset {dataset_id} returned {len(indices)} " f"indices, expected {self.batch_size}" ) if hasattr(sampled, "indices"): sampled.indices[:] = [int(i) for i in indices] return sampled return [int(i) for i in indices] return torch.randint( low=0, high=dataset_len, size=(self.batch_size,), generator=generator, ).tolist() def _build_batch_schedule(self, total_batches, *, seed_offset: int = 0): positive_ids = [i for i, weight in enumerate(self.weights.tolist()) if weight > 0] if not positive_ids: raise ValueError("TaskMixtureSampler received no positive mixture ratios") positive_weights = self.weights[positive_ids] positive_weights = positive_weights / positive_weights.sum() raw_counts = positive_weights * total_batches batch_counts = torch.floor(raw_counts).long() batches_left = total_batches - int(batch_counts.sum().item()) if batches_left > 0: frac = raw_counts - batch_counts.double() for frac_index in torch.argsort(frac, descending=True)[:batches_left].tolist(): batch_counts[frac_index] += 1 if self.task_block_batches <= 1: schedule = [] for local_idx, dataset_id in enumerate(positive_ids): schedule.extend([dataset_id] * int(batch_counts[local_idx].item())) g = torch.Generator() g.manual_seed(self.seed + self.epoch + int(seed_offset)) shuffle_order = torch.randperm(len(schedule), generator=g).tolist() schedule = [schedule[i] for i in shuffle_order] else: blocks = [] for local_idx, dataset_id in enumerate(positive_ids): remaining = int(batch_counts[local_idx].item()) while remaining > 0: block_size = min(self.task_block_batches, remaining) blocks.append([dataset_id] * block_size) remaining -= block_size g = torch.Generator() g.manual_seed(self.seed + self.epoch + int(seed_offset)) block_order = torch.randperm(len(blocks), generator=g).tolist() schedule = [dataset_id for block_idx in block_order for dataset_id in blocks[block_idx]] if len(schedule) != total_batches: raise RuntimeError( f"TaskMixtureSampler built invalid schedule: expected {total_batches} batches, got {len(schedule)}" ) return schedule @staticmethod def _format_batch(dataset_id: int, sampled): indices = getattr(sampled, "indices", sampled) current_shard = getattr(sampled, "current_shard", None) if current_shard: next_shard = getattr(sampled, "next_shard", None) sample_hints = getattr(sampled, "sample_hints", None) if sample_hints is not None: if len(sample_hints) != len(indices): raise RuntimeError( f"OSS sample hint count mismatch: hints={len(sample_hints)} indices={len(indices)}" ) return [ ( dataset_id, int(sample_id), str(current_shard), str(next_shard) if next_shard else "", str(hint[0]), str(hint[1]), int(hint[2]), int(hint[3]), int(hint[4]), ) for sample_id, hint in zip(indices, sample_hints) ] return [ (dataset_id, int(sample_id), str(current_shard), str(next_shard) if next_shard else "") for sample_id in indices ] return [(dataset_id, int(sample_id)) for sample_id in indices] def __iter__(self): num_batches_rank = len(self) balanced_dataset_id = self._balanced_dataset_for_rank() if balanced_dataset_id is not None: # Keep one task on each rank for the epoch. This gives an exact # equal task mixture on every global optimizer step and preserves # OSS shard locality/prefetching instead of switching tasks every # few batches. for local_batch_idx in range(num_batches_rank): g = torch.Generator() g.manual_seed( self.seed + self.epoch * num_batches_rank * self.world_size + local_batch_idx * self.world_size + self.rank ) sampled = self._sample_dataset_indices( balanced_dataset_id, g, lane_id=local_batch_idx % self.worker_lanes, ) yield self._format_batch(balanced_dataset_id, sampled) return if self.sync_across_ranks: batch_schedule = self._build_batch_schedule(num_batches_rank) for local_batch_idx, dataset_id in enumerate(batch_schedule): g = torch.Generator() g.manual_seed( self.seed + self.epoch * num_batches_rank * self.world_size + local_batch_idx * self.world_size + self.rank ) sampled = self._sample_dataset_indices( dataset_id, g, lane_id=local_batch_idx % self.worker_lanes, ) yield self._format_batch(dataset_id, sampled) return if self.rank_local_blocks and self.task_block_batches > 1: batch_schedule = self._build_batch_schedule( num_batches_rank, seed_offset=(self.rank + 1) * 1_000_003, ) for local_batch_idx, dataset_id in enumerate(batch_schedule): g = torch.Generator() g.manual_seed( self.seed + self.epoch * num_batches_rank * self.world_size + local_batch_idx * self.world_size + self.rank ) sampled = self._sample_dataset_indices( dataset_id, g, lane_id=local_batch_idx % self.worker_lanes, ) yield self._format_batch(dataset_id, sampled) return total_batches = num_batches_rank * self.world_size batch_schedule = self._build_batch_schedule(total_batches) local_batch_idx = 0 for batch_idx, dataset_id in enumerate(batch_schedule): if batch_idx % self.world_size != self.rank: continue g = torch.Generator() g.manual_seed(self.seed + self.epoch * total_batches + batch_idx) sampled = self._sample_dataset_indices( dataset_id, g, lane_id=local_batch_idx % self.worker_lanes, ) yield self._format_batch(dataset_id, sampled) local_batch_idx += 1 def __len__(self): return math.ceil(self.num_samples_rank / self.batch_size) class OssTaskBatchSampler(Sampler): """Batch sampler that keeps evaluation batches inside one OSS shard. The normal validation ``RandomSampler`` scatters 64 indices over many large OSS files, causing unnecessary random slice reads. This sampler asks the dataset's existing ``sample_shard_aware_indices`` implementation for a batch and forwards the shard offsets as index metadata. ``Dataset`` consumes that metadata in ``__getitems__`` to avoid repeated manifest or SQLite lookups while loading the batch. ``num_batches`` is the number of batches before Accelerate shards them across ranks. Callers should round it up to a multiple of world size when they want a fixed global validation budget. """ def __init__( self, dataset, batch_size: int, num_batches: int, *, seed: int = 42, worker_lanes: int = 1, ): self.dataset = dataset self.batch_size = max(int(batch_size), 1) self.num_batches = max(int(num_batches), 0) self.seed = int(seed) self.worker_lanes = max(int(worker_lanes), 1) self._prepared_batches = None self._fixed_batches = None self.fixed_batches = os.environ.get( "PSI_OMNI_OSS_EVAL_FIXED_BATCHES", "0" ).strip().lower() in {"1", "true", "yes", "on"} @staticmethod def _copy_batches(batches): # Sample entries are immutable tuples. Copy the two list levels so # local-path remapping never mutates the canonical evaluation set. return [list(batch) for batch in batches] @staticmethod def _format_sampled(sampled): indices = getattr(sampled, "indices", sampled) current_shard = getattr(sampled, "current_shard", None) if not current_shard: return [int(index) for index in indices] next_shard = getattr(sampled, "next_shard", None) sample_hints = getattr(sampled, "sample_hints", None) if sample_hints is None: return [ ( int(index), str(current_shard), str(next_shard) if next_shard else "", ) for index in indices ] if len(sample_hints) != len(indices): raise RuntimeError( f"OSS eval sample hint count mismatch: hints={len(sample_hints)} " f"indices={len(indices)}" ) return [ ( int(index), str(current_shard), str(next_shard) if next_shard else "", str(hint[0]), str(hint[1]), int(hint[2]), int(hint[3]), int(hint[4]), ) for index, hint in zip(indices, sample_hints) ] def _iter_batches(self): sample_fn = getattr(self.dataset, "sample_shard_aware_indices", None) for batch_idx in range(self.num_batches): generator = torch.Generator() generator.manual_seed(self.seed + batch_idx) if callable(sample_fn): try: sampled = sample_fn( self.batch_size, generator, lane_id=batch_idx % self.worker_lanes, ) except TypeError as exc: if "lane_id" not in str(exc): raise sampled = sample_fn(self.batch_size, generator) else: sampled = torch.randint( low=0, high=max(len(self.dataset), 1), size=(self.batch_size,), generator=generator, ).tolist() if sampled is None: sampled = torch.randint( low=0, high=max(len(self.dataset), 1), size=(self.batch_size,), generator=generator, ).tolist() yield self._format_sampled(sampled) def prepare_batches(self): """Materialize the deterministic eval schedule before data loading. The schedule is consumed exactly once by ``__iter__``. This permits the trainer to stage its shard files locally while the last training steps continue, without sampling a second or different validation sequence. """ if self._prepared_batches is None: if self.fixed_batches: if self._fixed_batches is None: self._fixed_batches = list(self._iter_batches()) self._prepared_batches = self._copy_batches(self._fixed_batches) else: self._prepared_batches = list(self._iter_batches()) return self._prepared_batches def prepared_shard_paths(self) -> set[str]: batches = self.prepare_batches() return { str(batch[0][1]) for batch in batches if batch and isinstance(batch[0], (tuple, list)) and len(batch[0]) >= 3 } def remap_prepared_shards(self, path_mapping: dict[str, str]) -> None: batches = self.prepare_batches() remapped_batches = [] for batch in batches: remapped_batch = [] for sample in batch: if not isinstance(sample, (tuple, list)) or len(sample) < 3: remapped_batch.append(sample) continue values = list(sample) values[1] = path_mapping.get(str(values[1]), str(values[1])) if values[2]: values[2] = path_mapping.get(str(values[2]), str(values[2])) remapped_batch.append(tuple(values)) remapped_batches.append(remapped_batch) self._prepared_batches = remapped_batches def __iter__(self): if self._prepared_batches is not None: batches = self._prepared_batches self._prepared_batches = None yield from batches return if self.fixed_batches: if self._fixed_batches is None: self._fixed_batches = list(self._iter_batches()) yield from self._copy_batches(self._fixed_batches) return yield from self._iter_batches() def __len__(self): return self.num_batches @dataclass class DatasetSpec: # dataset: torch.utils.data.Dataset dataset_length: int prob: float # mixture ratio image_size: tuple[int, int] # e.g. 224, 448 tokens_per_image: int # (image_size / patch)^2 class TokenMixtureSampler(Sampler): """ Multi-node safe dataset mixture sampler that samples from datasets with different image resolutoion while keeping the total number of tokens per batch (approximately) fixed. """ def __init__( self, specs: list[DatasetSpec], tokens_per_batch: int, # per rank num_batches_per_rank: int, seed: int = 0, ): self.specs = specs self.tokens_per_batch = tokens_per_batch self.num_batches_per_rank = num_batches_per_rank self.seed = seed self.probs = [s.prob for s in specs] self.epoch = 0 # Distributed settings self.rank = int(os.environ.get("RANK", 0)) self.world_size = int(os.environ.get("WORLD_SIZE", 1)) def set_epoch(self, epoch): self.epoch = epoch # overwatch.info(f"TokenMixtureSampler set to epoch {epoch}") def __iter__(self): # assert self.epoch is not None, "TokenMixtureSampler epoch not set. Please call set_epoch(epoch) before iterating." # For each global batch index, synchronize dataset_id selection across all ranks for batch_idx in range(self.num_batches_per_rank * self.world_size): if batch_idx % self.world_size != self.rank: continue # Use a deterministic RNG for this global! batch to pick dataset_id and indices batch_seed = self.seed + self.epoch + batch_idx # // self.world_size local_rng = random.Random(batch_seed) dataset_id = local_rng.choices(range(len(self.specs)), weights=self.probs, k=1)[0] # print(f"[TokenMixtureSampler] rank={self.rank} seed={batch_seed} dataset_id={dataset_id}") spec = self.specs[dataset_id] batch_size = max(1, self.tokens_per_batch // spec.tokens_per_image) local_batch_rng = random.Random(self.seed + self.epoch + batch_idx) indices = [ (dataset_id, local_batch_rng.randrange(spec.dataset_length)) for _ in range(batch_size) ] # overwatch.info(f"rank {overwatch.rank()}: {self.epoch} batch {batch_idx} dataset_id {dataset_id} batch_size {batch_size}") yield indices def __len__(self): return self.num_batches_per_rank