import torch from torch.utils.data import Sampler from accelerate import Accelerator import math import os import random from dataclasses import dataclass from psi.utils import initialize_overwatch overwatch = initialize_overwatch(__name__) class BatchMixtureSampler(Sampler): """ Multi-node safe dataset mixture sampler. Example: datasets = [ds1, ds2], ratio = [4, 1] """ def __init__(self, dataset_lens, mixture_ratios, num_samples_per_epoch, batch_size, seed=42): self.dataset_lens = dataset_lens self.weights = torch.tensor(mixture_ratios, dtype=torch.double) self.weights /= self.weights.sum() self.num_samples = num_samples_per_epoch self.batch_size = batch_size self.seed = seed # Distributed settings self.rank = int(os.environ.get("RANK", 0)) self.world_size = int(os.environ.get("WORLD_SIZE", 1)) # each rank gets this many samples self.num_samples_rank = math.ceil(self.num_samples / self.world_size) self.epoch = 0 self.batch_counts = self._build_batch_counts() def _build_batch_counts(self): positive_ids = [i for i, weight in enumerate(self.weights.tolist()) if weight > 0] if not positive_ids: raise ValueError("BatchMixtureSampler received no positive mixture ratios") counts = torch.zeros(len(self.dataset_lens), dtype=torch.long) if self.batch_size >= len(positive_ids): counts[positive_ids] = 1 remaining = self.batch_size - len(positive_ids) if remaining > 0: positive_weights = self.weights[positive_ids] positive_weights = positive_weights / positive_weights.sum() raw_extra = positive_weights * remaining extra_counts = torch.floor(raw_extra).long() counts[positive_ids] += extra_counts extra_left = remaining - int(extra_counts.sum().item()) if extra_left > 0: frac = raw_extra - extra_counts.double() for frac_index in torch.argsort(frac, descending=True)[:extra_left].tolist(): counts[positive_ids[frac_index]] += 1 else: for dataset_id in torch.argsort(self.weights, descending=True)[:self.batch_size].tolist(): counts[dataset_id] += 1 return counts.tolist() def set_epoch(self, epoch): self.epoch = epoch def __iter__(self): total_batches = len(self) * self.world_size for batch_idx in range(total_batches): if batch_idx % self.world_size != self.rank: continue g = torch.Generator() g.manual_seed(self.seed + self.epoch * total_batches + batch_idx) batch = [] for dataset_id, count in enumerate(self.batch_counts): if count <= 0: continue dataset_len = int(self.dataset_lens[dataset_id]) rand_indices = torch.randint( low=0, high=dataset_len, size=(count,), generator=g, ).tolist() batch.extend((dataset_id, sample_id) for sample_id in rand_indices) shuffle_order = torch.randperm(len(batch), generator=g).tolist() yield [batch[i] for i in shuffle_order] def __len__(self): return math.ceil(self.num_samples_rank / self.batch_size) class NoReplacementBatchMixtureSampler(Sampler): """ Batch mixture sampler that walks each dataset through a shuffled queue. Unlike BatchMixtureSampler, sample ids are not drawn with replacement inside an epoch. When one dataset queue is exhausted, a new shuffled pass for that dataset starts. This keeps task ratios fixed at the batch level while making every active task traverse its own samples before repeating them. """ def __init__(self, dataset_lens, mixture_ratios, num_samples_per_epoch, batch_size, seed=42): self.dataset_lens = [int(length) for length in dataset_lens] self.weights = torch.tensor(mixture_ratios, dtype=torch.double) self.weights /= self.weights.sum() self.num_samples = int(num_samples_per_epoch) self.batch_size = int(batch_size) self.seed = int(seed) self.rank = int(os.environ.get("RANK", 0)) self.world_size = int(os.environ.get("WORLD_SIZE", 1)) self.epoch = 0 self.batch_counts = self._build_batch_counts() self.num_batches_rank = self._build_num_batches_rank() def _build_batch_counts(self): positive_ids = [i for i, weight in enumerate(self.weights.tolist()) if weight > 0] if not positive_ids: raise ValueError("NoReplacementBatchMixtureSampler received no positive mixture ratios") counts = torch.zeros(len(self.dataset_lens), dtype=torch.long) if self.batch_size >= len(positive_ids): counts[positive_ids] = 1 remaining = self.batch_size - len(positive_ids) if remaining > 0: positive_weights = self.weights[positive_ids] positive_weights = positive_weights / positive_weights.sum() raw_extra = positive_weights * remaining extra_counts = torch.floor(raw_extra).long() counts[positive_ids] += extra_counts extra_left = remaining - int(extra_counts.sum().item()) if extra_left > 0: frac = raw_extra - extra_counts.double() for frac_index in torch.argsort(frac, descending=True)[:extra_left].tolist(): counts[positive_ids[frac_index]] += 1 else: for dataset_id in torch.argsort(self.weights, descending=True)[:self.batch_size].tolist(): counts[dataset_id] += 1 return counts.tolist() def _build_num_batches_rank(self): requested_batches = self.num_samples // (self.batch_size * self.world_size) feasible_batches = [] for dataset_id, count in enumerate(self.batch_counts): if count <= 0: continue feasible_batches.append(self.dataset_lens[dataset_id] // (int(count) * self.world_size)) if not feasible_batches: raise ValueError("NoReplacementBatchMixtureSampler has no positive batch counts") max_no_replacement_batches = min(feasible_batches) num_batches = min(requested_batches, max_no_replacement_batches) if num_batches <= 0: raise ValueError( "NoReplacementBatchMixtureSampler cannot form one full distributed batch without replacement: " f"dataset_lens={self.dataset_lens}, batch_counts={self.batch_counts}, " f"world_size={self.world_size}, batch_size={self.batch_size}" ) return num_batches def set_epoch(self, epoch): self.epoch = int(epoch) def _reshuffle_dataset(self, dataset_id, pass_id): dataset_len = self.dataset_lens[dataset_id] if dataset_len <= 0: raise ValueError(f"Dataset {dataset_id} has non-positive length: {dataset_len}") g = torch.Generator() g.manual_seed(self.seed + self.epoch * 1_000_003 + dataset_id * 10_007 + pass_id) return torch.randperm(dataset_len, generator=g).tolist() def __iter__(self): num_batches_rank = len(self) total_batches = num_batches_rank * self.world_size dataset_orders = [[] for _ in self.dataset_lens] dataset_offsets = [0 for _ in self.dataset_lens] dataset_passes = [0 for _ in self.dataset_lens] def take(dataset_id): if dataset_offsets[dataset_id] >= len(dataset_orders[dataset_id]): if dataset_passes[dataset_id] > 0: raise RuntimeError( f"Dataset {dataset_id} was exhausted inside a no-replacement epoch" ) dataset_orders[dataset_id] = self._reshuffle_dataset(dataset_id, dataset_passes[dataset_id]) dataset_offsets[dataset_id] = 0 dataset_passes[dataset_id] += 1 sample_id = dataset_orders[dataset_id][dataset_offsets[dataset_id]] dataset_offsets[dataset_id] += 1 return sample_id for batch_idx in range(total_batches): g = torch.Generator() g.manual_seed(self.seed + self.epoch * total_batches + batch_idx) batch = [] for dataset_id, count in enumerate(self.batch_counts): for _ in range(max(0, int(count))): batch.append((dataset_id, take(dataset_id))) shuffle_order = torch.randperm(len(batch), generator=g).tolist() if batch_idx % self.world_size == self.rank: yield [batch[i] for i in shuffle_order] def __len__(self): return self.num_batches_rank @dataclass class DatasetSpec: # dataset: torch.utils.data.Dataset dataset_length: int prob: float # mixture ratio image_size: tuple[int, int] # e.g. 224, 448 tokens_per_image: int # (image_size / patch)^2 class TokenMixtureSampler(Sampler): """ Multi-node safe dataset mixture sampler that samples from datasets with different image resolutoion while keeping the total number of tokens per batch (approximately) fixed. """ def __init__( self, specs: list[DatasetSpec], tokens_per_batch: int, # per rank num_batches_per_rank: int, seed: int = 0, ): self.specs = specs self.tokens_per_batch = tokens_per_batch self.num_batches_per_rank = num_batches_per_rank self.seed = seed self.probs = [s.prob for s in specs] self.epoch = 0 # Distributed settings self.rank = int(os.environ.get("RANK", 0)) self.world_size = int(os.environ.get("WORLD_SIZE", 1)) def set_epoch(self, epoch): self.epoch = epoch # overwatch.info(f"TokenMixtureSampler set to epoch {epoch}") def __iter__(self): # assert self.epoch is not None, "TokenMixtureSampler epoch not set. Please call set_epoch(epoch) before iterating." # For each global batch index, synchronize dataset_id selection across all ranks for batch_idx in range(self.num_batches_per_rank * self.world_size): if batch_idx % self.world_size != self.rank: continue # Use a deterministic RNG for this global! batch to pick dataset_id and indices batch_seed = self.seed + self.epoch + batch_idx # // self.world_size local_rng = random.Random(batch_seed) dataset_id = local_rng.choices(range(len(self.specs)), weights=self.probs, k=1)[0] # print(f"[TokenMixtureSampler] rank={self.rank} seed={batch_seed} dataset_id={dataset_id}") spec = self.specs[dataset_id] batch_size = max(1, self.tokens_per_batch // spec.tokens_per_image) local_batch_rng = random.Random(self.seed + self.epoch + batch_idx) indices = [ (dataset_id, local_batch_rng.randrange(spec.dataset_length)) for _ in range(batch_size) ] # overwatch.info(f"rank {overwatch.rank()}: {self.epoch} batch {batch_idx} dataset_id {dataset_id} batch_size {batch_size}") yield indices def __len__(self): return self.num_batches_per_rank