Download src/psi/data/sampler.py from Tinghong-Ye22/code-Psi0-yth-v19: direct link, hf CLI and curl.
- Browser
- Download file 6.15 kB
-
https://huggingface.co/Tinghong-Ye22/code-Psi0-yth-v19/resolve/main/src/psi/data/sampler.py
- Command line
-
hf download hf://Tinghong-Ye22/code-Psi0-yth-v19/src/psi/data/sampler.py
-
curl -L -o sampler.py https://huggingface.co/Tinghong-Ye22/code-Psi0-yth-v19/resolve/main/src/psi/data/sampler.py
6.15 kB
| import torch | |
| from torch.utils.data import Sampler | |
| from accelerate import Accelerator | |
| import math | |
| import os | |
| import random | |
| from dataclasses import dataclass | |
| from psi.utils import initialize_overwatch | |
| overwatch = initialize_overwatch(__name__) | |
| class BatchMixtureSampler(Sampler): | |
| """ | |
| Multi-node safe dataset mixture sampler. | |
| Example: datasets = [ds1, ds2], ratio = [4, 1] | |
| """ | |
| def __init__(self, dataset_lens, mixture_ratios, num_samples_per_epoch, batch_size, seed=42): | |
| self.dataset_lens = dataset_lens | |
| self.weights = torch.tensor(mixture_ratios, dtype=torch.double) | |
| self.weights /= self.weights.sum() | |
| self.num_samples = num_samples_per_epoch | |
| self.batch_size = batch_size | |
| self.seed = seed | |
| # Distributed settings | |
| self.rank = int(os.environ.get("RANK", 0)) | |
| self.world_size = int(os.environ.get("WORLD_SIZE", 1)) | |
| # each rank gets this many samples | |
| self.num_samples_rank = math.ceil(self.num_samples / self.world_size) | |
| self.epoch = 0 | |
| self.batch_counts = self._build_batch_counts() | |
| def _build_batch_counts(self): | |
| positive_ids = [i for i, weight in enumerate(self.weights.tolist()) if weight > 0] | |
| if not positive_ids: | |
| raise ValueError("BatchMixtureSampler received no positive mixture ratios") | |
| counts = torch.zeros(len(self.dataset_lens), dtype=torch.long) | |
| if self.batch_size >= len(positive_ids): | |
| counts[positive_ids] = 1 | |
| remaining = self.batch_size - len(positive_ids) | |
| if remaining > 0: | |
| positive_weights = self.weights[positive_ids] | |
| positive_weights = positive_weights / positive_weights.sum() | |
| raw_extra = positive_weights * remaining | |
| extra_counts = torch.floor(raw_extra).long() | |
| counts[positive_ids] += extra_counts | |
| extra_left = remaining - int(extra_counts.sum().item()) | |
| if extra_left > 0: | |
| frac = raw_extra - extra_counts.double() | |
| for frac_index in torch.argsort(frac, descending=True)[:extra_left].tolist(): | |
| counts[positive_ids[frac_index]] += 1 | |
| else: | |
| for dataset_id in torch.argsort(self.weights, descending=True)[:self.batch_size].tolist(): | |
| counts[dataset_id] += 1 | |
| return counts.tolist() | |
| def set_epoch(self, epoch): | |
| self.epoch = epoch | |
| def __iter__(self): | |
| num_batches_rank = len(self) | |
| total_batches = num_batches_rank * self.world_size | |
| for batch_idx in range(total_batches): | |
| if batch_idx % self.world_size != self.rank: | |
| continue | |
| g = torch.Generator() | |
| g.manual_seed(self.seed + self.epoch * total_batches + batch_idx) | |
| batch = [] | |
| for dataset_id, count in enumerate(self.batch_counts): | |
| if count <= 0: | |
| continue | |
| dataset_len = int(self.dataset_lens[dataset_id]) | |
| rand_indices = torch.randint( | |
| low=0, | |
| high=dataset_len, | |
| size=(count,), | |
| generator=g, | |
| ).tolist() | |
| batch.extend((dataset_id, sample_id) for sample_id in rand_indices) | |
| shuffle_order = torch.randperm(len(batch), generator=g).tolist() | |
| yield [batch[i] for i in shuffle_order] | |
| def __len__(self): | |
| return math.ceil(self.num_samples_rank / self.batch_size) | |
| class DatasetSpec: | |
| # dataset: torch.utils.data.Dataset | |
| dataset_length: int | |
| prob: float # mixture ratio | |
| image_size: tuple[int, int] # e.g. 224, 448 | |
| tokens_per_image: int # (image_size / patch)^2 | |
| class TokenMixtureSampler(Sampler): | |
| """ | |
| Multi-node safe dataset mixture sampler that samples from datasets with different image resolutoion while | |
| keeping the total number of tokens per batch (approximately) fixed. | |
| """ | |
| def __init__( | |
| self, | |
| specs: list[DatasetSpec], | |
| tokens_per_batch: int, # per rank | |
| num_batches_per_rank: int, | |
| seed: int = 0, | |
| ): | |
| self.specs = specs | |
| self.tokens_per_batch = tokens_per_batch | |
| self.num_batches_per_rank = num_batches_per_rank | |
| self.seed = seed | |
| self.probs = [s.prob for s in specs] | |
| self.epoch = 0 | |
| # Distributed settings | |
| self.rank = int(os.environ.get("RANK", 0)) | |
| self.world_size = int(os.environ.get("WORLD_SIZE", 1)) | |
| def set_epoch(self, epoch): | |
| self.epoch = epoch | |
| # overwatch.info(f"TokenMixtureSampler set to epoch {epoch}") | |
| def __iter__(self): | |
| # assert self.epoch is not None, "TokenMixtureSampler epoch not set. Please call set_epoch(epoch) before iterating." | |
| # For each global batch index, synchronize dataset_id selection across all ranks | |
| for batch_idx in range(self.num_batches_per_rank * self.world_size): | |
| if batch_idx % self.world_size != self.rank: | |
| continue | |
| # Use a deterministic RNG for this global! batch to pick dataset_id and indices | |
| batch_seed = self.seed + self.epoch + batch_idx # // self.world_size | |
| local_rng = random.Random(batch_seed) | |
| dataset_id = local_rng.choices(range(len(self.specs)), weights=self.probs, k=1)[0] | |
| # print(f"[TokenMixtureSampler] rank={self.rank} seed={batch_seed} dataset_id={dataset_id}") | |
| spec = self.specs[dataset_id] | |
| batch_size = max(1, self.tokens_per_batch // spec.tokens_per_image) | |
| local_batch_rng = random.Random(self.seed + self.epoch + batch_idx) | |
| indices = [ | |
| (dataset_id, local_batch_rng.randrange(spec.dataset_length)) | |
| for _ in range(batch_size) | |
| ] | |
| # overwatch.info(f"rank {overwatch.rank()}: {self.epoch} batch {batch_idx} dataset_id {dataset_id} batch_size {batch_size}") | |
| yield indices | |
| def __len__(self): | |
| return self.num_batches_per_rank | |