Download dpo.py from zerdovzad/WarpStateAI: direct link, hf CLI and curl.
- Browser
- Download file 35.2 kB
-
https://huggingface.co/zerdovzad/WarpStateAI/resolve/main/dpo.py
- Command line
-
hf download hf://zerdovzad/WarpStateAI/dpo.py
-
curl -L -o dpo.py https://huggingface.co/zerdovzad/WarpStateAI/resolve/main/dpo.py
35.2 kB
| #!/usr/bin/env python3 | |
| """BF16 DPO with an immutable SFT reference, exact resume and response-only loss. | |
| The reference is scored once, before allocating the policy optimizer. Its cache | |
| is bound to weights, data, masking, execution and source hashes. Policy training | |
| uses fresh FP32 Adam state and saves only into its own output directory. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import contextlib | |
| import gc | |
| import hashlib | |
| import json | |
| import math | |
| import os | |
| from pathlib import Path | |
| import random | |
| import signal | |
| import time | |
| import types | |
| import numpy as np | |
| import torch | |
| import torch.distributed as dist | |
| import torch.nn.functional as F | |
| from torch.utils.checkpoint import checkpoint as activation_checkpoint | |
| from posttrain import (WORKSPACE, atomic_json, base_pipeline_lock, digest, | |
| restore_rng, rng_state, update_indices, validate_paths) | |
| from ops.dpo_data import MASK_FORMAT, collate_pairs, load_data | |
| from ops.posttrain_data import RENDER_FORMAT, renderer_sha256 | |
| FORMAT = 'warpstate-dpo-v1' | |
| CACHE_FORMAT = 'warpstate-dpo-reference-v1' | |
| STOP = False | |
| DEFAULTS = dict(sequence_length=2048, micro_batch=2, grad_accum=8, lr=5e-7, | |
| min_lr=0., warmup_ratio=.1, epochs=1, max_steps=None, | |
| weight_decay=0., seed=42, eval_every=100, save_every=100, | |
| eval_batches=32, compile_core=True, bf16_residuals=True, | |
| gradient_checkpointing=False, workers=4, deterministic=True, | |
| clip_grad_norm=1., beta=5., objective='length-normalized-dpo', | |
| head_chunk_size=256) | |
| def load_plan(path): | |
| plan = json.loads(Path(path).read_text()) | |
| if plan.get('format') != 'warpstate-dpo-plan-v1' or plan.get('stage') != 'dpo': | |
| raise ValueError('Expected warpstate-dpo-plan-v1 stage dpo') | |
| supplied = plan.get('training', {}) | |
| unknown = set(supplied) - set(DEFAULTS) | |
| if unknown: | |
| raise ValueError(f'Unknown DPO training settings: {sorted(unknown)}') | |
| settings = DEFAULTS | supplied | |
| for key in ('sequence_length', 'micro_batch', 'grad_accum', 'epochs', 'eval_every', | |
| 'save_every', 'eval_batches', 'workers', 'head_chunk_size'): | |
| if type(settings[key]) is not int or settings[key] < 1: | |
| raise ValueError(f'{key} must be a positive integer') | |
| if settings['max_steps'] is not None and (type(settings['max_steps']) is not int or settings['max_steps'] < 1): | |
| raise ValueError('max_steps must be null or a positive integer') | |
| for key in ('lr', 'min_lr', 'warmup_ratio', 'weight_decay', 'clip_grad_norm', 'beta'): | |
| value = settings[key] | |
| if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) or value < 0: | |
| raise ValueError(f'{key} must be finite and nonnegative') | |
| if not 0 <= settings['warmup_ratio'] < 1 or not 0 <= settings['min_lr'] <= settings['lr']: | |
| raise ValueError('Invalid LR schedule') | |
| if not settings['lr'] or not settings['clip_grad_norm'] or not settings['beta']: | |
| raise ValueError('lr, clip_grad_norm and beta must be positive') | |
| for key in ('compile_core', 'bf16_residuals', 'gradient_checkpointing', 'deterministic'): | |
| if type(settings[key]) is not bool: | |
| raise ValueError(f'{key} must be a boolean') | |
| if type(settings['seed']) is not int or settings['seed'] < 0: | |
| raise ValueError('seed must be a nonnegative integer') | |
| if settings['objective'] not in ('length-normalized-dpo', 'summed-dpo'): | |
| raise ValueError('Unsupported DPO objective') | |
| return plan, settings | |
| def source_identity(): | |
| from posttrain import source_identity as shared_sources | |
| root = Path(__file__).parent | |
| return shared_sources() | {str(path.relative_to(root)): digest(path) | |
| for path in (Path(__file__), root / 'ops/dpo_data.py')} | |
| def identity_digest(identity): | |
| return hashlib.sha256(json.dumps(identity, sort_keys=True, separators=(',', ':'), | |
| allow_nan=False).encode()).hexdigest() | |
| def learning_rate(step, total_steps, settings): | |
| """Fixed linear warmup/decay schedule, independent of a pause boundary.""" | |
| warmup = min(math.ceil(settings['warmup_ratio'] * total_steps), max(0, total_steps - 1)) | |
| if step < warmup: | |
| return settings['lr'] * (step + 1) / warmup | |
| progress = min(1., (step - warmup) / max(1, total_steps - warmup - 1)) | |
| return settings['min_lr'] + (settings['lr'] - settings['min_lr']) * (1. - progress) | |
| def reference_identity(weights_hash, manifest_hash, tokenizer_hash, settings, device, world): | |
| return dict(format=CACHE_FORMAT, weights_sha256=weights_hash, data_sha256=manifest_hash, | |
| tokenizer_sha256=tokenizer_hash, renderer_sha256=renderer_sha256(), | |
| mask_format=MASK_FORMAT, sources=source_identity(), settings={key: settings[key] | |
| for key in ('sequence_length', 'micro_batch', 'head_chunk_size', 'compile_core', | |
| 'bf16_residuals', 'deterministic', 'workers')}, | |
| dropout=0., logps='response-token-sum-float32', dtype='bfloat16', | |
| device_type=device.type, world_size=world, torch_version=torch.__version__) | |
| def validate_cache(directory, identity, sizes): | |
| directory = Path(directory) | |
| manifest = json.loads((directory / 'manifest.json').read_text()) | |
| if (manifest.get('format') != CACHE_FORMAT or manifest.get('complete') is not True or | |
| manifest.get('identity') != identity or manifest.get('identity_sha256') != identity_digest(identity)): | |
| raise ValueError('Reference cache identity/completion differs') | |
| result = [] | |
| for name, size in zip(('train.npy', 'val.npy'), sizes): | |
| path = directory / name | |
| info = manifest.get('files', {}).get(name, {}) | |
| if not path.is_file() or path.is_symlink() or digest(path) != info.get('sha256') or info.get('rows') != size: | |
| raise ValueError(f'Reference cache file changed: {name}') | |
| values = np.load(path, allow_pickle=False, mmap_mode='r') | |
| if values.shape != (size, 2) or values.dtype != np.float64 or not np.isfinite(values).all(): | |
| raise ValueError(f'Invalid reference log probabilities: {name}') | |
| result.append(values) | |
| return *result, digest(directory / 'manifest.json') | |
| def apply_sequence_logps(model, chunk_size): | |
| """Return per-sequence log-probability sums without retaining vocabulary logits. | |
| Vocabulary matmuls use BF16 autocast; cross-entropy reductions use FP32. Each | |
| head chunk is recomputed in backward, so DPO keeps a vector of branch scores. | |
| The backbone modules and canonical parameter/state names are unchanged. | |
| """ | |
| def forward(self, input_ids, targets=None, *, return_logits=False): | |
| if targets is None or return_logits: | |
| raise ValueError('DPO forward requires response targets and no returned logits') | |
| hidden = self.embed(input_ids) | |
| for depth in range(self.cfg.logical_depth): | |
| hidden = hidden * (1. + self.depth_scale[depth]) + self.depth_bias[depth] | |
| core = self.cores[depth % self.cfg.n_cores] | |
| if self.training and self.cfg.gradient_checkpointing: | |
| hidden = activation_checkpoint(core, hidden, use_reentrant=False) | |
| else: | |
| hidden = core(hidden) | |
| hidden = self.final_norm(hidden) | |
| def score(h, weight, labels): | |
| logits = F.linear(h, weight).float() | |
| losses = F.cross_entropy(logits.reshape(-1, logits.size(-1)), labels.reshape(-1), | |
| reduction='none', ignore_index=-100) | |
| return -losses.reshape(labels.shape).sum(dim=1) | |
| logps = hidden.new_zeros((hidden.shape[0],), dtype=torch.float32) | |
| for start in range(0, hidden.shape[1], chunk_size): | |
| args = (hidden[:, start:start + chunk_size], self.embed.weight, | |
| targets[:, start:start + chunk_size]) | |
| value = activation_checkpoint(score, *args, use_reentrant=False) if torch.is_grad_enabled() else score(*args) | |
| logps = logps + value | |
| return None, logps | |
| model.forward = types.MethodType(forward, model) | |
| def preference_metrics(policy, reference, counts, beta, objective): | |
| """Pair vectors, retaining gradients only through policy log probabilities.""" | |
| reference = reference.detach().to(policy.dtype) | |
| if objective == 'length-normalized-dpo': | |
| policy = policy / counts.clamp_min(1) | |
| reference = reference / counts.clamp_min(1) | |
| elif objective != 'summed-dpo': | |
| raise ValueError('Unsupported DPO objective') | |
| rewards = beta * (policy - reference) | |
| margin = rewards[:, 0] - rewards[:, 1] | |
| return -F.logsigmoid(margin), margin, (margin > 0).to(policy.dtype) | |
| def stop_requested(args, device, world): | |
| value = STOP or bool(args.stop_file and Path(args.stop_file).exists()) | |
| if world > 1: | |
| flag = torch.tensor(int(value), device=device) | |
| dist.all_reduce(flag, op=dist.ReduceOp.MAX) | |
| value = bool(flag.item()) | |
| return value | |
| def load_model(checkpoint, settings, device, tokenizer): | |
| from warpstate import WarpStateConfig, WarpStateLM | |
| from run_40b import compile_training_cores | |
| from ops.residual_experiment import apply_bf16_residuals | |
| cfg = WarpStateConfig(**checkpoint['config']) | |
| cfg.gradient_checkpointing = settings['gradient_checkpointing'] | |
| # DPO compares a deterministic policy/reference scoring rule, with dropout disabled. | |
| cfg.dropout = 0. | |
| cfg.validate() | |
| if cfg.vocab_size != tokenizer.vocab_size() or settings['sequence_length'] % cfg.chunk_size: | |
| raise ValueError('Model/tokenizer vocabulary or sequence/chunk mismatch') | |
| with torch.device('meta'): | |
| raw = WarpStateLM(cfg) | |
| raw.load_state_dict(checkpoint['model'], strict=True, assign=True) | |
| # Clone on CPU as well: mmap initialization is immutable and must not alias training. | |
| raw.to(device=device, dtype=torch.float32) | |
| if device.type == 'cpu': | |
| for parameter in raw.parameters(): | |
| parameter.data = parameter.data.clone() | |
| if settings['compile_core']: | |
| compile_training_cores(raw) | |
| apply_sequence_logps(raw, settings['head_chunk_size']) | |
| if settings['bf16_residuals']: | |
| apply_bf16_residuals(raw) | |
| return raw, cfg | |
| def atomic_array(path, array): | |
| temporary = Path(str(path) + '.tmp') | |
| with temporary.open('wb') as stream: | |
| np.save(stream, array, allow_pickle=False) | |
| stream.flush() | |
| os.fsync(stream.fileno()) | |
| os.replace(temporary, path) | |
| def prepare_reference(args, identity, records, settings, device, rank, world, tokenizer): | |
| """Create/reuse distributed resumable score shards, then seal a checked cache.""" | |
| directory = Path(args.reference_cache_dir) | |
| if (directory / 'manifest.json').exists(): | |
| return validate_cache(directory, identity, [len(rows) for rows in records]) | |
| directory.mkdir(parents=True, exist_ok=True) | |
| marker = directory / 'identity.json' | |
| if marker.exists() and json.loads(marker.read_text()) != identity: | |
| raise ValueError('Partial reference cache identity differs') | |
| if rank == 0: | |
| atomic_json(marker, identity) | |
| if world > 1: | |
| dist.barrier() | |
| checkpoint = torch.load(args.weights, map_location='cpu', weights_only=False, mmap=True) | |
| raw, _ = load_model(checkpoint, settings, device, tokenizer) | |
| raw.requires_grad_(False).eval() | |
| del checkpoint | |
| pad_id = tokenizer.pad_id() if tokenizer.pad_id() >= 0 else tokenizer.eos_id() | |
| started = time.monotonic() | |
| try: | |
| for split, rows in zip(('train', 'val'), records): | |
| indices = list(range(rank * settings['micro_batch'], len(rows), world * settings['micro_batch'])) | |
| shard = directory / f'{split}-rank-{rank}.npz' | |
| done, scored_indices, scores = 0, [], [] | |
| if shard.exists(): | |
| with np.load(shard, allow_pickle=False) as old: | |
| if str(old['identity']) != identity_digest(identity): | |
| raise ValueError('Partial reference shard identity differs') | |
| done = int(old['batches']) | |
| scored_indices = old['indices'].tolist() | |
| scores = old['scores'].tolist() | |
| expected_indices = [i for start in indices[:done] | |
| for i in range(start, min(start + settings['micro_batch'], len(rows)))] | |
| if scored_indices != expected_indices or len(scores) != len(scored_indices): | |
| raise ValueError('Partial reference shard cursor differs') | |
| def save_shard(): | |
| temporary = Path(str(shard) + '.tmp') | |
| with temporary.open('wb') as stream: | |
| np.savez(stream, identity=identity_digest(identity), batches=done, | |
| indices=np.asarray(scored_indices, dtype=np.int64), | |
| scores=np.asarray(scores, dtype=np.float64).reshape(-1, 2)) | |
| stream.flush() | |
| os.fsync(stream.fileno()) | |
| os.replace(temporary, shard) | |
| # Every rank executes equal stop/barrier counts, even on uneven final shards. | |
| max_batches = math.ceil(len(rows) / (world * settings['micro_batch'])) | |
| for batch in range(max_batches): | |
| if stop_requested(args, device, world): | |
| save_shard() | |
| return None | |
| if batch < done or batch >= len(indices): | |
| continue | |
| start = indices[batch] | |
| selected = list(range(start, min(start + settings['micro_batch'], len(rows)))) | |
| padded = [rows[i] for i in selected] + [None] * (settings['micro_batch'] - len(selected)) | |
| x, y, _ = collate_pairs(padded, settings['sequence_length'], pad_id) | |
| with torch.no_grad(), torch.autocast(device.type, dtype=torch.bfloat16, cache_enabled=False): | |
| _, logps = raw(x.to(device), y.to(device), return_logits=False) | |
| values = logps.reshape(-1, 2).double().cpu().numpy()[:len(selected)] | |
| if not np.isfinite(values).all(): | |
| raise RuntimeError('Non-finite frozen reference scores') | |
| scored_indices.extend(selected) | |
| scores.extend(values.tolist()) | |
| done = batch + 1 | |
| if done % 128 == 0 or done == len(indices): | |
| save_shard() | |
| print(json.dumps(dict(event='reference_progress', rank=rank, split=split, | |
| batches=done, total_batches=len(indices), scored_pairs=len(scores), | |
| elapsed_seconds=time.monotonic() - started)), flush=True) | |
| save_shard() | |
| if world > 1: | |
| dist.barrier() | |
| if rank == 0: | |
| files = {} | |
| for split, rows in zip(('train', 'val'), records): | |
| combined = np.full((len(rows), 2), np.nan, dtype=np.float64) | |
| seen = np.zeros(len(rows), dtype=np.bool_) | |
| for owner in range(world): | |
| with np.load(directory / f'{split}-rank-{owner}.npz', allow_pickle=False) as shard: | |
| if str(shard['identity']) != identity_digest(identity): | |
| raise ValueError('Reference shard identity differs') | |
| index = shard['indices'] | |
| if seen[index].any(): | |
| raise ValueError('Duplicate reference indices') | |
| combined[index] = shard['scores'] | |
| seen[index] = True | |
| if not seen.all() or not np.isfinite(combined).all(): | |
| raise ValueError('Reference cache is incomplete') | |
| path = directory / f'{split}.npy' | |
| atomic_array(path, combined) | |
| files[path.name] = dict(sha256=digest(path), rows=len(rows)) | |
| atomic_json(directory / 'manifest.json', dict(format=CACHE_FORMAT, complete=True, | |
| identity=identity, identity_sha256=identity_digest(identity), files=files)) | |
| if world > 1: | |
| dist.barrier() | |
| return validate_cache(directory, identity, [len(rows) for rows in records]) | |
| finally: | |
| del raw | |
| # Patched bound forwards and functional BF16 wrappers form cycles; | |
| # collect them before allocating the policy and FP32 Adam state. | |
| gc.collect() | |
| if device.type == 'cuda': | |
| torch.cuda.empty_cache() | |
| def training_update(model, optimizer, microbatches, global_pairs, world, device, settings): | |
| optimizer.zero_grad(set_to_none=True) | |
| totals = torch.zeros(5, device=device, dtype=torch.float64) | |
| for index, (x, y, active, reference) in enumerate(microbatches): | |
| sync = model.no_sync() if world > 1 and index < len(microbatches) - 1 else contextlib.nullcontext() | |
| x, y, active, reference = (value.to(device) for value in (x, y, active, reference)) | |
| counts = y.ne(-100).sum(dim=1).reshape(-1, 2) | |
| with sync: | |
| with torch.autocast(device.type, dtype=torch.bfloat16, cache_enabled=False): | |
| _, logps = model(x, y, return_logits=False) | |
| logps = logps.reshape(-1, 2) | |
| losses, margins, accuracy = preference_metrics(logps, reference, counts, | |
| settings['beta'], settings['objective']) | |
| if not torch.isfinite(losses).all(): | |
| raise RuntimeError('Non-finite DPO training loss') | |
| # DDP averages rank gradients; this is the true global pair mean, | |
| # including a partial final update and entirely empty rank slots. | |
| (losses * active).sum().mul(world / global_pairs).backward() | |
| totals += torch.stack(((losses.detach() * active).double().sum(), | |
| (margins.detach() * active).double().sum(), | |
| (accuracy.detach() * active).double().sum(), | |
| ((-logps[:, 0].detach() / counts[:, 0].clamp_min(1)) * active).double().sum(), | |
| active.double().sum())) | |
| norm = torch.nn.utils.clip_grad_norm_(model.parameters(), settings['clip_grad_norm'], error_if_nonfinite=True) | |
| optimizer.step() | |
| if world > 1: | |
| dist.all_reduce(totals) | |
| if int(totals[-1]) != global_pairs: | |
| raise RuntimeError('Distributed DPO pair count differs') | |
| values = (totals[:4] / totals[-1]).tolist() | |
| return dict(zip(('loss', 'reward_margin', 'reward_accuracy', 'chosen_nll'), values)), float(norm) | |
| def evaluate(model, records, reference, settings, device, pad_id, *, full=False): | |
| state = rng_state(device) | |
| was_training = model.training | |
| model.eval() | |
| totals = torch.zeros(4, dtype=torch.float64, device=device) | |
| count = 0 | |
| limit = len(records) if full else min(len(records), settings['eval_batches'] * settings['micro_batch']) | |
| try: | |
| with torch.no_grad(): | |
| for start in range(0, limit, settings['micro_batch']): | |
| selected = records[start:min(start + settings['micro_batch'], limit)] | |
| n = len(selected) | |
| selected += [None] * (settings['micro_batch'] - n) | |
| x, y, active = collate_pairs(selected, settings['sequence_length'], pad_id) | |
| x, y, active = x.to(device), y.to(device), active.to(device) | |
| ref = torch.zeros((settings['micro_batch'], 2), device=device) | |
| ref[:n] = torch.tensor(np.array(reference[start:start + n]), device=device) | |
| with torch.autocast(device.type, dtype=torch.bfloat16, cache_enabled=False): | |
| _, logps = model(x, y, return_logits=False) | |
| logps = logps.reshape(-1, 2) | |
| counts = y.ne(-100).sum(dim=1).reshape(-1, 2) | |
| losses, margins, accuracy = preference_metrics(logps, ref, counts, | |
| settings['beta'], settings['objective']) | |
| totals += torch.stack(((losses * active).double().sum(), (margins * active).double().sum(), | |
| (accuracy * active).double().sum(), | |
| ((-logps[:, 0] / counts[:, 0].clamp_min(1)) * active).double().sum())) | |
| count += n | |
| finally: | |
| model.train(was_training) | |
| restore_rng(state, device) | |
| values = (totals / count).tolist() | |
| if not all(math.isfinite(value) for value in values): | |
| raise RuntimeError('Non-finite DPO validation') | |
| return dict(zip(('val_loss', 'reward_margin', 'reward_accuracy', 'chosen_nll'), values)) | dict(evaluated_samples=count) | |
| def validate_resume(checkpoint, signature, world): | |
| if checkpoint.get('format') != FORMAT or checkpoint.get('signature') != signature: | |
| raise ValueError('DPO resume signature differs; use a new output directory') | |
| if len(checkpoint.get('rng_by_rank', [])) != world: | |
| raise ValueError('DPO checkpoint lacks exact per-rank RNG state') | |
| def validate_reference_path(args): | |
| cache = Path(args.reference_cache_dir).resolve() | |
| paths = [Path(path).resolve() for path in (args.work_dir, args.data_dir, args.weights.parent)] | |
| if not cache.is_relative_to(WORKSPACE) or cache == WORKSPACE: | |
| raise ValueError('Reference cache must stay inside the workspace') | |
| if any(cache == path or cache.is_relative_to(path) or path.is_relative_to(cache) for path in paths): | |
| raise ValueError('Reference cache must be separate from training outputs and inputs') | |
| return cache | |
| def train(args): | |
| import sentencepiece as spm | |
| from ops.checkpoint_writer import DistributedCheckpointWriter | |
| work, data_dir, weights, tokenizer_path, config = validate_paths( | |
| args.work_dir, args.data_dir, args.weights, args.tokenizer, args.config) | |
| cache = validate_reference_path(args) | |
| plan, settings = load_plan(config) | |
| rank = int(os.environ.get('RANK', '0')) | |
| world = int(os.environ.get('WORLD_SIZE', '1')) | |
| local_rank = int(os.environ.get('LOCAL_RANK', '0')) | |
| device = torch.device('cuda', local_rank) if args.device == 'cuda' else torch.device('cpu') | |
| if device.type == 'cuda': | |
| torch.cuda.set_device(device) | |
| if not torch.cuda.is_available() or not torch.cuda.is_bf16_supported(): | |
| raise RuntimeError('BF16-capable CUDA GPU required') | |
| torch.set_num_threads(settings['workers']) | |
| torch.use_deterministic_algorithms(settings['deterministic']) | |
| random.seed(settings['seed'] + rank) | |
| np.random.seed(settings['seed'] + rank) | |
| torch.manual_seed(settings['seed'] + rank) | |
| tokenizer_hash, base_hash = digest(tokenizer_path), digest(weights) | |
| sp = spm.SentencePieceProcessor(model_file=str(tokenizer_path)) | |
| train_data, val_data, manifest, manifest_hash = load_data(data_dir, sp, tokenizer_hash, settings['sequence_length']) | |
| source = torch.load(weights, map_location='cpu', weights_only=False, mmap=True) | |
| if (source.get('format') != 'warpstate-chat-v1' or source.get('inference_only') is not True or | |
| source.get('stage') != 'sft' or source.get('tokenizer_sha256') != tokenizer_hash): | |
| raise ValueError('Initialize DPO from an independent, tokenizer-matched SFT model export') | |
| expected = plan.get('sft', plan.get('base', {})) | |
| for key in ('step', 'tokens_seen', 'tokenizer_sha256'): | |
| if key in expected and expected[key] != source.get(key): | |
| raise ValueError(f'SFT source {key} differs from plan') | |
| ancestor = source.get('origin', {}) | |
| for key in ('step', 'tokens_seen'): | |
| expected_key = 'base_' + key | |
| value = ancestor.get(key, ancestor.get(expected_key)) | |
| if expected_key in expected and expected[expected_key] != value: | |
| raise ValueError(f'SFT source {expected_key} differs from plan') | |
| origin = {key: value for key, value in source.items() if key != 'model'} | |
| origin.update(weights_sha256=base_hash, weights_path=str(weights)) | |
| del source | |
| identity = reference_identity(base_hash, manifest_hash, tokenizer_hash, settings, device, world) | |
| if world > 1: | |
| dist.init_process_group(backend='nccl' if device.type == 'cuda' else 'gloo') | |
| try: | |
| reference = prepare_reference(args, identity, (train_data, val_data), settings, device, rank, world, sp) | |
| if reference is None: | |
| print(json.dumps(dict(event='reference_paused', rank=rank)), flush=True) | |
| return | |
| reference_train, reference_val, reference_hash = reference | |
| if getattr(args, 'reference_only', False): | |
| print(json.dumps(dict(event='reference_complete', reference_sha256=reference_hash)), flush=True) | |
| return | |
| signature = dict(plan=plan, settings=settings, data_sha256=manifest_hash, | |
| tokenizer_sha256=tokenizer_hash, renderer=RENDER_FORMAT, mask=MASK_FORMAT, | |
| base_sha256=base_hash, reference_sha256=reference_hash, | |
| world_size=world, device_type=device.type, torch_version=torch.__version__, sources=source_identity()) | |
| latest = work / 'latest.pt' | |
| resuming = latest.exists() | |
| saved = torch.load(latest if resuming else weights, map_location='cpu', weights_only=False, mmap=True) | |
| if resuming: | |
| validate_resume(saved, signature, world) | |
| origin = saved['origin'] | |
| raw, cfg = load_model(saved, settings, device, sp) | |
| raw.requires_grad_(True) | |
| optimizer = torch.optim.AdamW(raw.parameters(), lr=settings['lr'], betas=(.9, .95), | |
| eps=1e-8, weight_decay=settings['weight_decay'], fused=device.type == 'cuda') | |
| step = cursor = tokens_seen = response_tokens_seen = 0 | |
| if resuming: | |
| optimizer.load_state_dict(saved['optimizer']) | |
| step, cursor = saved['step'], saved['sample_cursor'] | |
| tokens_seen, response_tokens_seen = saved['tokens_seen'], saved['response_tokens_seen'] | |
| resume_rng = saved['rng_by_rank'][rank] if resuming else None | |
| del saved | |
| if world > 1: | |
| model = torch.nn.parallel.DistributedDataParallel(raw, | |
| device_ids=[local_rank] if device.type == 'cuda' else None, | |
| broadcast_buffers=False, gradient_as_bucket_view=True, bucket_cap_mb=128) | |
| else: | |
| model = raw | |
| if resume_rng is not None: | |
| restore_rng(resume_rng, device) | |
| global_batch = settings['micro_batch'] * settings['grad_accum'] * world | |
| total_steps = math.ceil(len(train_data) / global_batch) * settings['epochs'] | |
| if settings['max_steps'] is not None: | |
| total_steps = min(total_steps, settings['max_steps']) | |
| end_step = min(total_steps, step + args.max_steps) if args.max_steps is not None else total_steps | |
| work.mkdir(parents=True, exist_ok=True) | |
| if rank == 0: | |
| atomic_json(work / 'settings.json', dict(signature=signature, origin=origin, | |
| total_steps=total_steps, train_rows=len(train_data), val_rows=len(val_data))) | |
| if step < total_steps: | |
| (work / 'completion.json').unlink(missing_ok=True) | |
| pad_id = sp.pad_id() if sp.pad_id() >= 0 else sp.eos_id() | |
| last_validation = None | |
| last_validation_step = None | |
| def validation(full=False): | |
| values = [evaluate(raw, val_data, reference_val, settings, device, pad_id, full=full) if rank == 0 else None] | |
| if world > 1: | |
| dist.broadcast_object_list(values, src=0, device=device) | |
| return values[0] | |
| def metric(values): | |
| if rank == 0: | |
| values.update(timestamp=time.time(), stage='dpo', step=step, tokens_seen=tokens_seen, | |
| response_tokens_seen=response_tokens_seen, sample_cursor=cursor, | |
| objective=settings['objective']) | |
| with (work / 'metrics.jsonl').open('a') as stream: | |
| stream.write(json.dumps(values, allow_nan=False) + '\n') | |
| if values.get('event') == 'validation': | |
| atomic_json(work / 'evaluation.json', dict(values, data_sha256=manifest_hash, | |
| base_sha256=base_hash, reference_sha256=reference_hash, | |
| tokenizer_sha256=tokenizer_hash, mask_format=MASK_FORMAT, | |
| available_validation_samples=len(val_data))) | |
| print(json.dumps(values, allow_nan=False), flush=True) | |
| def save(writer): | |
| state = rng_state(device) | |
| states = [None] * world | |
| if world > 1: | |
| dist.all_gather_object(states, state) | |
| else: | |
| states = [state] | |
| writer.submit(state_factory=lambda: dict(format=FORMAT, signature=signature, origin=origin, | |
| config=vars(cfg), model=raw.state_dict(), optimizer=optimizer.state_dict(), step=step, | |
| sample_cursor=cursor, tokens_seen=tokens_seen, response_tokens_seen=response_tokens_seen, | |
| val_loss=last_validation['val_loss'], validation=last_validation, | |
| val_step=last_validation_step, rng_by_rank=states, completed=step >= total_steps, | |
| total_steps=total_steps)) | |
| with DistributedCheckpointWriter(latest, rank=rank, device=device, distributed=world > 1) as writer: | |
| last_validation = validation(full=True) | |
| last_validation_step = step | |
| metric(dict(event='validation', full_holdout=True, **last_validation)) | |
| last_saved_step = step if resuming else -1 | |
| model.train() | |
| while step < end_step and not stop_requested(args, device, world): | |
| writer.check() | |
| indices, next_cursor = update_indices(len(train_data), settings['seed'], cursor, | |
| settings['epochs'], global_batch) | |
| if not indices: | |
| break | |
| microbatches = [] | |
| for micro in range(settings['grad_accum']): | |
| offset = (micro * world + rank) * settings['micro_batch'] | |
| selected = indices[offset:offset + settings['micro_batch']] | |
| padded = [train_data[i] for i in selected] + [None] * (settings['micro_batch'] - len(selected)) | |
| x, y, active = collate_pairs(padded, settings['sequence_length'], pad_id) | |
| ref = torch.zeros((settings['micro_batch'], 2), dtype=torch.float32) | |
| if selected: | |
| ref[:len(selected)] = torch.tensor(np.array(reference_train[selected]), dtype=torch.float32) | |
| microbatches.append((x, y, active, ref)) | |
| lr = learning_rate(step, total_steps, settings) | |
| for group in optimizer.param_groups: | |
| group['lr'] = lr | |
| started = time.perf_counter() | |
| values, norm = training_update(model, optimizer, microbatches, len(indices), world, device, settings) | |
| if device.type == 'cuda': | |
| torch.cuda.synchronize(device) | |
| seconds = time.perf_counter() - started | |
| step += 1 | |
| cursor = next_cursor | |
| actual_tokens = sum(len(train_data[i][branch]['input_ids']) - 1 | |
| for i in indices for branch in ('chosen', 'rejected')) | |
| tokens_seen += actual_tokens | |
| response_tokens_seen += sum(sum(value != -100 for value in train_data[i][branch]['labels'][1:]) | |
| for i in indices for branch in ('chosen', 'rejected')) | |
| metric(dict(event='train', **values, lr=lr, grad_norm=norm, update_seconds=seconds, | |
| tokens_per_second=actual_tokens / seconds, update_samples=len(indices))) | |
| if step % settings['eval_every'] == 0 or step == end_step: | |
| full = step >= total_steps | |
| last_validation = validation(full=full) | |
| last_validation_step = step | |
| metric(dict(event='validation', full_holdout=full, **last_validation)) | |
| if step % settings['save_every'] == 0 or step == end_step: | |
| save(writer) | |
| last_saved_step = step | |
| if last_saved_step != step: | |
| save(writer) | |
| writer.flush() | |
| if rank == 0 and step >= total_steps: | |
| atomic_json(work / 'completion.json', dict(format='warpstate-dpo-completion-v1', | |
| stage='dpo', completed=True, paused_for_evaluation=True, step=step, | |
| total_steps=total_steps, sample_cursor=cursor, tokens_seen=tokens_seen, | |
| response_tokens_seen=response_tokens_seen, val_loss=last_validation['val_loss'], | |
| evaluation_file=str(work / 'evaluation.json'), checkpoint=str(latest), | |
| reference_sha256=reference_hash, timestamp=time.time())) | |
| metric(dict(event='complete' if step >= total_steps else 'paused', total_steps=total_steps, | |
| checkpoint=str(latest))) | |
| validate_cache(cache, identity, (len(train_data), len(val_data))) | |
| if digest(weights) != base_hash: | |
| raise RuntimeError('Frozen SFT initialization changed during DPO') | |
| finally: | |
| if world > 1 and dist.is_initialized(): | |
| dist.destroy_process_group() | |
| def main(): | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| for name in ('config', 'work-dir', 'data-dir', 'weights', 'tokenizer', 'reference-cache-dir'): | |
| parser.add_argument('--' + name, required=True, type=Path) | |
| parser.add_argument('--max-steps', type=int) | |
| parser.add_argument('--stop-file', type=Path) | |
| parser.add_argument('--device', choices=('cuda', 'cpu'), default='cuda') | |
| parser.add_argument('--reference-only', action='store_true') | |
| args = parser.parse_args() | |
| if args.max_steps is not None and args.max_steps < 1: | |
| parser.error('--max-steps must be positive') | |
| if args.stop_file is None: | |
| args.stop_file = args.work_dir / 'control/training-stop-request' | |
| if not args.stop_file.resolve().is_relative_to(WORKSPACE): | |
| parser.error('--stop-file must stay inside the workspace') | |
| def stop(signum, frame): | |
| global STOP | |
| STOP = True | |
| for signum in (signal.SIGTERM, signal.SIGINT, signal.SIGUSR1): | |
| signal.signal(signum, stop) | |
| from run_40b import process_lock | |
| validate_paths(args.work_dir, args.data_dir, args.weights, args.tokenizer, args.config) | |
| validate_reference_path(args) | |
| args.work_dir.mkdir(parents=True, exist_ok=True) | |
| args.reference_cache_dir.mkdir(parents=True, exist_ok=True) | |
| plan, _ = load_plan(args.config) | |
| rank = int(os.environ.get('RANK', '0')) | |
| with process_lock(args.work_dir / f'rank-{rank}.lock'), \ | |
| process_lock(args.reference_cache_dir / f'rank-{rank}.lock'), base_pipeline_lock(plan): | |
| train(args) | |
| if __name__ == '__main__': | |
| main() | |