#!/usr/bin/env python3 """BF16 DPO with an immutable SFT reference, exact resume and response-only loss. The reference is scored once, before allocating the policy optimizer. Its cache is bound to weights, data, masking, execution and source hashes. Policy training uses fresh FP32 Adam state and saves only into its own output directory. """ from __future__ import annotations import argparse import contextlib import gc import hashlib import json import math import os from pathlib import Path import random import signal import time import types import numpy as np import torch import torch.distributed as dist import torch.nn.functional as F from torch.utils.checkpoint import checkpoint as activation_checkpoint from posttrain import (WORKSPACE, atomic_json, base_pipeline_lock, digest, restore_rng, rng_state, update_indices, validate_paths) from ops.dpo_data import MASK_FORMAT, collate_pairs, load_data from ops.posttrain_data import RENDER_FORMAT, renderer_sha256 FORMAT = 'warpstate-dpo-v1' CACHE_FORMAT = 'warpstate-dpo-reference-v1' STOP = False DEFAULTS = dict(sequence_length=2048, micro_batch=2, grad_accum=8, lr=5e-7, min_lr=0., warmup_ratio=.1, epochs=1, max_steps=None, weight_decay=0., seed=42, eval_every=100, save_every=100, eval_batches=32, compile_core=True, bf16_residuals=True, gradient_checkpointing=False, workers=4, deterministic=True, clip_grad_norm=1., beta=5., objective='length-normalized-dpo', head_chunk_size=256) def load_plan(path): plan = json.loads(Path(path).read_text()) if plan.get('format') != 'warpstate-dpo-plan-v1' or plan.get('stage') != 'dpo': raise ValueError('Expected warpstate-dpo-plan-v1 stage dpo') supplied = plan.get('training', {}) unknown = set(supplied) - set(DEFAULTS) if unknown: raise ValueError(f'Unknown DPO training settings: {sorted(unknown)}') settings = DEFAULTS | supplied for key in ('sequence_length', 'micro_batch', 'grad_accum', 'epochs', 'eval_every', 'save_every', 'eval_batches', 'workers', 'head_chunk_size'): if type(settings[key]) is not int or settings[key] < 1: raise ValueError(f'{key} must be a positive integer') if settings['max_steps'] is not None and (type(settings['max_steps']) is not int or settings['max_steps'] < 1): raise ValueError('max_steps must be null or a positive integer') for key in ('lr', 'min_lr', 'warmup_ratio', 'weight_decay', 'clip_grad_norm', 'beta'): value = settings[key] if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) or value < 0: raise ValueError(f'{key} must be finite and nonnegative') if not 0 <= settings['warmup_ratio'] < 1 or not 0 <= settings['min_lr'] <= settings['lr']: raise ValueError('Invalid LR schedule') if not settings['lr'] or not settings['clip_grad_norm'] or not settings['beta']: raise ValueError('lr, clip_grad_norm and beta must be positive') for key in ('compile_core', 'bf16_residuals', 'gradient_checkpointing', 'deterministic'): if type(settings[key]) is not bool: raise ValueError(f'{key} must be a boolean') if type(settings['seed']) is not int or settings['seed'] < 0: raise ValueError('seed must be a nonnegative integer') if settings['objective'] not in ('length-normalized-dpo', 'summed-dpo'): raise ValueError('Unsupported DPO objective') return plan, settings def source_identity(): from posttrain import source_identity as shared_sources root = Path(__file__).parent return shared_sources() | {str(path.relative_to(root)): digest(path) for path in (Path(__file__), root / 'ops/dpo_data.py')} def identity_digest(identity): return hashlib.sha256(json.dumps(identity, sort_keys=True, separators=(',', ':'), allow_nan=False).encode()).hexdigest() def learning_rate(step, total_steps, settings): """Fixed linear warmup/decay schedule, independent of a pause boundary.""" warmup = min(math.ceil(settings['warmup_ratio'] * total_steps), max(0, total_steps - 1)) if step < warmup: return settings['lr'] * (step + 1) / warmup progress = min(1., (step - warmup) / max(1, total_steps - warmup - 1)) return settings['min_lr'] + (settings['lr'] - settings['min_lr']) * (1. - progress) def reference_identity(weights_hash, manifest_hash, tokenizer_hash, settings, device, world): return dict(format=CACHE_FORMAT, weights_sha256=weights_hash, data_sha256=manifest_hash, tokenizer_sha256=tokenizer_hash, renderer_sha256=renderer_sha256(), mask_format=MASK_FORMAT, sources=source_identity(), settings={key: settings[key] for key in ('sequence_length', 'micro_batch', 'head_chunk_size', 'compile_core', 'bf16_residuals', 'deterministic', 'workers')}, dropout=0., logps='response-token-sum-float32', dtype='bfloat16', device_type=device.type, world_size=world, torch_version=torch.__version__) def validate_cache(directory, identity, sizes): directory = Path(directory) manifest = json.loads((directory / 'manifest.json').read_text()) if (manifest.get('format') != CACHE_FORMAT or manifest.get('complete') is not True or manifest.get('identity') != identity or manifest.get('identity_sha256') != identity_digest(identity)): raise ValueError('Reference cache identity/completion differs') result = [] for name, size in zip(('train.npy', 'val.npy'), sizes): path = directory / name info = manifest.get('files', {}).get(name, {}) if not path.is_file() or path.is_symlink() or digest(path) != info.get('sha256') or info.get('rows') != size: raise ValueError(f'Reference cache file changed: {name}') values = np.load(path, allow_pickle=False, mmap_mode='r') if values.shape != (size, 2) or values.dtype != np.float64 or not np.isfinite(values).all(): raise ValueError(f'Invalid reference log probabilities: {name}') result.append(values) return *result, digest(directory / 'manifest.json') def apply_sequence_logps(model, chunk_size): """Return per-sequence log-probability sums without retaining vocabulary logits. Vocabulary matmuls use BF16 autocast; cross-entropy reductions use FP32. Each head chunk is recomputed in backward, so DPO keeps a vector of branch scores. The backbone modules and canonical parameter/state names are unchanged. """ def forward(self, input_ids, targets=None, *, return_logits=False): if targets is None or return_logits: raise ValueError('DPO forward requires response targets and no returned logits') hidden = self.embed(input_ids) for depth in range(self.cfg.logical_depth): hidden = hidden * (1. + self.depth_scale[depth]) + self.depth_bias[depth] core = self.cores[depth % self.cfg.n_cores] if self.training and self.cfg.gradient_checkpointing: hidden = activation_checkpoint(core, hidden, use_reentrant=False) else: hidden = core(hidden) hidden = self.final_norm(hidden) def score(h, weight, labels): logits = F.linear(h, weight).float() losses = F.cross_entropy(logits.reshape(-1, logits.size(-1)), labels.reshape(-1), reduction='none', ignore_index=-100) return -losses.reshape(labels.shape).sum(dim=1) logps = hidden.new_zeros((hidden.shape[0],), dtype=torch.float32) for start in range(0, hidden.shape[1], chunk_size): args = (hidden[:, start:start + chunk_size], self.embed.weight, targets[:, start:start + chunk_size]) value = activation_checkpoint(score, *args, use_reentrant=False) if torch.is_grad_enabled() else score(*args) logps = logps + value return None, logps model.forward = types.MethodType(forward, model) def preference_metrics(policy, reference, counts, beta, objective): """Pair vectors, retaining gradients only through policy log probabilities.""" reference = reference.detach().to(policy.dtype) if objective == 'length-normalized-dpo': policy = policy / counts.clamp_min(1) reference = reference / counts.clamp_min(1) elif objective != 'summed-dpo': raise ValueError('Unsupported DPO objective') rewards = beta * (policy - reference) margin = rewards[:, 0] - rewards[:, 1] return -F.logsigmoid(margin), margin, (margin > 0).to(policy.dtype) def stop_requested(args, device, world): value = STOP or bool(args.stop_file and Path(args.stop_file).exists()) if world > 1: flag = torch.tensor(int(value), device=device) dist.all_reduce(flag, op=dist.ReduceOp.MAX) value = bool(flag.item()) return value def load_model(checkpoint, settings, device, tokenizer): from warpstate import WarpStateConfig, WarpStateLM from run_40b import compile_training_cores from ops.residual_experiment import apply_bf16_residuals cfg = WarpStateConfig(**checkpoint['config']) cfg.gradient_checkpointing = settings['gradient_checkpointing'] # DPO compares a deterministic policy/reference scoring rule, with dropout disabled. cfg.dropout = 0. cfg.validate() if cfg.vocab_size != tokenizer.vocab_size() or settings['sequence_length'] % cfg.chunk_size: raise ValueError('Model/tokenizer vocabulary or sequence/chunk mismatch') with torch.device('meta'): raw = WarpStateLM(cfg) raw.load_state_dict(checkpoint['model'], strict=True, assign=True) # Clone on CPU as well: mmap initialization is immutable and must not alias training. raw.to(device=device, dtype=torch.float32) if device.type == 'cpu': for parameter in raw.parameters(): parameter.data = parameter.data.clone() if settings['compile_core']: compile_training_cores(raw) apply_sequence_logps(raw, settings['head_chunk_size']) if settings['bf16_residuals']: apply_bf16_residuals(raw) return raw, cfg def atomic_array(path, array): temporary = Path(str(path) + '.tmp') with temporary.open('wb') as stream: np.save(stream, array, allow_pickle=False) stream.flush() os.fsync(stream.fileno()) os.replace(temporary, path) def prepare_reference(args, identity, records, settings, device, rank, world, tokenizer): """Create/reuse distributed resumable score shards, then seal a checked cache.""" directory = Path(args.reference_cache_dir) if (directory / 'manifest.json').exists(): return validate_cache(directory, identity, [len(rows) for rows in records]) directory.mkdir(parents=True, exist_ok=True) marker = directory / 'identity.json' if marker.exists() and json.loads(marker.read_text()) != identity: raise ValueError('Partial reference cache identity differs') if rank == 0: atomic_json(marker, identity) if world > 1: dist.barrier() checkpoint = torch.load(args.weights, map_location='cpu', weights_only=False, mmap=True) raw, _ = load_model(checkpoint, settings, device, tokenizer) raw.requires_grad_(False).eval() del checkpoint pad_id = tokenizer.pad_id() if tokenizer.pad_id() >= 0 else tokenizer.eos_id() started = time.monotonic() try: for split, rows in zip(('train', 'val'), records): indices = list(range(rank * settings['micro_batch'], len(rows), world * settings['micro_batch'])) shard = directory / f'{split}-rank-{rank}.npz' done, scored_indices, scores = 0, [], [] if shard.exists(): with np.load(shard, allow_pickle=False) as old: if str(old['identity']) != identity_digest(identity): raise ValueError('Partial reference shard identity differs') done = int(old['batches']) scored_indices = old['indices'].tolist() scores = old['scores'].tolist() expected_indices = [i for start in indices[:done] for i in range(start, min(start + settings['micro_batch'], len(rows)))] if scored_indices != expected_indices or len(scores) != len(scored_indices): raise ValueError('Partial reference shard cursor differs') def save_shard(): temporary = Path(str(shard) + '.tmp') with temporary.open('wb') as stream: np.savez(stream, identity=identity_digest(identity), batches=done, indices=np.asarray(scored_indices, dtype=np.int64), scores=np.asarray(scores, dtype=np.float64).reshape(-1, 2)) stream.flush() os.fsync(stream.fileno()) os.replace(temporary, shard) # Every rank executes equal stop/barrier counts, even on uneven final shards. max_batches = math.ceil(len(rows) / (world * settings['micro_batch'])) for batch in range(max_batches): if stop_requested(args, device, world): save_shard() return None if batch < done or batch >= len(indices): continue start = indices[batch] selected = list(range(start, min(start + settings['micro_batch'], len(rows)))) padded = [rows[i] for i in selected] + [None] * (settings['micro_batch'] - len(selected)) x, y, _ = collate_pairs(padded, settings['sequence_length'], pad_id) with torch.no_grad(), torch.autocast(device.type, dtype=torch.bfloat16, cache_enabled=False): _, logps = raw(x.to(device), y.to(device), return_logits=False) values = logps.reshape(-1, 2).double().cpu().numpy()[:len(selected)] if not np.isfinite(values).all(): raise RuntimeError('Non-finite frozen reference scores') scored_indices.extend(selected) scores.extend(values.tolist()) done = batch + 1 if done % 128 == 0 or done == len(indices): save_shard() print(json.dumps(dict(event='reference_progress', rank=rank, split=split, batches=done, total_batches=len(indices), scored_pairs=len(scores), elapsed_seconds=time.monotonic() - started)), flush=True) save_shard() if world > 1: dist.barrier() if rank == 0: files = {} for split, rows in zip(('train', 'val'), records): combined = np.full((len(rows), 2), np.nan, dtype=np.float64) seen = np.zeros(len(rows), dtype=np.bool_) for owner in range(world): with np.load(directory / f'{split}-rank-{owner}.npz', allow_pickle=False) as shard: if str(shard['identity']) != identity_digest(identity): raise ValueError('Reference shard identity differs') index = shard['indices'] if seen[index].any(): raise ValueError('Duplicate reference indices') combined[index] = shard['scores'] seen[index] = True if not seen.all() or not np.isfinite(combined).all(): raise ValueError('Reference cache is incomplete') path = directory / f'{split}.npy' atomic_array(path, combined) files[path.name] = dict(sha256=digest(path), rows=len(rows)) atomic_json(directory / 'manifest.json', dict(format=CACHE_FORMAT, complete=True, identity=identity, identity_sha256=identity_digest(identity), files=files)) if world > 1: dist.barrier() return validate_cache(directory, identity, [len(rows) for rows in records]) finally: del raw # Patched bound forwards and functional BF16 wrappers form cycles; # collect them before allocating the policy and FP32 Adam state. gc.collect() if device.type == 'cuda': torch.cuda.empty_cache() def training_update(model, optimizer, microbatches, global_pairs, world, device, settings): optimizer.zero_grad(set_to_none=True) totals = torch.zeros(5, device=device, dtype=torch.float64) for index, (x, y, active, reference) in enumerate(microbatches): sync = model.no_sync() if world > 1 and index < len(microbatches) - 1 else contextlib.nullcontext() x, y, active, reference = (value.to(device) for value in (x, y, active, reference)) counts = y.ne(-100).sum(dim=1).reshape(-1, 2) with sync: with torch.autocast(device.type, dtype=torch.bfloat16, cache_enabled=False): _, logps = model(x, y, return_logits=False) logps = logps.reshape(-1, 2) losses, margins, accuracy = preference_metrics(logps, reference, counts, settings['beta'], settings['objective']) if not torch.isfinite(losses).all(): raise RuntimeError('Non-finite DPO training loss') # DDP averages rank gradients; this is the true global pair mean, # including a partial final update and entirely empty rank slots. (losses * active).sum().mul(world / global_pairs).backward() totals += torch.stack(((losses.detach() * active).double().sum(), (margins.detach() * active).double().sum(), (accuracy.detach() * active).double().sum(), ((-logps[:, 0].detach() / counts[:, 0].clamp_min(1)) * active).double().sum(), active.double().sum())) norm = torch.nn.utils.clip_grad_norm_(model.parameters(), settings['clip_grad_norm'], error_if_nonfinite=True) optimizer.step() if world > 1: dist.all_reduce(totals) if int(totals[-1]) != global_pairs: raise RuntimeError('Distributed DPO pair count differs') values = (totals[:4] / totals[-1]).tolist() return dict(zip(('loss', 'reward_margin', 'reward_accuracy', 'chosen_nll'), values)), float(norm) def evaluate(model, records, reference, settings, device, pad_id, *, full=False): state = rng_state(device) was_training = model.training model.eval() totals = torch.zeros(4, dtype=torch.float64, device=device) count = 0 limit = len(records) if full else min(len(records), settings['eval_batches'] * settings['micro_batch']) try: with torch.no_grad(): for start in range(0, limit, settings['micro_batch']): selected = records[start:min(start + settings['micro_batch'], limit)] n = len(selected) selected += [None] * (settings['micro_batch'] - n) x, y, active = collate_pairs(selected, settings['sequence_length'], pad_id) x, y, active = x.to(device), y.to(device), active.to(device) ref = torch.zeros((settings['micro_batch'], 2), device=device) ref[:n] = torch.tensor(np.array(reference[start:start + n]), device=device) with torch.autocast(device.type, dtype=torch.bfloat16, cache_enabled=False): _, logps = model(x, y, return_logits=False) logps = logps.reshape(-1, 2) counts = y.ne(-100).sum(dim=1).reshape(-1, 2) losses, margins, accuracy = preference_metrics(logps, ref, counts, settings['beta'], settings['objective']) totals += torch.stack(((losses * active).double().sum(), (margins * active).double().sum(), (accuracy * active).double().sum(), ((-logps[:, 0] / counts[:, 0].clamp_min(1)) * active).double().sum())) count += n finally: model.train(was_training) restore_rng(state, device) values = (totals / count).tolist() if not all(math.isfinite(value) for value in values): raise RuntimeError('Non-finite DPO validation') return dict(zip(('val_loss', 'reward_margin', 'reward_accuracy', 'chosen_nll'), values)) | dict(evaluated_samples=count) def validate_resume(checkpoint, signature, world): if checkpoint.get('format') != FORMAT or checkpoint.get('signature') != signature: raise ValueError('DPO resume signature differs; use a new output directory') if len(checkpoint.get('rng_by_rank', [])) != world: raise ValueError('DPO checkpoint lacks exact per-rank RNG state') def validate_reference_path(args): cache = Path(args.reference_cache_dir).resolve() paths = [Path(path).resolve() for path in (args.work_dir, args.data_dir, args.weights.parent)] if not cache.is_relative_to(WORKSPACE) or cache == WORKSPACE: raise ValueError('Reference cache must stay inside the workspace') if any(cache == path or cache.is_relative_to(path) or path.is_relative_to(cache) for path in paths): raise ValueError('Reference cache must be separate from training outputs and inputs') return cache def train(args): import sentencepiece as spm from ops.checkpoint_writer import DistributedCheckpointWriter work, data_dir, weights, tokenizer_path, config = validate_paths( args.work_dir, args.data_dir, args.weights, args.tokenizer, args.config) cache = validate_reference_path(args) plan, settings = load_plan(config) rank = int(os.environ.get('RANK', '0')) world = int(os.environ.get('WORLD_SIZE', '1')) local_rank = int(os.environ.get('LOCAL_RANK', '0')) device = torch.device('cuda', local_rank) if args.device == 'cuda' else torch.device('cpu') if device.type == 'cuda': torch.cuda.set_device(device) if not torch.cuda.is_available() or not torch.cuda.is_bf16_supported(): raise RuntimeError('BF16-capable CUDA GPU required') torch.set_num_threads(settings['workers']) torch.use_deterministic_algorithms(settings['deterministic']) random.seed(settings['seed'] + rank) np.random.seed(settings['seed'] + rank) torch.manual_seed(settings['seed'] + rank) tokenizer_hash, base_hash = digest(tokenizer_path), digest(weights) sp = spm.SentencePieceProcessor(model_file=str(tokenizer_path)) train_data, val_data, manifest, manifest_hash = load_data(data_dir, sp, tokenizer_hash, settings['sequence_length']) source = torch.load(weights, map_location='cpu', weights_only=False, mmap=True) if (source.get('format') != 'warpstate-chat-v1' or source.get('inference_only') is not True or source.get('stage') != 'sft' or source.get('tokenizer_sha256') != tokenizer_hash): raise ValueError('Initialize DPO from an independent, tokenizer-matched SFT model export') expected = plan.get('sft', plan.get('base', {})) for key in ('step', 'tokens_seen', 'tokenizer_sha256'): if key in expected and expected[key] != source.get(key): raise ValueError(f'SFT source {key} differs from plan') ancestor = source.get('origin', {}) for key in ('step', 'tokens_seen'): expected_key = 'base_' + key value = ancestor.get(key, ancestor.get(expected_key)) if expected_key in expected and expected[expected_key] != value: raise ValueError(f'SFT source {expected_key} differs from plan') origin = {key: value for key, value in source.items() if key != 'model'} origin.update(weights_sha256=base_hash, weights_path=str(weights)) del source identity = reference_identity(base_hash, manifest_hash, tokenizer_hash, settings, device, world) if world > 1: dist.init_process_group(backend='nccl' if device.type == 'cuda' else 'gloo') try: reference = prepare_reference(args, identity, (train_data, val_data), settings, device, rank, world, sp) if reference is None: print(json.dumps(dict(event='reference_paused', rank=rank)), flush=True) return reference_train, reference_val, reference_hash = reference if getattr(args, 'reference_only', False): print(json.dumps(dict(event='reference_complete', reference_sha256=reference_hash)), flush=True) return signature = dict(plan=plan, settings=settings, data_sha256=manifest_hash, tokenizer_sha256=tokenizer_hash, renderer=RENDER_FORMAT, mask=MASK_FORMAT, base_sha256=base_hash, reference_sha256=reference_hash, world_size=world, device_type=device.type, torch_version=torch.__version__, sources=source_identity()) latest = work / 'latest.pt' resuming = latest.exists() saved = torch.load(latest if resuming else weights, map_location='cpu', weights_only=False, mmap=True) if resuming: validate_resume(saved, signature, world) origin = saved['origin'] raw, cfg = load_model(saved, settings, device, sp) raw.requires_grad_(True) optimizer = torch.optim.AdamW(raw.parameters(), lr=settings['lr'], betas=(.9, .95), eps=1e-8, weight_decay=settings['weight_decay'], fused=device.type == 'cuda') step = cursor = tokens_seen = response_tokens_seen = 0 if resuming: optimizer.load_state_dict(saved['optimizer']) step, cursor = saved['step'], saved['sample_cursor'] tokens_seen, response_tokens_seen = saved['tokens_seen'], saved['response_tokens_seen'] resume_rng = saved['rng_by_rank'][rank] if resuming else None del saved if world > 1: model = torch.nn.parallel.DistributedDataParallel(raw, device_ids=[local_rank] if device.type == 'cuda' else None, broadcast_buffers=False, gradient_as_bucket_view=True, bucket_cap_mb=128) else: model = raw if resume_rng is not None: restore_rng(resume_rng, device) global_batch = settings['micro_batch'] * settings['grad_accum'] * world total_steps = math.ceil(len(train_data) / global_batch) * settings['epochs'] if settings['max_steps'] is not None: total_steps = min(total_steps, settings['max_steps']) end_step = min(total_steps, step + args.max_steps) if args.max_steps is not None else total_steps work.mkdir(parents=True, exist_ok=True) if rank == 0: atomic_json(work / 'settings.json', dict(signature=signature, origin=origin, total_steps=total_steps, train_rows=len(train_data), val_rows=len(val_data))) if step < total_steps: (work / 'completion.json').unlink(missing_ok=True) pad_id = sp.pad_id() if sp.pad_id() >= 0 else sp.eos_id() last_validation = None last_validation_step = None def validation(full=False): values = [evaluate(raw, val_data, reference_val, settings, device, pad_id, full=full) if rank == 0 else None] if world > 1: dist.broadcast_object_list(values, src=0, device=device) return values[0] def metric(values): if rank == 0: values.update(timestamp=time.time(), stage='dpo', step=step, tokens_seen=tokens_seen, response_tokens_seen=response_tokens_seen, sample_cursor=cursor, objective=settings['objective']) with (work / 'metrics.jsonl').open('a') as stream: stream.write(json.dumps(values, allow_nan=False) + '\n') if values.get('event') == 'validation': atomic_json(work / 'evaluation.json', dict(values, data_sha256=manifest_hash, base_sha256=base_hash, reference_sha256=reference_hash, tokenizer_sha256=tokenizer_hash, mask_format=MASK_FORMAT, available_validation_samples=len(val_data))) print(json.dumps(values, allow_nan=False), flush=True) def save(writer): state = rng_state(device) states = [None] * world if world > 1: dist.all_gather_object(states, state) else: states = [state] writer.submit(state_factory=lambda: dict(format=FORMAT, signature=signature, origin=origin, config=vars(cfg), model=raw.state_dict(), optimizer=optimizer.state_dict(), step=step, sample_cursor=cursor, tokens_seen=tokens_seen, response_tokens_seen=response_tokens_seen, val_loss=last_validation['val_loss'], validation=last_validation, val_step=last_validation_step, rng_by_rank=states, completed=step >= total_steps, total_steps=total_steps)) with DistributedCheckpointWriter(latest, rank=rank, device=device, distributed=world > 1) as writer: last_validation = validation(full=True) last_validation_step = step metric(dict(event='validation', full_holdout=True, **last_validation)) last_saved_step = step if resuming else -1 model.train() while step < end_step and not stop_requested(args, device, world): writer.check() indices, next_cursor = update_indices(len(train_data), settings['seed'], cursor, settings['epochs'], global_batch) if not indices: break microbatches = [] for micro in range(settings['grad_accum']): offset = (micro * world + rank) * settings['micro_batch'] selected = indices[offset:offset + settings['micro_batch']] padded = [train_data[i] for i in selected] + [None] * (settings['micro_batch'] - len(selected)) x, y, active = collate_pairs(padded, settings['sequence_length'], pad_id) ref = torch.zeros((settings['micro_batch'], 2), dtype=torch.float32) if selected: ref[:len(selected)] = torch.tensor(np.array(reference_train[selected]), dtype=torch.float32) microbatches.append((x, y, active, ref)) lr = learning_rate(step, total_steps, settings) for group in optimizer.param_groups: group['lr'] = lr started = time.perf_counter() values, norm = training_update(model, optimizer, microbatches, len(indices), world, device, settings) if device.type == 'cuda': torch.cuda.synchronize(device) seconds = time.perf_counter() - started step += 1 cursor = next_cursor actual_tokens = sum(len(train_data[i][branch]['input_ids']) - 1 for i in indices for branch in ('chosen', 'rejected')) tokens_seen += actual_tokens response_tokens_seen += sum(sum(value != -100 for value in train_data[i][branch]['labels'][1:]) for i in indices for branch in ('chosen', 'rejected')) metric(dict(event='train', **values, lr=lr, grad_norm=norm, update_seconds=seconds, tokens_per_second=actual_tokens / seconds, update_samples=len(indices))) if step % settings['eval_every'] == 0 or step == end_step: full = step >= total_steps last_validation = validation(full=full) last_validation_step = step metric(dict(event='validation', full_holdout=full, **last_validation)) if step % settings['save_every'] == 0 or step == end_step: save(writer) last_saved_step = step if last_saved_step != step: save(writer) writer.flush() if rank == 0 and step >= total_steps: atomic_json(work / 'completion.json', dict(format='warpstate-dpo-completion-v1', stage='dpo', completed=True, paused_for_evaluation=True, step=step, total_steps=total_steps, sample_cursor=cursor, tokens_seen=tokens_seen, response_tokens_seen=response_tokens_seen, val_loss=last_validation['val_loss'], evaluation_file=str(work / 'evaluation.json'), checkpoint=str(latest), reference_sha256=reference_hash, timestamp=time.time())) metric(dict(event='complete' if step >= total_steps else 'paused', total_steps=total_steps, checkpoint=str(latest))) validate_cache(cache, identity, (len(train_data), len(val_data))) if digest(weights) != base_hash: raise RuntimeError('Frozen SFT initialization changed during DPO') finally: if world > 1 and dist.is_initialized(): dist.destroy_process_group() def main(): parser = argparse.ArgumentParser(description=__doc__) for name in ('config', 'work-dir', 'data-dir', 'weights', 'tokenizer', 'reference-cache-dir'): parser.add_argument('--' + name, required=True, type=Path) parser.add_argument('--max-steps', type=int) parser.add_argument('--stop-file', type=Path) parser.add_argument('--device', choices=('cuda', 'cpu'), default='cuda') parser.add_argument('--reference-only', action='store_true') args = parser.parse_args() if args.max_steps is not None and args.max_steps < 1: parser.error('--max-steps must be positive') if args.stop_file is None: args.stop_file = args.work_dir / 'control/training-stop-request' if not args.stop_file.resolve().is_relative_to(WORKSPACE): parser.error('--stop-file must stay inside the workspace') def stop(signum, frame): global STOP STOP = True for signum in (signal.SIGTERM, signal.SIGINT, signal.SIGUSR1): signal.signal(signum, stop) from run_40b import process_lock validate_paths(args.work_dir, args.data_dir, args.weights, args.tokenizer, args.config) validate_reference_path(args) args.work_dir.mkdir(parents=True, exist_ok=True) args.reference_cache_dir.mkdir(parents=True, exist_ok=True) plan, _ = load_plan(args.config) rank = int(os.environ.get('RANK', '0')) with process_lock(args.work_dir / f'rank-{rank}.lock'), \ process_lock(args.reference_cache_dir / f'rank-{rank}.lock'), base_pipeline_lock(plan): train(args) if __name__ == '__main__': main()