Download run_40b.py from zerdovzad/WarpStateAI: direct link, hf CLI and curl.
- Browser
- Download file 51 kB
-
https://huggingface.co/zerdovzad/WarpStateAI/resolve/main/run_40b.py
- Command line
-
hf download hf://zerdovzad/WarpStateAI/run_40b.py
-
curl -L -o run_40b.py https://huggingface.co/zerdovzad/WarpStateAI/resolve/main/run_40b.py
51 kB
| #!/usr/bin/env python3 | |
| """One entry point: isolated dependencies, resumable 40B corpus, full GPU training.""" | |
| from __future__ import annotations | |
| import argparse | |
| import contextlib | |
| import hashlib | |
| import itertools | |
| import json | |
| import math | |
| import os | |
| from pathlib import Path | |
| import pickle | |
| import random | |
| import shutil | |
| import signal | |
| import subprocess | |
| import sys | |
| import time | |
| ROOT = Path(__file__).resolve().parent | |
| STOP = False | |
| RECIPE = 1 | |
| def arguments(): | |
| p = argparse.ArgumentParser(description=__doc__) | |
| initialization = p.add_mutually_exclusive_group() | |
| initialization.add_argument('--weights', type=Path, help='Your existing trusted WarpState checkpoint; used only to initialize a new run') | |
| initialization.add_argument('--from-scratch', action='store_true', help='Initialize a new model from --config and --seed; existing runs still resume') | |
| p.add_argument('--work-dir', type=Path, default=ROOT / 'work') | |
| p.add_argument('--tokenizer', type=Path, default=ROOT / 'tokenizer.model') | |
| p.add_argument('--config', type=Path, default=ROOT / 'configs/warpstate_2b.json') | |
| p.add_argument('--stage', choices=['all', 'prepare', 'train'], default='all') | |
| p.add_argument('--verify-data', action='store_true', help='Read the entire corpus and check SHA256 before training') | |
| p.add_argument('--dataset', default='HuggingFaceFW/fineweb-edu') | |
| p.add_argument('--dataset-config', default='sample-100BT') | |
| p.add_argument('--revision', default='main', help='Resolved to an immutable Hub commit on first preparation') | |
| p.add_argument('--train-tokens', type=int, default=40_000_000_000) | |
| p.add_argument('--val-tokens', type=int, default=1_048_576) | |
| p.add_argument('--workers', type=int, default=min(16, os.cpu_count() or 4)) | |
| p.add_argument('--documents-per-batch', type=int, default=256) | |
| p.add_argument('--seq-len', type=int, default=2048) | |
| p.add_argument('--micro-batch', type=int, default=8) | |
| p.add_argument('--grad-accum', type=int, default=8) | |
| p.add_argument('--lr', type=float, default=1.5e-4) | |
| p.add_argument('--min-lr', type=float, default=1.5e-5) | |
| p.add_argument('--warmup-steps', type=int, default=1000) | |
| p.add_argument('--weight-decay', type=float, default=0.1) | |
| p.add_argument('--activation-checkpointing', action=argparse.BooleanOptionalAction, default=True) | |
| p.add_argument('--autocast-cache', action=argparse.BooleanOptionalAction, default=True, | |
| help='Disable on a shared GPU to avoid retaining BF16 copies of all weights') | |
| p.add_argument('--vram-limit-gib', type=float, help='Cap the PyTorch CUDA allocator, excluding CUDA context/library memory') | |
| p.add_argument('--vram-headroom-gib', type=float, default=3.0, | |
| help='Require this much additional free device memory before applying a VRAM limit') | |
| p.add_argument('--loss-chunk-size', type=int, default=1024) | |
| p.add_argument('--loss-mode', choices=['checkpointed', 'full', 'compiled-full'], default='checkpointed', | |
| help='Full loss retains vocabulary logits to avoid head recomputation; requires additional VRAM') | |
| p.add_argument('--bf16-residuals', action=argparse.BooleanOptionalAction, default=False, | |
| help='Use BF16 residual activations while retaining FP32 parameters, gradients and optimizer state') | |
| p.add_argument('--activation-mode', choices=['baseline', 'swiglu', 'rmsnorm', 'all'], default='baseline', | |
| help='Optional BF16 activation/normalization kernels; requires explicit reconfiguration on resume') | |
| p.add_argument('--matmul-plan', type=Path, | |
| help='Explicit BF16 shape/tile recipe; missing shapes retain ATen') | |
| p.add_argument('--gradient-mode', choices=['eager', 'compiled-autograd', 'compiled-autograd-overlap'], default='eager', | |
| help='Optional compiled backward and FP32 gradient accumulation') | |
| p.add_argument('--backward-layout', choices=['baseline', 'inspect', 'paired'], default='baseline', | |
| help='Opt-in production-shape packed attention backward graph experiment') | |
| p.add_argument('--overlap-swap-collectives', action='store_true', | |
| help='Allow wait/copy groups past later collective launches, preserving launch order') | |
| p.add_argument('--overlap-runtime-estimates', action='store_true', | |
| help='Use measured, rank-aligned GEMM estimates for communication scheduling') | |
| p.add_argument('--overlap-producer-schedule', action='store_true', | |
| help='Move ready gradient producers with collectives under zero estimated memory growth') | |
| p.add_argument('--overlap-memory-budget', type=float, default=0.0, | |
| help='Allowed fractional increase in estimated scheduler peak memory (0 to 0.25)') | |
| p.add_argument('--cache-core-weight-casts', action=argparse.BooleanOptionalAction, default=False, | |
| help='Reuse differentiable BF16 core weight views across one accumulation update') | |
| p.add_argument('--ddp-bucket-cap-mb', type=float, default=128, | |
| help='Gradient communication bucket capacity in MiB') | |
| p.add_argument('--ddp-batched-grad-copy', action=argparse.BooleanOptionalAction, default=False, | |
| help='Batch FP32 gradient-to-bucket copying and scaling') | |
| p.add_argument('--compile', action='store_true', help='Optional Linux Inductor; initial compilation can take minutes') | |
| p.add_argument('--compile-core', action='store_true', help='Compile individual training blocks while preserving canonical checkpoint keys') | |
| p.add_argument('--compile-mode', choices=['default', 'reduce-overhead', 'max-autotune-no-cudagraphs'], | |
| default='default', help='Compilation mode for --compile-core or --compile') | |
| p.add_argument('--reconfigure', action='store_true', help='Allow execution changes on resume while preserving context, global batch, data and optimizer schedule') | |
| p.add_argument('--eval-every', type=int, default=500) | |
| p.add_argument('--eval-batches', type=int, default=8) | |
| p.add_argument('--eval-micro-batch', type=int, default=1, help='Validation batch size independent of training microbatch') | |
| p.add_argument('--save-every', type=int, default=1000) | |
| p.add_argument('--async-checkpoint', action=argparse.BooleanOptionalAction, default=False, | |
| help='Stage an immutable full CPU snapshot on rank zero, then overlap disk writing with training') | |
| p.add_argument('--max-steps', type=int, help='Stop and save after N additional steps, keeping the 40B schedule') | |
| p.add_argument('--stop-at-step', type=int, | |
| help='Evaluate, save and stop at this absolute optimizer step, keeping the 40B schedule') | |
| p.add_argument('--seed', type=int, default=1337) | |
| p.add_argument('--no-bootstrap', action='store_true', help='Use an already provisioned environment') | |
| p.add_argument('--test-small-gpu', action='store_true', help='Tests only: bypass the 80 GiB device requirement') | |
| a = p.parse_args() | |
| for key in ('train_tokens','val_tokens','workers','documents_per_batch','seq_len','micro_batch','grad_accum', | |
| 'loss_chunk_size','eval_every','eval_batches','eval_micro_batch','save_every','warmup_steps'): | |
| if getattr(a, key) < 1: | |
| p.error(key + ' must be positive') | |
| if a.max_steps is not None and a.max_steps < 1: | |
| p.error('--max-steps must be positive') | |
| if a.stop_at_step is not None and a.stop_at_step < 1: | |
| p.error('--stop-at-step must be positive') | |
| if a.vram_limit_gib is not None and (not math.isfinite(a.vram_limit_gib) or a.vram_limit_gib <= 0): | |
| p.error('--vram-limit-gib must be finite and positive') | |
| if not math.isfinite(a.vram_headroom_gib) or a.vram_headroom_gib < 0: | |
| p.error('--vram-headroom-gib must be finite and nonnegative') | |
| if not 0 < a.min_lr <= a.lr or a.weight_decay < 0: | |
| p.error('Invalid learning rate or weight decay') | |
| if a.train_tokens < 2 or a.val_tokens <= a.seq_len: | |
| p.error('Need at least 2 training tokens and more validation tokens than seq-len') | |
| if a.compile and a.compile_core: | |
| p.error('Choose --compile or --compile-core') | |
| if a.compile_mode != 'default' and not (a.compile or a.compile_core): | |
| p.error('A nondefault --compile-mode requires --compile-core or --compile') | |
| if not math.isfinite(a.ddp_bucket_cap_mb) or a.ddp_bucket_cap_mb * 1024**2 < 1: | |
| p.error('--ddp-bucket-cap-mb must be finite and at least one byte in MiB') | |
| if not math.isfinite(a.overlap_memory_budget) or not 0 <= a.overlap_memory_budget <= 0.25: | |
| p.error('--overlap-memory-budget must be finite and between 0 and 0.25') | |
| if a.gradient_mode != 'compiled-autograd-overlap' and ( | |
| a.overlap_swap_collectives or a.overlap_runtime_estimates or a.overlap_memory_budget | |
| or a.overlap_producer_schedule): | |
| p.error('Overlap scheduling options require --gradient-mode compiled-autograd-overlap') | |
| if a.overlap_producer_schedule and ( | |
| a.overlap_swap_collectives or a.overlap_runtime_estimates or a.overlap_memory_budget): | |
| p.error('--overlap-producer-schedule must be tested separately from other overlap scheduling options') | |
| if a.gradient_mode != 'eager' and a.ddp_batched_grad_copy: | |
| p.error('--ddp-batched-grad-copy uses the C++ reducer and cannot be combined with compiled autograd') | |
| if a.backward_layout != 'baseline': | |
| if (a.gradient_mode != 'compiled-autograd-overlap' or not a.compile_core | |
| or a.activation_checkpointing or not a.bf16_residuals | |
| or a.micro_batch != 4 or a.seq_len != 2048 | |
| or a.activation_mode != 'baseline' or a.matmul_plan is not None | |
| or a.compile_mode != 'default'): | |
| p.error('--backward-layout requires compiled-autograd-overlap, --compile-core, ' | |
| '--no-activation-checkpointing, --bf16-residuals, microbatch 4, sequence 2048, ' | |
| 'default compile mode, baseline activations and no matmul plan') | |
| if a.gradient_mode != 'eager' and a.activation_mode != 'baseline': | |
| p.error('Standalone activation modes require eager gradients; use baseline activations ' | |
| 'for compiled autograd, including matmul-plan epilogues') | |
| if a.cache_core_weight_casts and a.activation_checkpointing: | |
| p.error('--cache-core-weight-casts requires --no-activation-checkpointing') | |
| if a.cache_core_weight_casts and a.gradient_mode != 'eager': | |
| p.error('Core weight cast caching and compiled autograd must be tested separately') | |
| if a.compile_mode == 'reduce-overhead' and (a.cache_core_weight_casts or a.gradient_mode != 'eager'): | |
| p.error('Weight cast caching and compiled autograd require a compile mode without CUDA graphs') | |
| if a.matmul_plan is not None and a.cache_core_weight_casts: | |
| p.error('--matmul-plan cannot be combined with external core weight cast caching') | |
| if a.matmul_plan is not None and a.compile_mode == 'reduce-overhead': | |
| p.error('--matmul-plan currently requires a compile mode without CUDA graphs') | |
| if a.matmul_plan is not None and a.gradient_mode != 'eager' and a.activation_checkpointing: | |
| p.error('--matmul-plan with compiled autograd requires --no-activation-checkpointing') | |
| a.rank = int(os.environ.get('RANK', '0')) | |
| a.world_size = int(os.environ.get('WORLD_SIZE', '1')) | |
| a.local_rank = int(os.environ.get('LOCAL_RANK', '0')) | |
| if a.world_size > 1 and a.stage != 'train': | |
| p.error('torchrun supports --stage train only; prepare the corpus once beforehand') | |
| for name in ('work_dir','tokenizer','config','weights','matmul_plan'): | |
| if getattr(a,name) is not None: | |
| setattr(a,name,getattr(a,name).expanduser().resolve()) | |
| return a | |
| def bootstrap(a): | |
| if a.no_bootstrap: | |
| return | |
| if sys.platform != 'linux': | |
| raise RuntimeError('Automatic installation targets Linux. Tests may use --no-bootstrap.') | |
| if not (3, 12) <= sys.version_info[:2] <= (3, 14): | |
| raise RuntimeError('Use Python 3.12–3.14; Python 3.12 is recommended.') | |
| env = ROOT / '.server-venv' | |
| python = env / 'bin/python' | |
| if not python.exists(): | |
| subprocess.run([sys.executable, '-m', 'venv', str(env)], check=True) | |
| marker = env / 'warpstate-deps-v1.json' | |
| if not marker.exists(): | |
| subprocess.run([str(python), '-m', 'pip', 'install', '--upgrade', 'pip'], check=True) | |
| subprocess.run([str(python), '-m', 'pip', 'install', 'torch==2.14.0', | |
| '--index-url', 'https://download.pytorch.org/whl/cu130'], check=True) | |
| subprocess.run([str(python), '-m', 'pip', 'install', 'numpy==2.5.3', 'sentencepiece==0.2.2', | |
| 'datasets==5.0.1', 'huggingface-hub==1.30.0'], check=True) | |
| subprocess.run([str(python), '-m', 'pip', 'check'], check=True) | |
| freeze = subprocess.check_output([str(python), '-m', 'pip', 'freeze'], text=True) | |
| (env / 'requirements-resolved.txt').write_text(freeze) | |
| marker.write_text(json.dumps({'installed': time.time()})) | |
| os.execv(str(python), [str(python), '-u', str(Path(__file__).resolve()), *sys.argv[1:], '--no-bootstrap']) | |
| def digest(path): | |
| h = hashlib.sha256() | |
| with Path(path).open('rb') as f: | |
| for block in iter(lambda: f.read(1024*1024), b''): | |
| h.update(block) | |
| return h.hexdigest() | |
| def atomic(path, writer): | |
| path = Path(path) | |
| temporary = path.with_suffix(path.suffix + '.tmp') | |
| with temporary.open('wb') as f: | |
| writer(f) | |
| f.flush() | |
| os.fsync(f.fileno()) | |
| os.replace(temporary, path) | |
| def process_lock(path): | |
| with Path(path).open('a+b') as f: | |
| if os.name == 'nt': | |
| import msvcrt | |
| f.seek(0) | |
| f.write(b'0') | |
| f.flush() | |
| f.seek(0) | |
| msvcrt.locking(f.fileno(), msvcrt.LK_NBLCK, 1) | |
| else: | |
| import fcntl | |
| fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB) | |
| yield | |
| def data_signature(a): | |
| return {'recipe': RECIPE, 'dataset': a.dataset, 'config': a.dataset_config, 'requested_revision': a.revision, | |
| 'train_tokens': a.train_tokens, 'val_tokens': a.val_tokens, 'tokenizer_sha256': digest(a.tokenizer), | |
| 'dtype': '<u2', 'holdout': 'sha256(raw-text) modulo 1000 == 0; train excludes all holdout documents'} | |
| def is_holdout(text): | |
| return int.from_bytes(hashlib.sha256(text.encode('utf-8')).digest()[:8], 'little') % 1000 == 0 | |
| def check_manifest(a): | |
| directory = a.work_dir / 'data' | |
| manifest = json.loads((directory / 'manifest.json').read_text()) | |
| if manifest['signature'] != data_signature(a) or not manifest['complete']: | |
| raise ValueError('Prepared dataset does not match requested settings/tokenizer') | |
| for split, count in [('train', a.train_tokens), ('val', a.val_tokens)]: | |
| if (directory / (split + '.bin')).stat().st_size != count * 2: | |
| raise ValueError('Invalid token file size: ' + split) | |
| if getattr(a, 'verify_data', False) and digest(directory / (split+'.bin')) != manifest['sha256'][split]: | |
| raise ValueError('Dataset checksum mismatch: '+split) | |
| return manifest | |
| def prepare(a): | |
| import numpy as np | |
| import sentencepiece as spm | |
| import datasets | |
| from huggingface_hub import HfApi | |
| directory = a.work_dir / 'data' | |
| directory.mkdir(parents=True, exist_ok=True) | |
| if (directory / 'manifest.json').exists(): | |
| check_manifest(a) | |
| print('Dataset already complete and matches this tokenizer.', flush=True) | |
| return True | |
| signature = data_signature(a) | |
| sp = spm.SentencePieceProcessor(model_file=str(a.tokenizer)) | |
| if sp.vocab_size() > 65536 or sp.eos_id() < 0: | |
| raise ValueError('Need a uint16-compatible tokenizer with EOS') | |
| progress = directory / 'prepare-state.pkl' | |
| if progress.exists(): | |
| with progress.open('rb') as f: | |
| state = pickle.load(f) | |
| if state['signature'] != signature or state['datasets_version'] != datasets.__version__: | |
| raise ValueError('Preparation settings/version changed; use the original environment or a different work directory') | |
| else: | |
| if any((directory / (s+'.bin')).exists() for s in ('train','val')): | |
| raise RuntimeError('Token files without preparation state: refusing to overwrite') | |
| state = {'signature': signature, 'datasets_version': datasets.__version__, 'train': 0, 'val': 0, | |
| 'revision': HfApi().dataset_info(a.dataset, revision=a.revision).sha, 'iterator': None, 'documents': 0} | |
| atomic(progress, lambda f: pickle.dump(state, f)) | |
| remaining_bytes = 2*(a.train_tokens+a.val_tokens-state['train']-state['val']) | |
| reserve = 70*1024**3 if not a.test_small_gpu else 0 | |
| if shutil.disk_usage(directory).free < remaining_bytes + reserve: | |
| raise RuntimeError(f'Need {(remaining_bytes+reserve)/1024**3:.1f} GiB free for remaining data and checkpoints') | |
| ds = datasets.load_dataset(a.dataset, name=a.dataset_config, split='train', streaming=True, revision=state['revision']) | |
| print(f"Preparing {a.dataset}/{a.dataset_config} at {state['revision']}; " | |
| f"resuming train={state['train']:,}, val={state['val']:,}", flush=True) | |
| if state['iterator'] is not None: | |
| ds.load_state_dict(state['iterator']) | |
| iterator = iter(ds) | |
| with contextlib.ExitStack() as stack: | |
| stack.callback(lambda: getattr(iterator, 'close', lambda: None)()) | |
| files = {} | |
| for split in ('train', 'val'): | |
| path = directory / (split+'.bin') | |
| if not path.exists() and state[split]: | |
| raise RuntimeError('Missing committed data: '+split) | |
| f = stack.enter_context(path.open('r+b' if path.exists() else 'w+b')) | |
| if path.stat().st_size < state[split]*2: | |
| raise RuntimeError('Truncated committed data: '+split) | |
| f.truncate(state[split]*2) # Discard only uncommitted output after interruption. | |
| f.seek(state[split]*2) | |
| files[split] = f | |
| def commit(): | |
| for f in files.values(): | |
| f.flush() | |
| os.fsync(f.fileno()) | |
| state['iterator'] = ds.state_dict() | |
| atomic(progress, lambda f: pickle.dump(state, f, protocol=pickle.HIGHEST_PROTOCOL)) | |
| last_commit = state['train']+state['val'] | |
| started = time.monotonic() | |
| start_count = last_commit | |
| while state['train'] < a.train_tokens or state['val'] < a.val_tokens: | |
| rows = list(itertools.islice(iterator, a.documents_per_batch)) | |
| if not rows: | |
| commit() | |
| raise RuntimeError('Source exhausted before the requested token budget; do not repeat it silently') | |
| texts = [row['text'] for row in rows] | |
| encoded = sp.encode(texts, out_type=int, num_threads=a.workers) | |
| for text, ids in zip(texts, encoded): | |
| state['documents'] += 1 | |
| if not text.strip(): | |
| continue | |
| split = 'val' if is_holdout(text) else 'train' | |
| target = a.val_tokens if split == 'val' else a.train_tokens | |
| room = target - state[split] | |
| if room <= 0: | |
| continue | |
| tokens = np.asarray((ids+[sp.eos_id()])[:room], dtype='<u2') | |
| files[split].write(tokens.tobytes()) | |
| state[split] += len(tokens) | |
| count = state['train']+state['val'] | |
| if STOP or count-last_commit >= 8_000_000: | |
| commit() | |
| last_commit = count | |
| print(f"prepared train={state['train']:,}/{a.train_tokens:,}, val={state['val']:,}; " | |
| f"{(count-start_count)/max(time.monotonic()-started,1):,.0f} tokens/s", flush=True) | |
| if STOP: | |
| return False | |
| commit() | |
| manifest = {'signature': signature, 'source_revision': state['revision'], 'complete': True, | |
| 'documents_read': state['documents'], 'datasets_version': datasets.__version__, | |
| 'sha256': {s:digest(directory/(s+'.bin')) for s in ('train','val')}, | |
| 'source_url': 'https://huggingface.co/datasets/'+a.dataset} | |
| atomic(directory / 'manifest.json', lambda f: f.write(json.dumps(manifest, indent=2).encode())) | |
| check_manifest(a) | |
| print('Prepared the exact requested token counts.', flush=True) | |
| return True | |
| def batches(data, cursor, batch, length, valid, device): | |
| import numpy as np | |
| import torch | |
| # A cyclic contiguous pass: no random sampling with replacement over 40B tokens. | |
| indexes = (np.arange(batch*length+1, dtype=np.int64)+cursor) % len(data) | |
| tokens = np.asarray(data[indexes], dtype=np.int64) | |
| x = torch.from_numpy(tokens[:-1].reshape(batch, length).copy()) | |
| y = torch.from_numpy(tokens[1:].reshape(batch, length).copy()) | |
| y.view(-1)[valid:] = -100 | |
| if device.type == 'cuda': | |
| x, y = x.pin_memory(), y.pin_memory() | |
| return x.to(device, non_blocking=True), y.to(device, non_blocking=True) | |
| def train_signature(a, manifest): | |
| keys = ('seq_len','micro_batch','grad_accum','train_tokens','lr','min_lr','warmup_steps','weight_decay', | |
| 'seed','activation_checkpointing','loss_chunk_size','compile') | |
| settings = {k: getattr(a,k) for k in keys} | |
| settings.update(world_size=getattr(a, 'world_size', 1), compile_core=getattr(a, 'compile_core', False), | |
| loss_mode=getattr(a, 'loss_mode', 'checkpointed'), | |
| compile_mode=getattr(a, 'compile_mode', 'default'), | |
| bf16_residuals=getattr(a, 'bf16_residuals', False), | |
| activation_mode=getattr(a, 'activation_mode', 'baseline'), | |
| gradient_mode=getattr(a, 'gradient_mode', 'eager'), | |
| overlap_swap_collectives=getattr(a, 'overlap_swap_collectives', False), | |
| overlap_runtime_estimates=getattr(a, 'overlap_runtime_estimates', False), | |
| overlap_memory_budget=getattr(a, 'overlap_memory_budget', 0.0), | |
| overlap_producer_schedule=getattr(a, 'overlap_producer_schedule', False), | |
| matmul_execution=getattr(a, 'matmul_metadata', None), | |
| backward_layout_execution=getattr(a, 'backward_layout_metadata', None), | |
| cache_core_weight_casts=getattr(a, 'cache_core_weight_casts', False), | |
| ddp_bucket_cap_mb=getattr(a, 'ddp_bucket_cap_mb', 128), | |
| ddp_batched_grad_copy=getattr(a, 'ddp_batched_grad_copy', False)) | |
| return {'settings': settings, 'data': manifest} | |
| def check_resume_signature(old, new, reconfigure=False): | |
| """Execution can change explicitly; the objective and global schedule cannot.""" | |
| def normalized(signature): | |
| settings = dict(signature['settings']) | |
| settings.setdefault('world_size', 1) | |
| settings.setdefault('compile_core', False) | |
| settings.setdefault('loss_mode', 'checkpointed') | |
| settings.setdefault('compile_mode', 'default') | |
| settings.setdefault('bf16_residuals', False) | |
| settings.setdefault('activation_mode', 'baseline') | |
| settings.setdefault('gradient_mode', 'eager') | |
| settings.setdefault('overlap_swap_collectives', False) | |
| settings.setdefault('overlap_runtime_estimates', False) | |
| settings.setdefault('overlap_memory_budget', 0.0) | |
| settings.setdefault('overlap_producer_schedule', False) | |
| settings.setdefault('matmul_execution', None) | |
| settings.setdefault('backward_layout_execution', None) | |
| settings.setdefault('cache_core_weight_casts', False) | |
| settings.setdefault('ddp_bucket_cap_mb', 128) | |
| settings.setdefault('ddp_batched_grad_copy', False) | |
| return {'settings': settings, 'data': signature['data']} | |
| old, new = normalized(old), normalized(new) | |
| if old == new: | |
| return False | |
| message = 'Resume settings/data differ; keep original settings or use --reconfigure for execution-only changes' | |
| if not reconfigure or old['data'] != new['data']: | |
| raise ValueError(message) | |
| execution = {'micro_batch', 'grad_accum', 'world_size', 'activation_checkpointing', | |
| 'loss_chunk_size', 'loss_mode', 'compile', 'compile_core', 'compile_mode', | |
| 'bf16_residuals', 'activation_mode', 'gradient_mode', 'cache_core_weight_casts', 'matmul_execution', | |
| 'overlap_swap_collectives', 'overlap_runtime_estimates', 'overlap_memory_budget', | |
| 'overlap_producer_schedule', | |
| 'ddp_bucket_cap_mb', 'ddp_batched_grad_copy', 'backward_layout_execution'} | |
| objective = lambda s: {k: v for k, v in s.items() if k not in execution} | |
| batch = lambda s: s['seq_len']*s['micro_batch']*s['grad_accum']*s['world_size'] | |
| if objective(old['settings']) != objective(new['settings']) or batch(old['settings']) != batch(new['settings']): | |
| raise ValueError(message + '; objective and global tokens per update must stay fixed') | |
| return True | |
| def update_slices(target, micro_tokens, world_size, rank): | |
| """Disjoint contiguous slices; empty ranks still participate in the final backward.""" | |
| for iteration in range(math.ceil(target / (world_size*micro_tokens))): | |
| offset = (iteration*world_size + rank)*micro_tokens | |
| yield offset, max(0, min(micro_tokens, target-offset)) | |
| def compile_training_cores(model, mode='default'): | |
| import torch | |
| for core in model.cores: | |
| eager = core.forward | |
| compiled = torch.compile(eager, fullgraph=True, dynamic=False, mode=mode) | |
| def forward(*args, _core=core, _eager=eager, _compiled=compiled, **kwargs): | |
| # Validation does not need another compiled graph for a different batch size. | |
| fn = _compiled if _core.training and torch.is_grad_enabled() else _eager | |
| return fn(*args, **kwargs) | |
| core.forward = forward | |
| def configure_memory(a): | |
| """Apply the cap before model allocation; this does not reserve another job's memory.""" | |
| if a.vram_limit_gib is None: | |
| return | |
| import torch | |
| device = torch.device('cuda', getattr(a, 'local_rank', 0)) | |
| free, total = torch.cuda.mem_get_info(device) | |
| limit = int(a.vram_limit_gib * 1024**3) | |
| required = limit + int(a.vram_headroom_gib * 1024**3) | |
| if limit > total or free < required: | |
| raise RuntimeError(f'Only {free/1024**3:.2f} GiB free; need {required/1024**3:.2f} GiB ' | |
| 'for the requested allocator cap and headroom') | |
| torch.cuda.set_per_process_memory_fraction(limit / total, device) | |
| print(f'CUDA allocator cap {a.vram_limit_gib:.2f} GiB; free before model ' | |
| f'{free/1024**3:.2f} GiB; autocast cache={a.autocast_cache}', flush=True) | |
| def train(a): | |
| import numpy as np | |
| import sentencepiece as spm | |
| import torch | |
| import torch.distributed as dist | |
| from warpstate import WarpStateConfig, WarpStateLM | |
| manifest = check_manifest(a) | |
| matmul_plan = None | |
| if a.matmul_plan is not None: | |
| from ops.matmul_execution import load_matmul_plan | |
| matmul_plan = load_matmul_plan(a.matmul_plan) | |
| a.matmul_metadata = matmul_plan.metadata if matmul_plan else None | |
| a.backward_layout_metadata = None | |
| if a.backward_layout != 'baseline': | |
| from ops.backward_layout_execution import configure_backward_layout | |
| a.backward_layout_metadata = configure_backward_layout(a.backward_layout) | |
| if not torch.cuda.is_available() or not torch.cuda.is_bf16_supported(): | |
| raise RuntimeError('BF16 CUDA GPU required') | |
| device = torch.device('cuda', a.local_rank) | |
| distributed = a.world_size > 1 | |
| props = torch.cuda.get_device_properties(device) | |
| if not a.test_small_gpu and props.total_memory < 80*1024**3: | |
| raise RuntimeError('This preset requires the 96 GB RTX PRO 6000; use --test-small-gpu only for tiny tests') | |
| torch.set_num_threads(min(a.workers, 8)) | |
| torch.set_float32_matmul_precision('high') | |
| torch.backends.cuda.matmul.allow_tf32 = True | |
| random.seed(a.seed) | |
| np.random.seed(a.seed) | |
| torch.manual_seed(a.seed) | |
| torch.cuda.manual_seed(a.seed + a.rank) | |
| run = a.work_dir / 'run' | |
| run.mkdir(exist_ok=True) | |
| latest = run / 'latest.pt' | |
| resuming = latest.exists() | |
| source = latest if resuming else a.weights | |
| if not resuming and a.from_scratch: | |
| checkpoint = {} | |
| elif source is None or not source.is_file(): | |
| raise ValueError('First launch requires --weights /path/to/your/checkpoint.pt') | |
| else: | |
| checkpoint = torch.load(source, map_location='cpu', mmap=True, weights_only=False) | |
| cfg = WarpStateConfig(**checkpoint.get('config', json.loads(a.config.read_text()))) | |
| cfg.validate() | |
| if a.backward_layout != 'baseline': | |
| from ops.backward_layout_execution import validate_backward_layout_config | |
| validate_backward_layout_config(cfg) | |
| if a.seq_len % cfg.chunk_size: | |
| raise ValueError('seq-len must be divisible by model chunk_size') | |
| sp = spm.SentencePieceProcessor(model_file=str(a.tokenizer)) | |
| if cfg.vocab_size != sp.vocab_size(): | |
| raise ValueError('Tokenizer/model vocabulary mismatch; use the exact original tokenizer') | |
| signature = train_signature(a, manifest) | |
| changed = False | |
| if resuming: | |
| if checkpoint.get('format') != 'warpstate-server-v1': | |
| raise ValueError('Resume settings/data differ; unsupported checkpoint format') | |
| changed = check_resume_signature(checkpoint['signature'], signature, a.reconfigure) | |
| if not resuming and a.from_scratch: | |
| print(f'Initializing fresh FP32 weights on CPU with seed {a.seed}.', flush=True) | |
| model = WarpStateLM(cfg) | |
| else: | |
| with torch.device('meta'): | |
| model = WarpStateLM(cfg) | |
| model.load_state_dict(checkpoint['model'], assign=True) | |
| model.to(device=device, dtype=torch.float32).requires_grad_(True) | |
| # Prefer unrounded FP32 masters when importing the laptop's CPU-offload checkpoint. | |
| old_opt = checkpoint.get('optimizer', {}) | |
| if not resuming and old_opt.get('kind') == 'cpu_offload_adamw': | |
| with torch.no_grad(): | |
| for parameter, master in zip(model.parameters(), old_opt['masters'], strict=True): | |
| if parameter.shape != master.shape: | |
| raise ValueError('Master weight shape mismatch') | |
| parameter.copy_(master) | |
| cfg.gradient_checkpointing = a.activation_checkpointing | |
| model.loss_chunk_size = a.loss_chunk_size | |
| optimizer = torch.optim.AdamW(model.parameters(), lr=a.lr, betas=(0.9,0.95), eps=1e-8, | |
| weight_decay=a.weight_decay, fused=True) | |
| seen, step = 0, 0 | |
| if resuming: | |
| optimizer.load_state_dict(checkpoint['optimizer']) | |
| seen, step = checkpoint['tokens_seen'], checkpoint['step'] | |
| rank_states = checkpoint.get('rng_by_rank') | |
| if rank_states is None: | |
| rank_states = [{'torch': checkpoint['rng_torch'], 'cuda': checkpoint['rng_cuda'][0], | |
| 'numpy': checkpoint['rng_numpy'], 'python': checkpoint['rng_python']}] | |
| if a.rank < len(rank_states): | |
| state = rank_states[a.rank] | |
| torch.set_rng_state(state['torch']) | |
| torch.cuda.set_rng_state(state['cuda'], device) | |
| np.random.set_state(state['numpy']) | |
| random.setstate(state['python']) | |
| else: | |
| # A newly added rank gets a reproducible independent stream. | |
| random.seed(a.seed + a.rank) | |
| np.random.seed(a.seed + a.rank) | |
| torch.manual_seed(a.seed + a.rank) | |
| torch.cuda.manual_seed(a.seed + a.rank) | |
| origin = checkpoint.get('origin', | |
| {'initialization': 'random', 'seed': a.seed, 'previous_training_tokens': 0} | |
| if not resuming and a.from_scratch else | |
| {'path': str(source), 'previous_training_tokens': checkpoint.get('tokens_seen')}) | |
| history = list(checkpoint.get('execution_history', [])) | |
| if changed: | |
| history.append({'step': step, 'tokens': seen, 'timestamp': time.time(), | |
| 'from': checkpoint['signature']['settings'], 'to': signature['settings']}) | |
| if a.rank == 0: | |
| print(f'Reconfiguring execution at step {step}, tokens={seen:,}; preserving model, optimizer and schedule.', flush=True) | |
| del old_opt, checkpoint | |
| count = model.num_parameters() | |
| if shutil.disk_usage(run).free < count*28: | |
| raise RuntimeError('Need free space for checkpoint plus atomic replacement (about 53 GiB for 2B)') | |
| raw = model | |
| from ops.gradient_execution import OverlapSettings, make_backward, scale_loss | |
| from ops.distributed_execution import ddp_kwargs, ddp_runtime_settings | |
| overlap_settings = (OverlapSettings(swap_with_collectives=a.overlap_swap_collectives, | |
| use_runtime_estimations=a.overlap_runtime_estimates, | |
| producer_schedule=a.overlap_producer_schedule, | |
| peak_memory_budget=a.overlap_memory_budget) | |
| if a.gradient_mode == 'compiled-autograd-overlap' else None) | |
| backward = make_backward(a.gradient_mode, distributed=distributed, overlap_settings=overlap_settings) | |
| ddp_settings = ddp_runtime_settings(bucket_cap_mb=a.ddp_bucket_cap_mb, | |
| batched_grad_copy=a.ddp_batched_grad_copy) | |
| if a.activation_mode != 'baseline': | |
| from ops.activation_execution import apply_activation_execution | |
| apply_activation_execution(raw, a.activation_mode) | |
| if matmul_plan is not None: | |
| from ops.matmul_execution import apply_matmul_execution | |
| apply_matmul_execution(raw, matmul_plan) | |
| if a.compile_core: | |
| compile_training_cores(raw, mode=a.compile_mode) | |
| if a.loss_mode != 'checkpointed': | |
| from ops.loss_experiment import apply_loss_execution | |
| apply_loss_execution(raw, a.loss_mode) | |
| if a.bf16_residuals: | |
| from ops.residual_experiment import apply_bf16_residuals | |
| apply_bf16_residuals(raw) | |
| weight_cast_cache = None | |
| if a.cache_core_weight_casts: | |
| from ops.weight_cast_execution import apply_weight_cast_cache | |
| weight_cast_cache = apply_weight_cast_cache(raw) | |
| if a.compile: | |
| model = torch.compile(model, mode=a.compile_mode) # Optional: GPU-only optimizer, no CPU offload hooks. | |
| if distributed: | |
| model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[a.local_rank], | |
| output_device=a.local_rank, **ddp_kwargs(ddp_settings)) | |
| graph_gradients = (a.compile or a.compile_core) and a.compile_mode == 'reduce-overhead' | |
| if graph_gradients: | |
| # Replay must accumulate into stable buffers outside the graph's private pool. | |
| for parameter in model.parameters(): | |
| if parameter.requires_grad: | |
| parameter.grad = torch.zeros_like(parameter) | |
| train_data = np.memmap(a.work_dir/'data/train.bin', dtype='<u2', mode='r') | |
| val_data = np.memmap(a.work_dir/'data/val.bin', dtype='<u2', mode='r') | |
| if a.rank == 0: | |
| print(f'{a.world_size} x {props.name}: {props.total_memory/1024**3:.1f} GiB/device; {count:,} parameters; ' | |
| f'FP32 GPU AdamW + BF16 autocast; start tokens={seen:,}', flush=True) | |
| batch_tokens = a.micro_batch*a.seq_len | |
| update_tokens = batch_tokens*a.grad_accum*a.world_size | |
| total_steps = math.ceil(a.train_tokens/update_tokens) | |
| end_step = total_steps | |
| if a.max_steps is not None: | |
| end_step = min(end_step, step + a.max_steps) | |
| if a.stop_at_step is not None: | |
| end_step = min(end_step, a.stop_at_step) | |
| if a.rank == 0: | |
| if end_step < total_steps: | |
| print(f'Pause boundary: optimizer step {end_step}; loaded step {step}; ' | |
| f'learning-rate schedule remains {total_steps} steps.', flush=True) | |
| if step >= end_step: | |
| print('Pause boundary already reached; no further optimizer updates.', flush=True) | |
| (run/'settings.json').write_text(json.dumps({'signature':signature, 'gpu':props.name, | |
| 'torch':torch.__version__, 'cuda':torch.version.cuda, 'parameters':count, 'origin':origin, | |
| 'execution_history': history, | |
| 'runtime': {'stop_at_step': a.stop_at_step, 'max_steps': a.max_steps, 'end_step': end_step, | |
| 'vram_limit_gib': a.vram_limit_gib, 'vram_headroom_gib': a.vram_headroom_gib, | |
| 'autocast_cache': a.autocast_cache, 'eval_micro_batch': a.eval_micro_batch, | |
| 'eval_every': a.eval_every, 'save_every': a.save_every, | |
| 'async_checkpoint': a.async_checkpoint, | |
| 'bf16_residuals': a.bf16_residuals, | |
| 'activation_mode': a.activation_mode, 'gradient_mode': a.gradient_mode, | |
| 'overlap_swap_collectives': a.overlap_swap_collectives, | |
| 'overlap_runtime_estimates': a.overlap_runtime_estimates, | |
| 'overlap_memory_budget': a.overlap_memory_budget, | |
| 'overlap_producer_schedule': a.overlap_producer_schedule, | |
| 'matmul_execution': a.matmul_metadata, | |
| 'backward_layout_execution': a.backward_layout_metadata, | |
| 'cache_core_weight_casts': a.cache_core_weight_casts, | |
| **ddp_settings, | |
| 'blas_preference': str(torch.backends.cuda.preferred_blas_library()), | |
| 'cublaslt_workspace_size_env': os.getenv('CUBLASLT_WORKSPACE_SIZE'), | |
| 'cuda_visible_devices': os.getenv('CUDA_VISIBLE_DEVICES')}}, indent=2)) | |
| def evaluate(): | |
| model.eval() | |
| values = [] | |
| if a.rank == 0: | |
| validation_tokens = a.eval_micro_batch*a.seq_len | |
| with torch.no_grad(): | |
| for cursor in np.linspace(0, max(0,len(val_data)-validation_tokens-1), a.eval_batches, dtype=np.int64): | |
| x,y = batches(val_data,int(cursor),a.eval_micro_batch,a.seq_len,validation_tokens,device) | |
| with torch.autocast('cuda', dtype=torch.bfloat16, cache_enabled=a.autocast_cache): | |
| _, loss = raw(x,y,return_logits=False) | |
| values.append(float(loss)) | |
| model.train() | |
| value = sum(values)/len(values) if values else 0.0 | |
| if distributed: | |
| tensor = torch.tensor(value, dtype=torch.float64, device=device) | |
| dist.broadcast(tensor, src=0) | |
| value = float(tensor) | |
| if not math.isfinite(value): | |
| raise RuntimeError('Non-finite validation loss') | |
| return value | |
| def save(validation): | |
| state = {'torch': torch.get_rng_state(), 'cuda': torch.cuda.get_rng_state(device), | |
| 'numpy': np.random.get_state(), 'python': random.getstate()} | |
| rank_states = [state] | |
| if distributed: | |
| rank_states = [None]*a.world_size | |
| dist.all_gather_object(rank_states, state) | |
| def build_state(): | |
| print('Staging full CPU checkpoint...' if a.async_checkpoint else | |
| 'Saving full GPU-optimizer checkpoint...', flush=True) | |
| return {'format':'warpstate-server-v1', 'signature':signature, 'origin':origin, | |
| 'execution_history': history, | |
| 'config':vars(cfg), 'model':raw.state_dict(), 'optimizer':optimizer.state_dict(), | |
| 'step':step, 'tokens_seen':seen, 'val_loss':validation, | |
| 'val_step':last_val_step, 'val_tokens':last_val_tokens, 'rng_by_rank': rank_states, | |
| 'rng_torch':state['torch'], 'rng_cuda':[state['cuda']], | |
| 'rng_numpy':state['numpy'], 'rng_python':state['python']} | |
| if checkpoint_writer is not None: | |
| # Every rank waits for the frozen CPU snapshot or a coordinated error. | |
| # Only rank zero writes; subsequent updates overlap its CPU thread. | |
| staging_seconds = checkpoint_writer.submit(state_factory=build_state) | |
| if a.rank == 0: | |
| print(f'Staged step {step}, tokens={seen:,} for asynchronous checkpoint; ' | |
| f'synchronous CPU staging {staging_seconds:.2f}s.', flush=True) | |
| else: | |
| if a.rank == 0: | |
| obj = build_state() | |
| atomic(latest, lambda f: torch.save(obj,f)) | |
| print('Saved '+str(latest), flush=True) | |
| if distributed: | |
| dist.barrier() | |
| def stop_requested(): | |
| stop_file = os.environ.get('WARPSTATE_STOP_FILE') | |
| requested = STOP or bool(stop_file and Path(stop_file).is_file()) | |
| if not distributed: | |
| return requested | |
| flag = torch.tensor(int(requested), device=device) | |
| dist.all_reduce(flag, op=dist.ReduceOp.MAX) | |
| return bool(flag.item()) | |
| checkpoint_writer = None | |
| if a.async_checkpoint: | |
| from ops.checkpoint_writer import DistributedCheckpointWriter | |
| checkpoint_writer = DistributedCheckpointWriter(latest, rank=a.rank, | |
| device=device, distributed=distributed) | |
| with checkpoint_writer if checkpoint_writer is not None else contextlib.nullcontext(): | |
| last_val = evaluate() | |
| last_val_step, last_val_tokens = step, seen | |
| if a.rank == 0: | |
| print(f'initial validation loss: {last_val:.5f}', flush=True) | |
| while seen < a.train_tokens and step < end_step: | |
| if checkpoint_writer is not None: | |
| checkpoint_writer.check() | |
| if stop_requested(): | |
| save(last_val) | |
| break | |
| torch.cuda.synchronize() | |
| begin = time.perf_counter() | |
| optimizer.zero_grad(set_to_none=not graph_gradients) | |
| target = min(update_tokens,a.train_tokens-seen) | |
| warm = min(a.warmup_steps,total_steps) | |
| rate = (a.lr*(step+1)/warm if step < warm else a.min_lr+(a.lr-a.min_lr)*0.5* | |
| (1+math.cos(math.pi*(step-warm)/max(1,total_steps-warm)))) | |
| for group in optimizer.param_groups: | |
| group['lr'] = rate | |
| loss_sum = torch.zeros((),device=device) | |
| slices = list(update_slices(target, batch_tokens, a.world_size, a.rank)) | |
| if a.backward_layout == 'paired': | |
| from ops.backward_layout_execution import backward_layout_stats, require_backward_layout_dispatch | |
| layout_calls_before = backward_layout_stats()['runtime_calls'] | |
| with weight_cast_cache.update() if weight_cast_cache else contextlib.nullcontext(): | |
| for index, (offset, valid) in enumerate(slices): | |
| x,y = batches(train_data,seen+offset,a.micro_batch,a.seq_len,valid,device) | |
| sync = model.no_sync() if distributed and index < len(slices)-1 else contextlib.nullcontext() | |
| with sync: | |
| with torch.autocast('cuda',dtype=torch.bfloat16,cache_enabled=a.autocast_cache): | |
| _, loss = model(x,y,return_logits=False) | |
| scaled = scale_loss(loss, valid*a.world_size/target, a.gradient_mode) | |
| backward(scaled) | |
| loss_sum += loss.detach()*(valid/target) | |
| if a.backward_layout == 'paired': | |
| require_backward_layout_dispatch(min_calls=layout_calls_before + cfg.logical_depth * len(slices)) | |
| norm = torch.nn.utils.clip_grad_norm_(model.parameters(),1.0,error_if_nonfinite=True) | |
| optimizer.step() | |
| if distributed: | |
| dist.all_reduce(loss_sum, op=dist.ReduceOp.SUM) | |
| torch.cuda.synchronize() | |
| elapsed = time.perf_counter()-begin | |
| gpu_stats = [{'allocated': torch.cuda.max_memory_allocated(device)/1024**3, | |
| 'reserved': torch.cuda.max_memory_reserved(device)/1024**3, | |
| 'free': torch.cuda.mem_get_info(device)[0]/1024**3, 'seconds': elapsed}] | |
| if distributed: | |
| local_stats = gpu_stats[0] | |
| gpu_stats = [None]*a.world_size | |
| dist.all_gather_object(gpu_stats, local_stats) | |
| elapsed = max(item['seconds'] for item in gpu_stats) | |
| seen += target | |
| step += 1 | |
| record = {'step':step,'tokens':seen,'loss':float(loss_sum),'lr':rate,'grad_norm':float(norm), | |
| 'tokens_per_second':target/elapsed,'seconds':elapsed,'timestamp':time.time(), | |
| 'world_size': a.world_size, 'gpu_memory': gpu_stats, | |
| 'peak_vram_gib':max(item['allocated'] for item in gpu_stats), | |
| 'peak_reserved_vram_gib':max(item['reserved'] for item in gpu_stats), | |
| 'device_free_vram_gib':min(item['free'] for item in gpu_stats)} | |
| torch.cuda.reset_peak_memory_stats() | |
| ending = stop_requested() or seen == a.train_tokens or step >= end_step | |
| if step % a.eval_every == 0 or ending: | |
| last_val = evaluate() | |
| last_val_step, last_val_tokens = step, seen | |
| if a.rank == 0: | |
| with (run/'validation.jsonl').open('a') as f: | |
| f.write(json.dumps({'step':step,'tokens':seen,'loss':last_val})+'\n') | |
| record.update(val_loss=last_val, val_step=last_val_step, val_tokens=last_val_tokens) | |
| if a.backward_layout != 'baseline': | |
| from ops.backward_layout_execution import backward_layout_stats | |
| record['backward_layout'] = backward_layout_stats() | |
| if a.rank == 0: | |
| validation_log = f'val {last_val:.5f} (step {last_val_step}) | ' if last_val_step == step else '' | |
| print(f"step {step}/{total_steps} | {seen:,}/{a.train_tokens:,} tokens | loss {record['loss']:.4f} | " | |
| f"{validation_log}" | |
| f"{record['tokens_per_second']:,.0f} tok/s | peak {record['peak_vram_gib']:.2f} GiB allocated / " | |
| f"{record['peak_reserved_vram_gib']:.2f} GiB reserved | free {record['device_free_vram_gib']:.2f} GiB",flush=True) | |
| with (run/'metrics.jsonl').open('a') as f: | |
| f.write(json.dumps(record)+'\n') | |
| if step % a.save_every == 0 or ending: | |
| save(last_val) | |
| if ending: | |
| break | |
| def request_stop(signum, frame): | |
| global STOP | |
| STOP = True | |
| print('Stop requested; committing preparation or finishing the current optimizer update.', flush=True) | |
| def preflight(a): | |
| """Reject incompatible weights/hardware before a multi-day corpus preparation.""" | |
| import torch | |
| import sentencepiece as spm | |
| from warpstate import WarpStateConfig, WarpStateLM | |
| if not torch.cuda.is_available() or not torch.cuda.is_bf16_supported(): | |
| raise RuntimeError('Install a CUDA 13.0-compatible NVIDIA driver; BF16 CUDA is required') | |
| if not a.test_small_gpu and torch.cuda.get_device_properties(a.local_rank).total_memory < 80*1024**3: | |
| raise RuntimeError('The normal preset requires the 96 GB GPU') | |
| latest = a.work_dir/'run/latest.pt' | |
| source = latest if latest.exists() else a.weights | |
| fresh = not latest.exists() and a.from_scratch | |
| checkpoint = {} if fresh else torch.load(source, map_location='cpu', mmap=True, weights_only=False) | |
| if not fresh and 'model' not in checkpoint: | |
| raise ValueError('Expected a WarpState checkpoint with a model state dictionary, not arbitrary model weights') | |
| cfg = WarpStateConfig(**checkpoint.get('config',json.loads(a.config.read_text()))) | |
| cfg.validate() | |
| if cfg.vocab_size != spm.SentencePieceProcessor(model_file=str(a.tokenizer)).vocab_size(): | |
| raise ValueError('Tokenizer/model vocabulary mismatch') | |
| if a.seq_len % cfg.chunk_size: | |
| raise ValueError('Sequence length must be divisible by the local tile size') | |
| with torch.device('meta'): | |
| probe = WarpStateLM(cfg) | |
| if not fresh: | |
| probe.load_state_dict(checkpoint['model'], assign=True, strict=True) | |
| if not a.test_small_gpu and not 2_000_000_000 <= probe.num_parameters() <= 2_020_000_000: | |
| raise ValueError('This package expects the approximately 2B WarpState architecture') | |
| optimizer_state = checkpoint.get('optimizer', {}) | |
| if optimizer_state.get('kind') == 'cpu_offload_adamw': | |
| for parameter, master in zip(probe.parameters(), optimizer_state['masters'], strict=True): | |
| if parameter.shape != master.shape: | |
| raise ValueError('FP32 master weight shape mismatch') | |
| if latest.exists() and checkpoint.get('format') != 'warpstate-server-v1': | |
| raise ValueError('Use --weights to initialize a new server run; do not place a laptop checkpoint in work/run') | |
| print(f'Preflight: compatible WarpState {"configuration for fresh initialization" if fresh else "weights"} ' | |
| f'({probe.num_parameters():,} parameters) and CUDA GPU.', flush=True) | |
| def main(): | |
| a = arguments() | |
| if not a.tokenizer.is_file(): | |
| raise FileNotFoundError(a.tokenizer) | |
| if (a.stage != 'prepare' and not (a.work_dir/'run/latest.pt').exists() and not a.from_scratch | |
| and (a.weights is None or not a.weights.is_file())): | |
| raise ValueError('Supply --weights or explicitly choose --from-scratch before training') | |
| bootstrap(a) | |
| a.work_dir.mkdir(parents=True,exist_ok=True) | |
| os.environ.setdefault('HF_HOME',str(a.work_dir/'hf-cache')) | |
| os.environ.setdefault('TORCHINDUCTOR_CACHE_DIR',str(a.work_dir/'compile-cache')) | |
| signals = [signal.SIGINT, signal.SIGTERM] | |
| if hasattr(signal, 'SIGUSR1'): | |
| signals.append(signal.SIGUSR1) | |
| for sig in signals: | |
| signal.signal(sig,request_stop) | |
| distributed = a.world_size > 1 | |
| if a.stage != 'prepare': | |
| import torch | |
| torch.cuda.set_device(a.local_rank) | |
| configure_memory(a) | |
| if distributed: | |
| import torch.distributed as dist | |
| dist.init_process_group('nccl', device_id=torch.device('cuda', a.local_rank)) | |
| try: | |
| with contextlib.ExitStack() as stack: | |
| if a.rank == 0: | |
| stack.enter_context(process_lock(a.work_dir/'pipeline.lock')) | |
| if distributed: | |
| dist.barrier() | |
| if a.stage != 'prepare': | |
| preflight(a) | |
| if a.stage in ('all','prepare') and not prepare(a): | |
| return | |
| if a.stage in ('all','train'): | |
| train(a) | |
| finally: | |
| if distributed: | |
| dist.destroy_process_group() | |
| if __name__ == '__main__': | |
| main() | |