"""Pre-training loop for a single NanoDex run.""" import json import math import os import shutil import time from . import db, store from .config import TIERS, SEQ_LEN, VOCAB_SIZE # Metric cadence adapts to run length so short runs still get a real curve. def _cadence(total): return max(1, min(25, total // 200)) def total_steps_for(tier_key, target_tokens): tier = TIERS[tier_key] return max(1, int(target_tokens // tier.batch_tokens)) def _lr_at(step, total, base_lr, warmup_frac=0.02, min_frac=0.1): warmup = max(1, int(total * warmup_frac)) if step < warmup: return base_lr * (step + 1) / warmup prog = (step - warmup) / max(1, total - warmup) prog = min(max(prog, 0.0), 1.0) return base_lr * (min_frac + (1 - min_frac) * 0.5 * (1 + math.cos(math.pi * prog))) LIVE_SNAPSHOTS = 40 # snapshots spread over a run LIVE_MIN_SECONDS = 25 # ...but never more often than this def live_dir(job_id): return os.path.join(db.RUNS_DIR, job_id, "live") def _write_live_checkpoint(model, job_id, step, tokenizer_dir, first=False): """ Dump the current weights so the run page can generate text from the model mid-training. Written to a temp file and renamed, so a reader never sees a half-written checkpoint. """ import torch from safetensors.torch import save_file d = live_dir(job_id) os.makedirs(d, exist_ok=True) if first: cfg = model.config.to_dict() cfg["use_cache"] = True cfg["torch_dtype"] = "float32" with open(os.path.join(d, "config.json"), "w") as f: json.dump(cfg, f, indent=2) with open(os.path.join(d, "generation_config.json"), "w") as f: json.dump({"bos_token_id": 0, "eos_token_id": 0, "pad_token_id": 1}, f, indent=2) for fn in ("tokenizer.json", "tokenizer_config.json", "special_tokens_map.json"): src = os.path.join(tokenizer_dir, fn) if os.path.exists(src): shutil.copy(src, os.path.join(d, fn)) sd = {k: v.detach().to("cpu", torch.float32).contiguous() for k, v in model.state_dict().items() if not k.endswith("lm_head.weight")} # tied to the embedding tmp = os.path.join(d, "model.safetensors.tmp") save_file(sd, tmp, metadata={"format": "pt", "step": str(step)}) os.replace(tmp, os.path.join(d, "model.safetensors")) db.update_job(job_id, live_step=step, live_at=time.time()) def build_model(tier_key, vocab_size=VOCAB_SIZE): import torch try: # transformers >= 5 dropped the top-level re-exports from transformers.models.llama import LlamaConfig, LlamaForCausalLM except ImportError: from transformers import LlamaConfig, LlamaForCausalLM tier = TIERS[tier_key] cfg = LlamaConfig(**tier.to_hf_config_dict(vocab_size)) cfg.use_cache = False model = LlamaForCausalLM(cfg) return model def run_training(job, device, tokenizer_dir, cancel_flag, log): """ Execute one job to completion. `cancel_flag` is a callable returning True when the user asked to stop. Returns the output directory. """ import torch from .data import TokenStream job_id = job["id"] tier = TIERS[job["tier"]] target_tokens = int(job["target_tokens"]) total = total_steps_for(job["tier"], target_tokens) seq_len = SEQ_LEN use_cuda = device.startswith("cuda") torch.manual_seed(int(job.get("seed") or 1337)) model = build_model(job["tier"]).to(device) n_params = sum(p.numel() for p in model.parameters()) log(f"model: {tier.label} · {n_params:,} parameters · device={device}") if use_cuda: torch.backends.cuda.matmul.allow_tf32 = True torch.backends.cudnn.allow_tf32 = True micro = tier.micro_batch if use_cuda else max(4, tier.micro_batch // 32) tokens_per_micro = micro * seq_len accum = max(1, tier.batch_tokens // tokens_per_micro) log(f"batch: {micro} seq x {seq_len} tok x {accum} accum " f"= {micro * seq_len * accum:,} tokens/step · {total:,} steps") opt = torch.optim.AdamW( model.parameters(), lr=tier.lr, betas=(0.9, 0.95), weight_decay=0.1, eps=1e-8, fused=use_cuda, ) stream = TokenStream(seq_len, micro, seed=int(job.get("seed") or 1337)) amp_dtype = torch.bfloat16 if use_cuda else torch.float32 out_dir = os.path.join(db.RUNS_DIR, job_id) os.makedirs(out_dir, exist_ok=True) every = _cadence(total) live_every = max(1, total // LIVE_SNAPSHOTS) last_live = 0.0 live_started = False model.train() tokens_seen = 0 best = float("inf") t_start = time.time() t_win = time.time() tok_win = 0 for step in range(total): if cancel_flag(): log("cancelled by the user") db.update_job(job_id, status="cancelled", finished_at=time.time()) return None lr = _lr_at(step, total, tier.lr) for g in opt.param_groups: g["lr"] = lr while True: opt.zero_grad(set_to_none=True) loss_acc = 0.0 step_tokens = 0 try: for _ in range(accum): x = stream.next_batch().to(device, non_blocking=True) with torch.autocast(device_type="cuda" if use_cuda else "cpu", dtype=amp_dtype, enabled=use_cuda): # labels == input_ids: the model does the causal shift # itself. Passing a pre-shifted target would train it to # predict two tokens ahead of where generation reads. out = model(input_ids=x, labels=x) loss = out.loss / accum loss.backward() loss_acc += loss.item() step_tokens += x.numel() break except torch.cuda.OutOfMemoryError: if micro <= 4: raise micro = max(4, micro // 2) accum = max(1, tier.batch_tokens // (micro * seq_len)) opt.zero_grad(set_to_none=True) torch.cuda.empty_cache() stream = TokenStream(seq_len, micro, seed=int(job.get("seed") or 1337)) log(f"out of GPU memory — retrying with {micro} seq x {accum} accum") tokens_seen += step_tokens tok_win += step_tokens torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0) opt.step() best = min(best, loss_acc) # Snapshot for the in-page playground. if (step % live_every == 0 or step == total - 1) and \ (time.time() - last_live > LIVE_MIN_SECONDS or step == total - 1): try: _write_live_checkpoint(model, job_id, step, tokenizer_dir, first=not live_started) live_started = True last_live = time.time() except Exception as exc: log(f"live checkpoint skipped: {exc!r}") if step % every == 0 or step == total - 1: now = time.time() rate = tok_win / max(now - t_win, 1e-6) t_win, tok_win = now, 0 eta = (target_tokens - tokens_seen) / max(rate, 1.0) db.add_metric(job_id, step, tokens_seen, loss_acc, lr) db.update_job( job_id, step=step, tokens_seen=tokens_seen, loss=loss_acc, best_loss=best, lr=lr, tok_per_s=rate, eta_s=eta, ) if step % (every * 4) == 0 or step == total - 1: log(f"step {step:>6,}/{total:,} · loss {loss_acc:.4f} · " f"ppl {math.exp(min(loss_acc, 20)):.1f} · lr {lr:.2e} · " f"{rate / 1e3:.0f}k tok/s · ETA {eta / 60:.1f} min") # ---- save ---- log("saving checkpoint...") model.config.use_cache = True model = model.float().cpu() model.save_pretrained(out_dir, safe_serialization=True) for fn in ("tokenizer.json", "tokenizer_config.json", "special_tokens_map.json"): src = os.path.join(tokenizer_dir, fn) if os.path.exists(src): shutil.copy(src, os.path.join(out_dir, fn)) with open(os.path.join(out_dir, "generation_config.json"), "w") as f: json.dump({"bos_token_id": 0, "eos_token_id": 0, "pad_token_id": 1, "_from_model_config": True}, f, indent=2) with open(os.path.join(out_dir, "training_run.json"), "w") as f: json.dump({ "job_id": job_id, "tier": job["tier"], "label": tier.label, "n_params": n_params, "target_tokens": target_tokens, "tokens_seen": tokens_seen, "steps": total, "seq_len": seq_len, "final_loss": loss_acc, "best_loss": best, "peak_lr": tier.lr, "batch_tokens": tier.batch_tokens, "optimizer": "AdamW(0.9, 0.95) wd=0.1 clip=1.0", "schedule": "warmup 2% + cosine to 10%", "dataset": "HuggingFaceFW/fineweb-edu (sample-10BT)", "architecture": "LlamaForCausalLM (SiLU, RMSNorm, RoPE, GQA, tied embeddings)", "wall_time_s": time.time() - t_start, "trained_by": job["username"], }, f, indent=2) db.update_job(job_id, status="done", finished_at=time.time(), tokens_seen=tokens_seen, loss=loss_acc, best_loss=best, step=total, eta_s=0) # Mirror the finished run to the bucket so it survives the next rebuild. try: payload = db.export_run(job_id) if payload: store.save_run(job_id, out_dir, payload, log) except Exception as exc: log(f"could not archive the run to durable storage: {exc!r}") log(f"training finished in {(time.time() - t_start) / 60:.1f} min · " f"final loss {loss_acc:.4f}") return out_dir