Download nanodex/trainer.py from SLM-Archive/hyperdex-trainer: direct link, hf CLI and curl.
- Browser
- Download file 10 kB
-
https://huggingface.co/SLM-Archive/hyperdex-trainer/resolve/main/nanodex/trainer.py
- Command line
-
hf download hf://SLM-Archive/hyperdex-trainer/nanodex/trainer.py
-
curl -L -o trainer.py https://huggingface.co/SLM-Archive/hyperdex-trainer/resolve/main/nanodex/trainer.py
10 kB
| """Pre-training loop for a single NanoDex run.""" | |
| import json | |
| import math | |
| import os | |
| import shutil | |
| import time | |
| from . import db, store | |
| from .config import TIERS, SEQ_LEN, VOCAB_SIZE | |
| # Metric cadence adapts to run length so short runs still get a real curve. | |
| def _cadence(total): | |
| return max(1, min(25, total // 200)) | |
| def total_steps_for(tier_key, target_tokens): | |
| tier = TIERS[tier_key] | |
| return max(1, int(target_tokens // tier.batch_tokens)) | |
| def _lr_at(step, total, base_lr, warmup_frac=0.02, min_frac=0.1): | |
| warmup = max(1, int(total * warmup_frac)) | |
| if step < warmup: | |
| return base_lr * (step + 1) / warmup | |
| prog = (step - warmup) / max(1, total - warmup) | |
| prog = min(max(prog, 0.0), 1.0) | |
| return base_lr * (min_frac + (1 - min_frac) * 0.5 * (1 + math.cos(math.pi * prog))) | |
| LIVE_SNAPSHOTS = 40 # snapshots spread over a run | |
| LIVE_MIN_SECONDS = 25 # ...but never more often than this | |
| def live_dir(job_id): | |
| return os.path.join(db.RUNS_DIR, job_id, "live") | |
| def _write_live_checkpoint(model, job_id, step, tokenizer_dir, first=False): | |
| """ | |
| Dump the current weights so the run page can generate text from the model | |
| mid-training. Written to a temp file and renamed, so a reader never sees a | |
| half-written checkpoint. | |
| """ | |
| import torch | |
| from safetensors.torch import save_file | |
| d = live_dir(job_id) | |
| os.makedirs(d, exist_ok=True) | |
| if first: | |
| cfg = model.config.to_dict() | |
| cfg["use_cache"] = True | |
| cfg["torch_dtype"] = "float32" | |
| with open(os.path.join(d, "config.json"), "w") as f: | |
| json.dump(cfg, f, indent=2) | |
| with open(os.path.join(d, "generation_config.json"), "w") as f: | |
| json.dump({"bos_token_id": 0, "eos_token_id": 0, | |
| "pad_token_id": 1}, f, indent=2) | |
| for fn in ("tokenizer.json", "tokenizer_config.json", | |
| "special_tokens_map.json"): | |
| src = os.path.join(tokenizer_dir, fn) | |
| if os.path.exists(src): | |
| shutil.copy(src, os.path.join(d, fn)) | |
| sd = {k: v.detach().to("cpu", torch.float32).contiguous() | |
| for k, v in model.state_dict().items() | |
| if not k.endswith("lm_head.weight")} # tied to the embedding | |
| tmp = os.path.join(d, "model.safetensors.tmp") | |
| save_file(sd, tmp, metadata={"format": "pt", "step": str(step)}) | |
| os.replace(tmp, os.path.join(d, "model.safetensors")) | |
| db.update_job(job_id, live_step=step, live_at=time.time()) | |
| def build_model(tier_key, vocab_size=VOCAB_SIZE): | |
| import torch | |
| try: | |
| # transformers >= 5 dropped the top-level re-exports | |
| from transformers.models.llama import LlamaConfig, LlamaForCausalLM | |
| except ImportError: | |
| from transformers import LlamaConfig, LlamaForCausalLM | |
| tier = TIERS[tier_key] | |
| cfg = LlamaConfig(**tier.to_hf_config_dict(vocab_size)) | |
| cfg.use_cache = False | |
| model = LlamaForCausalLM(cfg) | |
| return model | |
| def run_training(job, device, tokenizer_dir, cancel_flag, log): | |
| """ | |
| Execute one job to completion. `cancel_flag` is a callable returning True | |
| when the user asked to stop. Returns the output directory. | |
| """ | |
| import torch | |
| from .data import TokenStream | |
| job_id = job["id"] | |
| tier = TIERS[job["tier"]] | |
| target_tokens = int(job["target_tokens"]) | |
| total = total_steps_for(job["tier"], target_tokens) | |
| seq_len = SEQ_LEN | |
| use_cuda = device.startswith("cuda") | |
| torch.manual_seed(int(job.get("seed") or 1337)) | |
| model = build_model(job["tier"]).to(device) | |
| n_params = sum(p.numel() for p in model.parameters()) | |
| log(f"model: {tier.label} · {n_params:,} parameters · device={device}") | |
| if use_cuda: | |
| torch.backends.cuda.matmul.allow_tf32 = True | |
| torch.backends.cudnn.allow_tf32 = True | |
| micro = tier.micro_batch if use_cuda else max(4, tier.micro_batch // 32) | |
| tokens_per_micro = micro * seq_len | |
| accum = max(1, tier.batch_tokens // tokens_per_micro) | |
| log(f"batch: {micro} seq x {seq_len} tok x {accum} accum " | |
| f"= {micro * seq_len * accum:,} tokens/step · {total:,} steps") | |
| opt = torch.optim.AdamW( | |
| model.parameters(), lr=tier.lr, betas=(0.9, 0.95), | |
| weight_decay=0.1, eps=1e-8, fused=use_cuda, | |
| ) | |
| stream = TokenStream(seq_len, micro, seed=int(job.get("seed") or 1337)) | |
| amp_dtype = torch.bfloat16 if use_cuda else torch.float32 | |
| out_dir = os.path.join(db.RUNS_DIR, job_id) | |
| os.makedirs(out_dir, exist_ok=True) | |
| every = _cadence(total) | |
| live_every = max(1, total // LIVE_SNAPSHOTS) | |
| last_live = 0.0 | |
| live_started = False | |
| model.train() | |
| tokens_seen = 0 | |
| best = float("inf") | |
| t_start = time.time() | |
| t_win = time.time() | |
| tok_win = 0 | |
| for step in range(total): | |
| if cancel_flag(): | |
| log("cancelled by the user") | |
| db.update_job(job_id, status="cancelled", finished_at=time.time()) | |
| return None | |
| lr = _lr_at(step, total, tier.lr) | |
| for g in opt.param_groups: | |
| g["lr"] = lr | |
| while True: | |
| opt.zero_grad(set_to_none=True) | |
| loss_acc = 0.0 | |
| step_tokens = 0 | |
| try: | |
| for _ in range(accum): | |
| x = stream.next_batch().to(device, non_blocking=True) | |
| with torch.autocast(device_type="cuda" if use_cuda else "cpu", | |
| dtype=amp_dtype, enabled=use_cuda): | |
| # labels == input_ids: the model does the causal shift | |
| # itself. Passing a pre-shifted target would train it to | |
| # predict two tokens ahead of where generation reads. | |
| out = model(input_ids=x, labels=x) | |
| loss = out.loss / accum | |
| loss.backward() | |
| loss_acc += loss.item() | |
| step_tokens += x.numel() | |
| break | |
| except torch.cuda.OutOfMemoryError: | |
| if micro <= 4: | |
| raise | |
| micro = max(4, micro // 2) | |
| accum = max(1, tier.batch_tokens // (micro * seq_len)) | |
| opt.zero_grad(set_to_none=True) | |
| torch.cuda.empty_cache() | |
| stream = TokenStream(seq_len, micro, | |
| seed=int(job.get("seed") or 1337)) | |
| log(f"out of GPU memory — retrying with {micro} seq x {accum} accum") | |
| tokens_seen += step_tokens | |
| tok_win += step_tokens | |
| torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0) | |
| opt.step() | |
| best = min(best, loss_acc) | |
| # Snapshot for the in-page playground. | |
| if (step % live_every == 0 or step == total - 1) and \ | |
| (time.time() - last_live > LIVE_MIN_SECONDS or step == total - 1): | |
| try: | |
| _write_live_checkpoint(model, job_id, step, tokenizer_dir, | |
| first=not live_started) | |
| live_started = True | |
| last_live = time.time() | |
| except Exception as exc: | |
| log(f"live checkpoint skipped: {exc!r}") | |
| if step % every == 0 or step == total - 1: | |
| now = time.time() | |
| rate = tok_win / max(now - t_win, 1e-6) | |
| t_win, tok_win = now, 0 | |
| eta = (target_tokens - tokens_seen) / max(rate, 1.0) | |
| db.add_metric(job_id, step, tokens_seen, loss_acc, lr) | |
| db.update_job( | |
| job_id, step=step, tokens_seen=tokens_seen, loss=loss_acc, | |
| best_loss=best, lr=lr, tok_per_s=rate, eta_s=eta, | |
| ) | |
| if step % (every * 4) == 0 or step == total - 1: | |
| log(f"step {step:>6,}/{total:,} · loss {loss_acc:.4f} · " | |
| f"ppl {math.exp(min(loss_acc, 20)):.1f} · lr {lr:.2e} · " | |
| f"{rate / 1e3:.0f}k tok/s · ETA {eta / 60:.1f} min") | |
| # ---- save ---- | |
| log("saving checkpoint...") | |
| model.config.use_cache = True | |
| model = model.float().cpu() | |
| model.save_pretrained(out_dir, safe_serialization=True) | |
| for fn in ("tokenizer.json", "tokenizer_config.json", "special_tokens_map.json"): | |
| src = os.path.join(tokenizer_dir, fn) | |
| if os.path.exists(src): | |
| shutil.copy(src, os.path.join(out_dir, fn)) | |
| with open(os.path.join(out_dir, "generation_config.json"), "w") as f: | |
| json.dump({"bos_token_id": 0, "eos_token_id": 0, "pad_token_id": 1, | |
| "_from_model_config": True}, f, indent=2) | |
| with open(os.path.join(out_dir, "training_run.json"), "w") as f: | |
| json.dump({ | |
| "job_id": job_id, "tier": job["tier"], "label": tier.label, | |
| "n_params": n_params, "target_tokens": target_tokens, | |
| "tokens_seen": tokens_seen, "steps": total, "seq_len": seq_len, | |
| "final_loss": loss_acc, "best_loss": best, | |
| "peak_lr": tier.lr, "batch_tokens": tier.batch_tokens, | |
| "optimizer": "AdamW(0.9, 0.95) wd=0.1 clip=1.0", | |
| "schedule": "warmup 2% + cosine to 10%", | |
| "dataset": "HuggingFaceFW/fineweb-edu (sample-10BT)", | |
| "architecture": "LlamaForCausalLM (SiLU, RMSNorm, RoPE, GQA, tied embeddings)", | |
| "wall_time_s": time.time() - t_start, | |
| "trained_by": job["username"], | |
| }, f, indent=2) | |
| db.update_job(job_id, status="done", finished_at=time.time(), | |
| tokens_seen=tokens_seen, loss=loss_acc, best_loss=best, | |
| step=total, eta_s=0) | |
| # Mirror the finished run to the bucket so it survives the next rebuild. | |
| try: | |
| payload = db.export_run(job_id) | |
| if payload: | |
| store.save_run(job_id, out_dir, payload, log) | |
| except Exception as exc: | |
| log(f"could not archive the run to durable storage: {exc!r}") | |
| log(f"training finished in {(time.time() - t_start) / 60:.1f} min · " | |
| f"final loss {loss_acc:.4f}") | |
| return out_dir | |