hyperdex-trainer / nanodex /trainer.py
GGUFGuy's picture
Duplicate from GGUFGuy/hyperdex-trainer
2a49d9a
Raw History Blame Contribute Delete
10 kB
"""Pre-training loop for a single NanoDex run."""
import json
import math
import os
import shutil
import time
from . import db, store
from .config import TIERS, SEQ_LEN, VOCAB_SIZE
# Metric cadence adapts to run length so short runs still get a real curve.
def _cadence(total):
return max(1, min(25, total // 200))
def total_steps_for(tier_key, target_tokens):
tier = TIERS[tier_key]
return max(1, int(target_tokens // tier.batch_tokens))
def _lr_at(step, total, base_lr, warmup_frac=0.02, min_frac=0.1):
warmup = max(1, int(total * warmup_frac))
if step < warmup:
return base_lr * (step + 1) / warmup
prog = (step - warmup) / max(1, total - warmup)
prog = min(max(prog, 0.0), 1.0)
return base_lr * (min_frac + (1 - min_frac) * 0.5 * (1 + math.cos(math.pi * prog)))
LIVE_SNAPSHOTS = 40 # snapshots spread over a run
LIVE_MIN_SECONDS = 25 # ...but never more often than this
def live_dir(job_id):
return os.path.join(db.RUNS_DIR, job_id, "live")
def _write_live_checkpoint(model, job_id, step, tokenizer_dir, first=False):
"""
Dump the current weights so the run page can generate text from the model
mid-training. Written to a temp file and renamed, so a reader never sees a
half-written checkpoint.
"""
import torch
from safetensors.torch import save_file
d = live_dir(job_id)
os.makedirs(d, exist_ok=True)
if first:
cfg = model.config.to_dict()
cfg["use_cache"] = True
cfg["torch_dtype"] = "float32"
with open(os.path.join(d, "config.json"), "w") as f:
json.dump(cfg, f, indent=2)
with open(os.path.join(d, "generation_config.json"), "w") as f:
json.dump({"bos_token_id": 0, "eos_token_id": 0,
"pad_token_id": 1}, f, indent=2)
for fn in ("tokenizer.json", "tokenizer_config.json",
"special_tokens_map.json"):
src = os.path.join(tokenizer_dir, fn)
if os.path.exists(src):
shutil.copy(src, os.path.join(d, fn))
sd = {k: v.detach().to("cpu", torch.float32).contiguous()
for k, v in model.state_dict().items()
if not k.endswith("lm_head.weight")} # tied to the embedding
tmp = os.path.join(d, "model.safetensors.tmp")
save_file(sd, tmp, metadata={"format": "pt", "step": str(step)})
os.replace(tmp, os.path.join(d, "model.safetensors"))
db.update_job(job_id, live_step=step, live_at=time.time())
def build_model(tier_key, vocab_size=VOCAB_SIZE):
import torch
try:
# transformers >= 5 dropped the top-level re-exports
from transformers.models.llama import LlamaConfig, LlamaForCausalLM
except ImportError:
from transformers import LlamaConfig, LlamaForCausalLM
tier = TIERS[tier_key]
cfg = LlamaConfig(**tier.to_hf_config_dict(vocab_size))
cfg.use_cache = False
model = LlamaForCausalLM(cfg)
return model
def run_training(job, device, tokenizer_dir, cancel_flag, log):
"""
Execute one job to completion. `cancel_flag` is a callable returning True
when the user asked to stop. Returns the output directory.
"""
import torch
from .data import TokenStream
job_id = job["id"]
tier = TIERS[job["tier"]]
target_tokens = int(job["target_tokens"])
total = total_steps_for(job["tier"], target_tokens)
seq_len = SEQ_LEN
use_cuda = device.startswith("cuda")
torch.manual_seed(int(job.get("seed") or 1337))
model = build_model(job["tier"]).to(device)
n_params = sum(p.numel() for p in model.parameters())
log(f"model: {tier.label} · {n_params:,} parameters · device={device}")
if use_cuda:
torch.backends.cuda.matmul.allow_tf32 = True
torch.backends.cudnn.allow_tf32 = True
micro = tier.micro_batch if use_cuda else max(4, tier.micro_batch // 32)
tokens_per_micro = micro * seq_len
accum = max(1, tier.batch_tokens // tokens_per_micro)
log(f"batch: {micro} seq x {seq_len} tok x {accum} accum "
f"= {micro * seq_len * accum:,} tokens/step · {total:,} steps")
opt = torch.optim.AdamW(
model.parameters(), lr=tier.lr, betas=(0.9, 0.95),
weight_decay=0.1, eps=1e-8, fused=use_cuda,
)
stream = TokenStream(seq_len, micro, seed=int(job.get("seed") or 1337))
amp_dtype = torch.bfloat16 if use_cuda else torch.float32
out_dir = os.path.join(db.RUNS_DIR, job_id)
os.makedirs(out_dir, exist_ok=True)
every = _cadence(total)
live_every = max(1, total // LIVE_SNAPSHOTS)
last_live = 0.0
live_started = False
model.train()
tokens_seen = 0
best = float("inf")
t_start = time.time()
t_win = time.time()
tok_win = 0
for step in range(total):
if cancel_flag():
log("cancelled by the user")
db.update_job(job_id, status="cancelled", finished_at=time.time())
return None
lr = _lr_at(step, total, tier.lr)
for g in opt.param_groups:
g["lr"] = lr
while True:
opt.zero_grad(set_to_none=True)
loss_acc = 0.0
step_tokens = 0
try:
for _ in range(accum):
x = stream.next_batch().to(device, non_blocking=True)
with torch.autocast(device_type="cuda" if use_cuda else "cpu",
dtype=amp_dtype, enabled=use_cuda):
# labels == input_ids: the model does the causal shift
# itself. Passing a pre-shifted target would train it to
# predict two tokens ahead of where generation reads.
out = model(input_ids=x, labels=x)
loss = out.loss / accum
loss.backward()
loss_acc += loss.item()
step_tokens += x.numel()
break
except torch.cuda.OutOfMemoryError:
if micro <= 4:
raise
micro = max(4, micro // 2)
accum = max(1, tier.batch_tokens // (micro * seq_len))
opt.zero_grad(set_to_none=True)
torch.cuda.empty_cache()
stream = TokenStream(seq_len, micro,
seed=int(job.get("seed") or 1337))
log(f"out of GPU memory — retrying with {micro} seq x {accum} accum")
tokens_seen += step_tokens
tok_win += step_tokens
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
opt.step()
best = min(best, loss_acc)
# Snapshot for the in-page playground.
if (step % live_every == 0 or step == total - 1) and \
(time.time() - last_live > LIVE_MIN_SECONDS or step == total - 1):
try:
_write_live_checkpoint(model, job_id, step, tokenizer_dir,
first=not live_started)
live_started = True
last_live = time.time()
except Exception as exc:
log(f"live checkpoint skipped: {exc!r}")
if step % every == 0 or step == total - 1:
now = time.time()
rate = tok_win / max(now - t_win, 1e-6)
t_win, tok_win = now, 0
eta = (target_tokens - tokens_seen) / max(rate, 1.0)
db.add_metric(job_id, step, tokens_seen, loss_acc, lr)
db.update_job(
job_id, step=step, tokens_seen=tokens_seen, loss=loss_acc,
best_loss=best, lr=lr, tok_per_s=rate, eta_s=eta,
)
if step % (every * 4) == 0 or step == total - 1:
log(f"step {step:>6,}/{total:,} · loss {loss_acc:.4f} · "
f"ppl {math.exp(min(loss_acc, 20)):.1f} · lr {lr:.2e} · "
f"{rate / 1e3:.0f}k tok/s · ETA {eta / 60:.1f} min")
# ---- save ----
log("saving checkpoint...")
model.config.use_cache = True
model = model.float().cpu()
model.save_pretrained(out_dir, safe_serialization=True)
for fn in ("tokenizer.json", "tokenizer_config.json", "special_tokens_map.json"):
src = os.path.join(tokenizer_dir, fn)
if os.path.exists(src):
shutil.copy(src, os.path.join(out_dir, fn))
with open(os.path.join(out_dir, "generation_config.json"), "w") as f:
json.dump({"bos_token_id": 0, "eos_token_id": 0, "pad_token_id": 1,
"_from_model_config": True}, f, indent=2)
with open(os.path.join(out_dir, "training_run.json"), "w") as f:
json.dump({
"job_id": job_id, "tier": job["tier"], "label": tier.label,
"n_params": n_params, "target_tokens": target_tokens,
"tokens_seen": tokens_seen, "steps": total, "seq_len": seq_len,
"final_loss": loss_acc, "best_loss": best,
"peak_lr": tier.lr, "batch_tokens": tier.batch_tokens,
"optimizer": "AdamW(0.9, 0.95) wd=0.1 clip=1.0",
"schedule": "warmup 2% + cosine to 10%",
"dataset": "HuggingFaceFW/fineweb-edu (sample-10BT)",
"architecture": "LlamaForCausalLM (SiLU, RMSNorm, RoPE, GQA, tied embeddings)",
"wall_time_s": time.time() - t_start,
"trained_by": job["username"],
}, f, indent=2)
db.update_job(job_id, status="done", finished_at=time.time(),
tokens_seen=tokens_seen, loss=loss_acc, best_loss=best,
step=total, eta_s=0)
# Mirror the finished run to the bucket so it survives the next rebuild.
try:
payload = db.export_run(job_id)
if payload:
store.save_run(job_id, out_dir, payload, log)
except Exception as exc:
log(f"could not archive the run to durable storage: {exc!r}")
log(f"training finished in {(time.time() - t_start) / 60:.1f} min · "
f"final loss {loss_acc:.4f}")
return out_dir